diff --git a/.config/markdownlint.jsonc b/.config/markdownlint.jsonc index f2c5d54..5a26c2f 100644 --- a/.config/markdownlint.jsonc +++ b/.config/markdownlint.jsonc @@ -11,6 +11,9 @@ "MD033": false, // First line need not be a top-level heading. "MD041": false, + // Duplicate headings are fine under DIFFERENT parents (e.g. a "Reach Green" + // step repeated per worked example); only same-level siblings must differ. + "MD024": { "siblings_only": true }, // Emphasis / strong markers: asterisk (deliberate preference; the table // formatter never touches emphasis, so there is no underscore conflict). diff --git a/.config/scripts/Initialize-DevEnvironment.Tests.ps1 b/.config/scripts/Initialize-DevEnvironment.Tests.ps1 index d2a5342..c92a21a 100644 --- a/.config/scripts/Initialize-DevEnvironment.Tests.ps1 +++ b/.config/scripts/Initialize-DevEnvironment.Tests.ps1 @@ -16,6 +16,8 @@ Describe 'Invoke-Initialize' -Tag 'Fast' { Mock Invoke-DevModuleInstall { } Mock Invoke-PreCommitInstall { } Mock Write-Information { } + Mock Test-SymlinkCapability { $true } + Mock git { 'true' } # core.symlinks } It 'returns 1 when git is missing' { @@ -44,4 +46,28 @@ Describe 'Invoke-Initialize' -Tag 'Fast' { Invoke-Initialize | Out-Null Should -Invoke Invoke-DevModuleInstall -Times 0 } + + It 'warns when the machine cannot create symlinks (Developer Mode off)' { + Mock Test-SymlinkCapability { $false } + Invoke-Initialize | Out-Null + Should -Invoke Write-Warning -ParameterFilter { $Message -match 'Developer Mode' } + } + + It 'warns when git core.symlinks is explicitly false' { + Mock git { 'false' } + Invoke-Initialize | Out-Null + Should -Invoke Write-Warning -ParameterFilter { $Message -match 'core\.symlinks' } + } + + It 'stays quiet about symlinks when the machine and git are capable' { + Invoke-Initialize | Out-Null + Should -Invoke Write-Warning -ParameterFilter { $Message -match 'symlink' } -Exactly -Times 0 + } +} + +Describe 'Test-SymlinkCapability' -Tag 'Fast' { + It 'probes an actual symlink creation and reports a boolean, never throws' { + # Machine-dependent result (Developer Mode / OS); the contract is the type. + Test-SymlinkCapability | Should -BeOfType [bool] + } } diff --git a/.config/scripts/Initialize-DevEnvironment.ps1 b/.config/scripts/Initialize-DevEnvironment.ps1 index 07c5f59..bd7b5e2 100644 --- a/.config/scripts/Initialize-DevEnvironment.ps1 +++ b/.config/scripts/Initialize-DevEnvironment.ps1 @@ -35,6 +35,20 @@ function Test-Elevated { ([Security.Principal.WindowsPrincipal]$id).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) } +function Test-SymlinkCapability { + <# Probe an actual symlink creation, not a settings query — Developer Mode / + SeCreateSymbolicLink can only be proven by trying. #> + $target = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) + $link = "$target.link" + try { + New-Item -ItemType File -Path $target -ErrorAction Stop | Out-Null + New-Item -ItemType SymbolicLink -Path $link -Target $target -ErrorAction Stop | Out-Null + $true + } + catch { $false } + finally { Remove-Item $link, $target -Force -ErrorAction SilentlyContinue } +} + function Invoke-DevModuleInstall { param([string]$Name, [string]$MinimumVersion) Install-Module -Name $Name -MinimumVersion $MinimumVersion -Scope CurrentUser -Force -Repository PSGallery @@ -76,6 +90,15 @@ function Invoke-Initialize { Write-Warning 'This shell is ELEVATED (admin). pre-commit will fail to fetch hooks. Close it and run VS Code / your terminal WITHOUT admin rights.' } + # Symlinked files (see AGENTS.md "Reuse") silently check out as plain text + # when the machine or git can't do symlinks — warn, don't fail. + if (-not (Test-SymlinkCapability)) { + Write-Warning 'Symlinks cannot be created here. On Windows, enable Developer Mode (Settings -> For developers) — do NOT use an elevated shell. Symlinked repo files will check out as plain text until then.' + } + elseif ((git config core.symlinks) -eq 'false') { + Write-Warning 'git config core.symlinks is false — git will check symlinks out as plain text files. Run: git config core.symlinks true (then re-checkout affected files).' + } + if (-not (Test-Command uv)) { Write-Error @' uv is not installed. It manages pre-commit in isolation. diff --git a/.config/scripts/Invoke-TestLane.Tests.ps1 b/.config/scripts/Invoke-TestLane.Tests.ps1 index e0a5a30..6640331 100644 --- a/.config/scripts/Invoke-TestLane.Tests.ps1 +++ b/.config/scripts/Invoke-TestLane.Tests.ps1 @@ -50,4 +50,24 @@ Describe 'Invoke-TestLane (isolated)' -Tag 'Standard' { Set-Content -LiteralPath (Join-Path $testsDir 'Dummy.Tests.ps1') Invoke-ScriptFile -Path $iso.Script -Arguments @('-Lane', 'Fast') | Should -Not -Be 0 } + + It 'exposes no pre-5 Pester to the run (legacy module path scrubbed)' -Skip:(-not $HasPester) { + # A legacy Pester on the Windows PowerShell path auto-loads when a test + # calls a Pester-3-only command (e.g. Assert-MockCalled), silently + # corrupting discovery of sibling files. The lane must expose only the + # pinned Pester so such a command fails loudly instead. + $iso = Get-IsolatedScript + $testsDir = Join-Path $iso.Root 'tests' + New-Item -ItemType Directory -Path $testsDir -Force | Out-Null + @' +Describe 'pester-isolation' -Tag 'Fast' { + It 'sees no Pester older than 5 on the module path' { + $legacy = Get-Module -ListAvailable Pester | + Where-Object { $_.Version -lt [version]'5.0.0' } + $legacy | Should -BeNullOrEmpty + } +} +'@ | Set-Content -LiteralPath (Join-Path $testsDir 'Isolation.Tests.ps1') + Invoke-ScriptFile -Path $iso.Script -Arguments @('-Lane', 'Fast') | Should -Be 0 + } } diff --git a/.config/scripts/Invoke-TestLane.ps1 b/.config/scripts/Invoke-TestLane.ps1 index 9f9da00..1fe7fb0 100644 --- a/.config/scripts/Invoke-TestLane.ps1 +++ b/.config/scripts/Invoke-TestLane.ps1 @@ -26,6 +26,15 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +# Expose only this pwsh's modules to the run. Windows PowerShell ships an old +# Pester (3.x) on its module path; a test that calls a Pester-3-only command +# (e.g. Assert-MockCalled) auto-loads that legacy Pester and corrupts discovery +# of sibling files. Dropping the Windows PowerShell paths keeps core cmdlets +# intact (they live under $PSHOME) while making a stray legacy command fail +# loudly. No-op off Windows, where those paths are absent. +$env:PSModulePath = ($env:PSModulePath -split [IO.Path]::PathSeparator | + Where-Object { $_ -and $_ -notmatch 'WindowsPowerShell' }) -join [IO.Path]::PathSeparator + $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..') | Select-Object -ExpandProperty Path $testFiles = Get-ChildItem -Path $repoRoot -Recurse -File -Filter '*.Tests.ps1' -ErrorAction SilentlyContinue | diff --git a/.config/scripts/README.md b/.config/scripts/README.md index cd07901..3df59d1 100644 --- a/.config/scripts/README.md +++ b/.config/scripts/README.md @@ -58,6 +58,11 @@ cost/risk — *not* by a unit/integration label. there are none. Standard/thorough hooks ship **commented-in-place** in the pre-commit config — uncomment when you have tests for them. +Tests are **co-located** beside the script they cover (`Foo.Tests.ps1` next to +`Foo.ps1`) — the repo-wide default. The exception is [`setup/`](../../setup/), +a user-facing runbook that segregates its tests into `setup/tests/`; discovery +recurses from the repo root, so either layout runs the same. + Keep the fast (commit) lane genuinely fast, or contributors reach for `--no-verify`. The floor is pwsh + Pester warm-up (~1-3s), not your assertions. diff --git a/.config/scripts/Test-AsciiDoc.Tests.ps1 b/.config/scripts/Test-AsciiDoc.Tests.ps1 index 647a457..dc3fad8 100644 --- a/.config/scripts/Test-AsciiDoc.Tests.ps1 +++ b/.config/scripts/Test-AsciiDoc.Tests.ps1 @@ -2,7 +2,13 @@ Set-StrictMode -Version Latest # Capability flags must be set at DISCOVERY time (-Skip is evaluated then). -$HasAsciidoctor = [bool](Get-Command asciidoctor -ErrorAction SilentlyContinue) +# Probe an actual run, not just the command's existence: a Windows shim +# (asciidoctor.bat) can be on PATH while the Ruby behind it is not. +$HasAsciidoctor = $false +if (Get-Command asciidoctor -ErrorAction SilentlyContinue) { + asciidoctor --version *> $null + $HasAsciidoctor = ($LASTEXITCODE -eq 0) +} BeforeAll { $script:Script = Join-Path $PSScriptRoot 'Test-AsciiDoc.ps1' diff --git a/.config/scripts/Test-PowerShellScript.Tests.ps1 b/.config/scripts/Test-PowerShellScript.Tests.ps1 index 94d3496..57f0444 100644 --- a/.config/scripts/Test-PowerShellScript.Tests.ps1 +++ b/.config/scripts/Test-PowerShellScript.Tests.ps1 @@ -18,10 +18,18 @@ Describe 'Test-PowerShellScript' -Tag 'Fast' { Describe 'Test-PowerShellScript (PSScriptAnalyzer)' -Tag 'Standard' { It 'exits 0 for a clean script' -Skip:(-not $HasPSSA) { $f = Join-Path $TestDrive 'clean.ps1' - "Write-Output 'hello'" | Set-Content -LiteralPath $f + "Set-StrictMode -Version Latest`nWrite-Output 'hello'" | Set-Content -LiteralPath $f Invoke-ScriptFile -Path $script:Script -Arguments @($f) | Should -Be 0 } + It 'names the finding count even when there is exactly one finding' -Skip:(-not $HasPSSA) { + $f = Join-Path $TestDrive 'one.ps1' + "Set-StrictMode -Version Latest`ngci" | Set-Content -LiteralPath $f # alias -> PSAvoidUsingCmdletAliases (Warning) + $output = & pwsh -NoProfile -File $script:Script $f 2>&1 | Out-String + $LASTEXITCODE | Should -Be 1 + $output | Should -Match 'reported \d+ issue' + } + It 'exits non-zero for a Warning-level violation' -Skip:(-not $HasPSSA) { $f = Join-Path $TestDrive 'dirty.ps1' 'gci' | Set-Content -LiteralPath $f # alias -> PSAvoidUsingCmdletAliases (Warning) diff --git a/.config/scripts/Test-PowerShellScript.ps1 b/.config/scripts/Test-PowerShellScript.ps1 index 4e721fd..1858dcd 100644 --- a/.config/scripts/Test-PowerShellScript.ps1 +++ b/.config/scripts/Test-PowerShellScript.ps1 @@ -53,7 +53,7 @@ $findings = foreach ($file in $Path) { if ($findings) { $findings | Format-Table -AutoSize | Out-String -Width 4096 | Write-Output - Write-Error "PSScriptAnalyzer reported $($findings.Count) issue(s)." + Write-Error "PSScriptAnalyzer reported $(@($findings).Count) issue(s)." exit 1 } diff --git a/.gitignore b/.gitignore index 470d9fa..6c83bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,14 @@ desktop.ini # --- Editor state ------------------------------------------------------------- # Ignore personal editor config, but KEEP the shipped tooling files. -.vscode/* -!.vscode/settings.json -!.vscode/extensions.json +/.vscode/* +!/.vscode/settings.json +!/.vscode/extensions.json +!/.vscode/README.md # --- transient work (kept out of tracking for now) ---------------------------- -.temp/ -.history/ +/.temp/ +!/.temp/.gitkeep +/.history/ # --- Project-specific (add below) --------------------------------------------- diff --git a/.vscode/README.md b/.vscode/README.md new file mode 100644 index 0000000..bbe506c --- /dev/null +++ b/.vscode/README.md @@ -0,0 +1,19 @@ +# Editor integration + +VS Code workspace settings that keep the editor in sync with the same rules +`pre-commit` enforces at commit time. See [`/AGENTS.md`](../AGENTS.md) for the +overall architecture. + +## The files + +| File | Purpose | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `settings.json` | Wires live linting to `.config/` sources (e.g. `markdownlint.config` `extends` [`.config/markdownlint.jsonc`](../.config/markdownlint.jsonc)) so editor and commit agree. Deliberately tooling-only — no personal prefs (theme, fonts). | +| `extensions.json` | Recommended extensions so the linters that run at commit also surface live in the editor. VS Code prompts contributors to install these on open. | + +## Why settings.json is tracked but scoped + +`.gitignore` keeps the rest of `.vscode/` ignored (so personal workspace state +doesn't leak into commits) while explicitly tracking `settings.json` and +`extensions.json`. Keep that split: personal preferences go in the user's own +VS Code profile, not here. diff --git a/AGENTS.md b/AGENTS.md index 9bf28c6..8b00564 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ PRs, full at the merge gate. | Policy rules | `.config/PSScriptAnalyzerRules/` | [`.config/scripts/README.md`](.config/scripts/README.md) | | Opt-in tooling | `.config/overlays/` | [`.config/overlays/vale/README.md`](.config/overlays/vale/README.md) | | One-time repo setup | `setup/` (delete after use) | [`setup/README.md`](setup/README.md) | +| Optional AI skills | `setup/optional-skills/` | [`setup/README.md`](setup/README.md) | Root convention files are declarative and self-documenting: `.editorconfig` (style), `.gitattributes` (eol=lf), `.gitignore`, `.claudeignore`. @@ -43,45 +44,46 @@ root). Key principles encoded there: - **Local hooks fail fast**, never self-install. Bootstrap is explicit (`.config/scripts/Initialize-DevEnvironment.ps1`). -## Reuse: prefer a symlink over a copy - -When the same file must exist in more than one place, **symlink it** instead of -copying — one source of truth, no drift (ai-lab copied such files and they -drifted). Git tracks symlinks natively. - -- **Windows:** enable **Developer Mode** (Settings → For developers) so symlinks - materialize *without admin rights* — do **not** reach for an elevated shell - (that breaks pre-commit, see the bootstrap note). Without it, git checks - symlinks out as plain text files holding the target path. Keep - `git config core.symlinks true`. -- Reserve copies for cases where the two files are meant to diverge. +## Development workflow: TDD + +Behavioral changes to the repo's scripts are developed test-first with the +`develop-with-tdd` skill. If that skill — or an equivalent TDD skill — is +already installed in your agent, use it; there is no need for a second copy. +If none exists, use the bundled one at +[`setup/optional-skills/skills/develop-with-tdd/`](setup/optional-skills/skills/develop-with-tdd/SKILL.md): +install it into your agent's skill directory (Claude Code: `.claude/skills/` +in the repo, or `~/.claude/skills/` for all repos), or follow its `SKILL.md` +in place. The bundle is deleted with the rest of `setup/` at the end of +bootstrap — install it *before* running `Complete-Setup.ps1` if you want to +keep it. + +## Reuse: prefer an include over a copy; a symlink only when no include exists + +When the same content must exist in more than one place, in order of +preference: + +1. **Content-level include** — the format's own indirection: the `@AGENTS.md` + import in `CLAUDE.md`, `extends` in markdownlint/VS Code settings, explicit + config paths in pre-commit hook `args`. Degrades *loudly* (a missing file + errors) and works on every machine — use it whenever the format offers one. +2. **Symlink** — when the format has no include mechanism. One source of + truth, no drift (ai-lab copied such files and they drifted). Git tracks + symlinks natively, but on Windows they only materialize with **Developer + Mode** (Settings → For developers) *and* `git config core.symlinks true` — + without both, git silently checks them out as plain text files holding the + target path, and nothing errors. `Initialize-DevEnvironment.ps1` probes + symlink creation and warns when the machine can't do it. Do **not** reach + for an elevated shell as a workaround (that breaks pre-commit, see the + bootstrap note). +3. **Copy** — only when the two files are meant to diverge. ## Retrofit an existing repo -To bring another repo (e.g. one using split per-language workflows and a -hand-rolled `.githooks/` hook) up to this architecture: - -1. **Adopt the orchestrator.** Copy `.pre-commit-config.yaml` and adapt the hook - list to the repo's languages. Remove any `core.hooksPath`/`.githooks/` setup — - pre-commit replaces it. Don't set `core.hooksPath`; the two conflict. -2. **Relocate configs** into `.config/`, referenced explicitly via hook `args` - (keep root-forced files at root). Factor markdown rules into - `markdownlint.jsonc` so the editor and CLI share them. -3. **Move custom checks into shared scripts.** Any bespoke lint/test logic that - lived in CI *and* a local hook becomes one `.config/scripts/*.ps1` invoked as - a `repo: local` hook — run identically local and in CI. Delete the duplicate. -4. **Collapse CI to one workflow** that runs `pre-commit run` (scoped on - `pull_request`, full on `merge_group`). Replace N split per-language workflows; - path-filtering is no longer needed because pre-commit scopes by file. -5. **Wire the editor** (`.vscode/settings.json`) and add recommendations. -6. **Copy the docs** (this file + the per-folder READMEs), trimming to the - retrofitted repo's actual tools. -7. **Purge phantoms.** Verify every tool exists (e.g. there is **no** - `asciidoctor-lint` gem — use `asciidoctor --failure-level` for syntax + Vale - for prose). - -Anti-pattern this replaces: local hook logic and CI workflows re-implementing -"the same" checks separately, which drift apart. One orchestrator, one source. +See [`setup/MIGRATION.md`](setup/MIGRATION.md) for the full step-by-step walkthrough of +bringing this architecture into a repo not scaffolded from this template — one +step per concept in the mechanism map above, in dependency order, including +the platform limitations you'll hit along the way (e.g. `merge_queue` on a +personal/user-owned repo). ## Opening a pull request diff --git a/CLAUDE.md b/CLAUDE.md index cd076cf..0a685fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # Claude Code This repo's tooling architecture, conventions, and retrofit guide live in -[AGENTS.md](AGENTS.md) — read it first. Use the `develop-with-tdd` skill for -behavioral changes to the scripts (test-first). +[AGENTS.md](AGENTS.md) — read it first (imported below in full); it also +defines the TDD workflow and where the bundled skill lives. @AGENTS.md diff --git a/README.md b/README.md index 705fc86..0f4a26f 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,5 @@ pwsh -NoProfile -File .config/scripts/Initialize-DevEnvironment.ps1 ``` For what the tooling does and why it's shaped this way, see -[.config/README.md](.config/README.md). +[.config/README.md](.config/README.md). Migrating this tooling into a repo +that isn't scaffolded from this template? See [setup/MIGRATION.md](setup/MIGRATION.md). diff --git a/setup/AI-Maintainer-Identity.adoc b/setup/AI-Maintainer-Identity.adoc index 28f9dcc..01ef5f6 100644 --- a/setup/AI-Maintainer-Identity.adoc +++ b/setup/AI-Maintainer-Identity.adoc @@ -32,6 +32,24 @@ With that in place, the branch ruleset (`Protect-MainBranch.ps1`) is enforced *against the agents*, and `CODEOWNERS` can require your review on the control surfaces while routine agent PRs flow through CI + auto-merge. +== Which agents need an identity you create + +Only agents that run *in your shell* use a credential you provide — those are +the ones this document is for: + +* *Locally-run agents* (Claude Code CLI, Codex CLI, aider, …) inherit the + `GH_TOKEN`/git credential of the shell they run in. Create *one App (or PAT) + per agent* — separate identities keep attribution clean and let you revoke + one agent without touching the others. +* *Vendor-hosted agents* (GitHub Copilot coding agent, Codex cloud) act + through the vendor's own pre-built GitHub App. You install and repo-scope + *theirs* — you cannot substitute your own. Verify their installation is + limited to the intended repos and grants no admin. + +Verification is scriptable either way: `Test-AIMaintainerIdentity.ps1`, run +from the agent's shell, fails unless the active credential is an installation +token or fine-grained PAT with write/push access and without admin. + == Choose a mechanism [cols="1,3,3",options="header"] @@ -72,6 +90,11 @@ PAT, and never grant `Administration`. === Create +`New-AIMaintainerApp.ps1` automates this section via GitHub's app-manifest +flow — it builds the manifest below, opens the browser for the single +confirmation GitHub requires (there is no pure-API path to App creation), and +saves the private key locally. The manual path, for reference: + . *Settings → Developer settings → GitHub Apps → New GitHub App.* . Name it per agent or one shared maintainer app (e.g. `denwin-ai-maintainer`). . *Repository permissions* — grant only: @@ -150,6 +173,8 @@ Mitigate by convention: == See also +* `setup/New-AIMaintainerApp.ps1` — scripted App creation (manifest flow). +* `setup/Test-AIMaintainerIdentity.ps1` — verify an agent shell's credential; run before `Complete-Setup.ps1`. * `setup/Protect-MainBranch.ps1` — the branch ruleset the identities are enforced against. * `setup/Enable-RepoSecurity.ps1` — server-side security toggles. * `.github/CODEOWNERS` — the control surfaces that require owner review. diff --git a/setup/CHANGELOG.md b/setup/CHANGELOG.md new file mode 100644 index 0000000..10b632c --- /dev/null +++ b/setup/CHANGELOG.md @@ -0,0 +1,79 @@ +# Changelog + +Notable changes to this template, following +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The current version +is tracked in [`version`](version) and bumped in the commit that ships it — +there are no git tags or releases, so version sections carry no release date +until one is cut. Rationale lives with each mechanism's doc +([`/AGENTS.md`](../AGENTS.md), per-folder READMEs, script headers) — this file +records only what changed. + +## [0.3.0] + +### Added + +- `setup/version` and this versioned changelog. +- `setup/Test-AIMaintainerIdentity.ps1` — fails unless the agent shell's + credential is an App installation token or fine-grained PAT without admin. +- `setup/New-AIMaintainerApp.ps1` — least-privilege GitHub App creation via + the app-manifest flow (one browser confirmation; no pure-API path exists). +- `setup/Complete-Setup.ps1` — removes `setup/` and opens it as the first PR; + refuses on template repos (`isTemplate`) and dirty worktrees. +- `setup/optional-skills/` — bundled skills (`develop-with-tdd` + its TDD + knowledge base, and `changelog-entry` for adding entries to this file); + install before `Complete-Setup.ps1` only if no equivalent exists. +- Symlink capability probe + warning in `Initialize-DevEnvironment.ps1`. + +### Changed + +- Setup-time tests moved to `setup/tests/`. +- `MIGRATION.md` rewritten as an executable runbook (exports the template to + `.temp/template/`; each step has a "done when"). +- `AGENTS.md` reuse rule: content-level include > symlink > copy. +- asciidoctor capability check probes an actual run (a `.bat` shim can be on + PATH without its Ruby backend). +- `MD024` relaxed to `siblings_only` in `.config/markdownlint.jsonc`. + +### Fixed + +- `Test-PowerShellScript.ps1` crashed under StrictMode when PSScriptAnalyzer + returned exactly one finding. +- "Clean script" test fixture predated the `Measure-RequireStrictMode` rule. + +## [0.2.0] + +### Added + +- `merge_queue` rule in `Protect-MainBranch.ps1`. + +### Removed + +- `merge_queue` from this repo's live rulesets — GitHub only supports it on + org-owned repos; `DenWin/Template` is user-owned. + +### Changed + +- `Protect-MainBranch.ps1` posts one ruleset per rule so a single rejected + rule (e.g. `merge_queue`) can't block the others. + +## [0.1.0] + +### Added + +- Pre-commit orchestration: managed hooks + `repo: local` pwsh scripts under + `.config/scripts/`, shared by local hooks and CI. +- Single CI workflow running the same pre-commit hooks (diff-scoped on PRs, + full at the merge gate). +- `.vscode/` editor integration extending the same lint configs. +- Custom PSScriptAnalyzer rule `Measure-RequireStrictMode`. +- semgrep rules as an opt-in overlay (`.config/overlays/`). +- `CODEOWNERS`, issue templates, PR template with mandatory Evidence section. +- `zizmor` hook and full-history `gitleaks-action` in CI. +- `setup/`: `Protect-MainBranch.ps1`, `Enable-RepoSecurity.ps1`, + `AI-Maintainer-Identity.adoc` (replacing `Enable-MergeQueue.ps1`). + +### Changed + +- `New-IsolatedScript` renamed `Get-IsolatedScript` (query verb convention). +- Dependabot bumps for SHA-pinned third-party Actions. +- semgrep wiring reverted to overlay-only (rules kept). diff --git a/setup/Complete-Setup.ps1 b/setup/Complete-Setup.ps1 new file mode 100644 index 0000000..23c958c --- /dev/null +++ b/setup/Complete-Setup.ps1 @@ -0,0 +1,137 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS + Finish the one-time setup: remove the setup/ folder and open it as the + repo's first pull request. Refuses to run on the template repo itself. + +.DESCRIPTION + The last step after bootstrapping a repo from this template — run it once + Protect-MainBranch.ps1 and Enable-RepoSecurity.ps1 have succeeded and the + AI-maintainer identity is in place (Test-AIMaintainerIdentity.ps1 from the + agent's shell). It: + + 1. FAILSAFE: aborts if the repo is a template repo (`isTemplate`), so the + template itself never loses its setup tooling, + 2. aborts on a dirty worktree, so the PR contains only the removal, + 3. branches, `git rm -r setup`, commits, pushes, and opens a PR whose + body fills the repo's PR template (Evidence included). + + Requires the GitHub CLI, authenticated with push access, run from the + repo root. + +.PARAMETER BranchName + Branch for the removal PR. +#> +[CmdletBinding()] +param( + [string]$BranchName = 'chore/remove-setup' +) + +Set-StrictMode -Version Latest + +function Test-GhCli { [bool](Get-Command gh -ErrorAction SilentlyContinue) } + +function Test-RemovalAllowed { + <# The template failsafe (pure; no side effects): a template repo keeps its + setup/ folder — only repos CREATED from it remove theirs. #> + param([Parameter(Mandatory)][object]$RepoInfo) + if ($RepoInfo.isTemplate) { + return @{ + Allowed = $false + Reason = "$($RepoInfo.nameWithOwner) is a template repo — its setup/ folder must stay for the repos bootstrapped from it." + } + } + @{ Allowed = $true; Reason = '' } +} + +function Get-RemovalPullRequestBody { + <# PR body honoring .github/pull_request_template.md — every section + filled, Evidence mandatory (pure; no side effects). #> + @' +## Goal + +Remove the one-time `setup/` tooling — it has served its purpose: branch +protection, server-side security settings, and the AI-maintainer identity are +in place. + +## Scope + +Deletes `setup/` only. No other file changes. + +## Risk & rollback + +Low — the folder is inert documentation/tooling once setup ran; nothing +references it at runtime. Rollback: revert this PR (the folder is preserved in +history and in the upstream template). + +## Evidence + +- `setup/Protect-MainBranch.ps1` and `setup/Enable-RepoSecurity.ps1` completed + successfully against this repo (rulesets and security settings visible in + repo settings). +- `setup/Test-AIMaintainerIdentity.ps1` passed from the agent shell. +- CI (`lint`) validates this PR like any other. +'@ +} + +function Invoke-SetupRemoval { + param([string]$BranchName = 'chore/remove-setup') + + # Return codes are the contract; keep Write-Error non-terminating. + $ErrorActionPreference = 'Continue' + + if (-not (Test-GhCli)) { + Write-Error 'GitHub CLI (gh) is not installed. https://cli.github.com/' + return 1 + } + $repoInfo = gh repo view --json isTemplate,nameWithOwner 2>$null | ConvertFrom-Json + if (-not $repoInfo) { + Write-Error 'Not a GitHub repository, or gh is not authenticated (`gh auth login`).' + return 1 + } + + $verdict = Test-RemovalAllowed -RepoInfo $repoInfo + if (-not $verdict.Allowed) { + Write-Error "FAILSAFE: $($verdict.Reason)" + return 1 + } + + if (git status --porcelain) { + Write-Error 'Worktree is not clean — commit or stash first, so the removal PR contains nothing else.' + return 1 + } + + git switch -c $BranchName + if ($LASTEXITCODE -ne 0) { + Write-Error "Could not create branch '$BranchName' (does it already exist?)." + return 1 + } + git rm -r -q -- setup + if ($LASTEXITCODE -ne 0) { + Write-Error 'git rm -r setup failed — is the working directory the repo root?' + return 1 + } + git commit -m 'Remove one-time setup tooling' + if ($LASTEXITCODE -ne 0) { + Write-Error 'Commit failed.' + return 1 + } + git push -u origin $BranchName + if ($LASTEXITCODE -ne 0) { + Write-Error 'Push failed — check your remote and permissions.' + return 1 + } + $prUrl = gh pr create --title 'Remove one-time setup tooling' --body (Get-RemovalPullRequestBody) + if ($LASTEXITCODE -ne 0) { + Write-Error 'Branch pushed, but opening the PR failed — open it manually (`gh pr create`).' + return 1 + } + + Write-Information "Setup removal PR opened: $prUrl" -InformationAction Continue + return 0 +} + +# Execute only when run directly, not when dot-sourced by a test. +if ($MyInvocation.InvocationName -ne '.') { + exit (Invoke-SetupRemoval -BranchName $BranchName) +} diff --git a/setup/MIGRATION.md b/setup/MIGRATION.md new file mode 100644 index 0000000..e871664 --- /dev/null +++ b/setup/MIGRATION.md @@ -0,0 +1,134 @@ +# MIGRATION.md — bringing this tooling into a different repo + +**Executable runbook for migrating this template's tooling into a repo that +wasn't scaffolded from it** — including repos that went the wrong direction +with hand-wired custom workflows. Written so an AI agent (or a human) can work +through it step by step inside the *target* repo; every step ends with a +verifiable "done when". + +For the architecture itself, see [`/AGENTS.md`](../AGENTS.md) — read that +first. [`CHANGELOG.md`](CHANGELOG.md) has the version history. + +This file lives in `setup/` alongside the one-time setup tooling — it's only +needed during the migration, so it goes away with the rest of the folder when +`Complete-Setup.ps1` runs (step 10). + +## The concepts you're migrating + +| # | Concept | Lives in | Doc | +| - | ------------------- | -------------------------------- | ------------------------------------------------------------------------ | +| 1 | Orchestration | `.pre-commit-config.yaml` (root) | [`/AGENTS.md`](../AGENTS.md) + inline comments | +| 2 | Configs | `.config/` | [`/.config/README.md`](../.config/README.md) | +| 3 | Linting & testing | `.config/scripts/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 4 | Policy rules | `.config/PSScriptAnalyzerRules/` | [`/.config/scripts/README.md`](../.config/scripts/README.md) | +| 5 | Opt-in tooling | `.config/overlays/` | [`/.config/overlays/vale/README.md`](../.config/overlays/vale/README.md) | +| 6 | CI & automation | `.github/` | [`/.github/README.md`](../.github/README.md) | +| 7 | Editor integration | `.vscode/` | [`/.vscode/README.md`](../.vscode/README.md) | +| 8 | One-time repo setup | `setup/` (delete after use) | [`README.md`](README.md) | + +Root convention files ride along as-is and need no adaptation: +`.editorconfig`, `.gitattributes` (`eol=lf`), `.gitignore`, `.claudeignore`. + +## Migration steps + +Work through these in order — later steps depend on earlier ones (CI can't be +collapsed to one workflow until the scripts it calls exist; the merge gate +can't be enabled until CI has run once). All commands run from the target +repo's root. + +1. **Export the template as the local reference copy.** + + ```bash + gh repo clone DenWin/Template .temp/template -- --depth 1 + ``` + + Check `.temp/` is gitignored in the target *before* the clone lands; add + `/.temp/` to `.gitignore` if it isn't. Every "copy X" below means: copy + from `.temp/template/X`, then adapt. + *Done when:* `.temp/template/AGENTS.md` exists and `git status` shows no + `.temp/` entries. + +2. **Adopt the orchestrator.** Copy `.pre-commit-config.yaml` and adapt the + hook list to the target repo's languages. Remove any `core.hooksPath` / + `.githooks/` setup — pre-commit replaces it; the two conflict if both are + set. Inventory the target's existing local hooks and CI workflows first: + every check they perform must end up either in a managed hook, a + `repo: local` script (step 4), or be consciously dropped — list the + mapping in the migration PR description. + *Done when:* `pre-commit run --all-files` executes (failures are fine — + they're the backlog for the next steps). + +3. **Relocate configs** into `.config/` (concept 2), referenced explicitly via + hook `args` — keep only root-*forced* files at root. Factor markdown rules + into `markdownlint.jsonc` so the CLI and editor share one source. + *Done when:* no linter config sits at root except forced ones, and + `pre-commit run --all-files` still finds every config. + +4. **Move custom checks into shared scripts** (concept 3). Any bespoke + lint/test logic duplicated between CI and a local hook becomes one + `.config/scripts/*.ps1` invoked as a `repo: local` hook, run identically in + both places. Delete the duplicate. Carry over + `.config/PSScriptAnalyzerRules/` (concept 4) if the target repo has + PowerShell to lint. Tests for setup-time scripts live in `setup/tests/`; + tests for permanent scripts sit next to the script — the Pester lane runner + discovers `*.Tests.ps1` recursively either way. + *Done when:* no check exists twice, and the test lanes + (`Invoke-TestLane.ps1 -Lane Fast`) run green or report "no tests". + +5. **Bring opt-in tooling across as inert overlays** (concept 5) — Vale, + semgrep, or anything else the source repo added as "complete but not + wired." Copy the folder and its `precommit-hook.yaml` fragment; don't paste + the fragment into the root config unless you're activating it now. + *Done when:* overlays exist but `pre-commit run --all-files` doesn't run + them. + +6. **Collapse CI to one workflow** (concept 6) that runs `pre-commit run` — + scoped to the diff on `pull_request`, full on `merge_group` and + `push: main`. Replace N split per-language workflows; path filtering is no + longer needed since pre-commit scopes by file. Carry over `CODEOWNERS`, + issue templates, and `pull_request_template.md` for the same + Evidence-mandatory PR contract. + *Done when:* exactly one lint workflow remains and a test PR shows the + `lint` check green. + +7. **Wire the editor** (concept 7): copy `.vscode/settings.json` (`extends` + the same `.config/markdownlint.jsonc`) and `.vscode/extensions.json`, and + check the target's `.gitignore` un-ignores exactly those two files (plus + `.vscode/README.md` if you copy that doc too) — see the ignore rule in this + repo's `.gitignore` for the pattern. + *Done when:* the three files are tracked (`git ls-files .vscode/` lists + them) and nothing else in `.vscode/` is. + +8. **Copy and run the one-time setup tooling** (concept 8) after the first + push and one CI run (so the `lint` check the ruleset requires actually + exists): copy the whole `setup/` folder (including this file and + `optional-skills/` — install the bundled `develop-with-tdd` skill only if + no equivalent TDD skill exists), then follow + [`README.md`](README.md) — protection, security settings, AI-maintainer + identity, in that order. + **If the target repo is owned by a personal/user account** (not an + organization), expect `merge_queue` to be rejected by GitHub — a platform + limitation, not a bug. `Protect-MainBranch.ps1` posts each rule as its own + ruleset for exactly this reason: the other four protections still land. + *Done when:* the rulesets appear in repo settings and + `Test-AIMaintainerIdentity.ps1` passes from the agent's shell. + +9. **Copy the remaining docs.** `AGENTS.md` plus the per-folder READMEs + (trimmed to the target repo's actual tools), updated to reflect what you + actually migrated. Purge phantoms: verify every referenced tool exists in + the target repo (e.g. there is no `asciidoctor-lint` gem — use + `asciidoctor --failure-level` for syntax and Vale for prose). A doc that + lists a nonexistent tool is worse than no doc. + *Done when:* every tool named in a doc can be invoked in the target repo. + +10. **Finish.** Delete the reference copy (`.temp/template/`), then run + `pwsh -NoProfile -File setup/Complete-Setup.ps1` — it removes `setup/` + and opens the removal PR (and refuses to run if the repo is itself a + template repo). + *Done when:* the removal PR is open and CI on it is green. + +## Anti-pattern this replaces + +Local hook logic and CI workflows re-implementing "the same" checks +separately, which drift apart over time. One orchestrator, one source, copied +concept by concept rather than file by file. diff --git a/setup/New-AIMaintainerApp.ps1 b/setup/New-AIMaintainerApp.ps1 new file mode 100644 index 0000000..6969b67 --- /dev/null +++ b/setup/New-AIMaintainerApp.ps1 @@ -0,0 +1,225 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS + Create a least-privilege GitHub App for an AI maintainer via GitHub's + app-manifest flow — one browser confirmation, no manual form-filling. + +.DESCRIPTION + GitHub offers no pure-API way to create an App; the manifest flow is the + scriptable floor (one click). This script: + + 1. serves a tiny local page that posts the App manifest to GitHub + (permissions: Contents RW, Pull requests RW, Metadata RO — no admin, + no workflows, webhook off, private app), + 2. opens your browser — you confirm the creation on github.com, + 3. receives the redirect, exchanges the temporary code for the App's + credentials, and saves the private key locally. + + Afterwards INSTALL the app on this repository only (the printed URL), then + mint short-lived installation tokens for the agent's shell — see + setup/AI-Maintainer-Identity.adoc. Verify the result with + setup/Test-AIMaintainerIdentity.ps1. Create one App per locally-run agent + (Claude Code, Codex CLI, …); vendor-hosted agents (Copilot coding agent, + Codex cloud) bring their own App — install and scope theirs instead. + +.PARAMETER AppName + App name (globally unique on GitHub). Defaults to '-ai-maintainer'. + +.PARAMETER Organization + Register the App under the org that owns the repo instead of your + personal account. Required when the repo is org-owned and the App should + live with it. + +.PARAMETER Port + Localhost port for the one-shot callback listener. + +.PARAMETER OutputDirectory + Where the App's private key (*.private-key.pem) is stored. Keep it out of + any repo; move it to your secret manager afterwards. +#> +[CmdletBinding()] +param( + [string]$AppName, + [switch]$Organization, + [ValidateRange(1024, 65535)] + [int]$Port = 8712, + [string]$OutputDirectory = (Join-Path $HOME '.github-apps') +) + +Set-StrictMode -Version Latest + +function Test-GhCli { [bool](Get-Command gh -ErrorAction SilentlyContinue) } + +function Get-AppManifest { + <# Build the app-manifest payload (pure; no side effects). Exactly the + least-privilege set from AI-Maintainer-Identity.adoc — extending this + list is a deliberate security decision, not a convenience tweak. #> + param( + [Parameter(Mandatory)][string]$AppName, + [Parameter(Mandatory)][string]$RedirectUrl + ) + @{ + name = $AppName + url = 'https://github.com' # required homepage; cosmetic for a private bot + public = $false + redirect_url = $RedirectUrl + hook_attributes = @{ active = $false; url = 'https://example.com/unused' } + default_permissions = @{ + contents = 'write' + pull_requests = 'write' + metadata = 'read' + } + } +} + +function Get-ManifestFormHtml { + <# Render the self-submitting form GitHub's manifest flow requires (pure). + Personal accounts post to /settings/apps/new; org-owned Apps to + /organizations//settings/apps/new. #> + param( + [Parameter(Mandatory)][hashtable]$Manifest, + [Parameter(Mandatory)][string]$Owner, + [switch]$Organization + ) + $target = $Organization ? + "https://github.com/organizations/$Owner/settings/apps/new" : + 'https://github.com/settings/apps/new' + $json = [System.Net.WebUtility]::HtmlEncode(($Manifest | ConvertTo-Json -Depth 5 -Compress)) + @" + + +

Redirecting to GitHub to create the app…

+
+ +
+ + +"@ +} + +function Save-AppCredential { + <# Persist the App's private key; returns the file path. #> + param( + [Parameter(Mandatory)][object]$App, + [Parameter(Mandatory)][string]$OutputDirectory + ) + New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null + $path = Join-Path $OutputDirectory "$($App.slug).private-key.pem" + Set-Content -Path $path -Value $App.pem -NoNewline + Protect-PrivateKeyFile -Path $path + $path +} + +function Protect-PrivateKeyFile { + <# Restrict file access to the current user only. #> + param([Parameter(Mandatory)][string]$Path) + + if ($IsWindows) { + # .NET Core dropped [System.IO.File]::SetAccessControl; seed the + # descriptor from the file itself and apply it with Set-Acl (PS7-native). + $currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().User + $acl = Get-Acl -Path $Path + $acl.SetOwner($currentUser) + $acl.SetAccessRuleProtection($true, $false) + $rule = [System.Security.AccessControl.FileSystemAccessRule]::new( + $currentUser, + [System.Security.AccessControl.FileSystemRights]::Read -bor [System.Security.AccessControl.FileSystemRights]::Write, + [System.Security.AccessControl.AccessControlType]::Allow + ) + $acl.SetAccessRule($rule) + Set-Acl -Path $Path -AclObject $acl + return + } + + [System.IO.File]::SetUnixFileMode( + $Path, + [System.IO.UnixFileMode]::UserRead -bor [System.IO.UnixFileMode]::UserWrite + ) +} + +function Wait-ManifestCode { + <# Serve the form page, open the browser, and block until GitHub redirects + back with the temporary code (interactive; Ctrl+C to abort). #> + param( + [Parameter(Mandatory)][string]$FormHtml, + [Parameter(Mandatory)][int]$Port + ) + $listener = [System.Net.HttpListener]::new() + $listener.Prefixes.Add("http://localhost:$Port/") + $listener.Start() + try { + Start-Process "http://localhost:$Port/" + Write-Information "Waiting for the browser round-trip on http://localhost:$Port/ (Ctrl+C to abort)…" -InformationAction Continue + while ($true) { + $ctx = $listener.GetContext() + $code = $ctx.Request.QueryString['code'] + $body = $code ? + 'App created — you can close this tab and return to the terminal.' : + $FormHtml + $bytes = [System.Text.Encoding]::UTF8.GetBytes($body) + $ctx.Response.ContentType = 'text/html; charset=utf-8' + $ctx.Response.OutputStream.Write($bytes, 0, $bytes.Length) + $ctx.Response.Close() + if ($code) { return $code } + } + } + finally { + $listener.Stop() + $listener.Close() + } +} + +function Invoke-AppCreation { + param( + [string]$AppName, + [switch]$Organization, + [int]$Port = 8712, + [string]$OutputDirectory = (Join-Path $HOME '.github-apps') + ) + + # Return codes are the contract; keep Write-Error non-terminating. + $ErrorActionPreference = 'Continue' + + if (-not (Test-GhCli)) { + Write-Error 'GitHub CLI (gh) is not installed. https://cli.github.com/' + return 1 + } + $repo = gh repo view --json nameWithOwner --jq '.nameWithOwner' 2>$null + if (-not $repo) { + Write-Error 'Not a GitHub repository, or gh is not authenticated (`gh auth login`).' + return 1 + } + $owner = ($repo -split '/')[0] + if (-not $AppName) { $AppName = "$owner-ai-maintainer" } + + $manifest = Get-AppManifest -AppName $AppName -RedirectUrl "http://localhost:$Port/callback" + $html = Get-ManifestFormHtml -Manifest $manifest -Owner $owner -Organization:$Organization + $code = Wait-ManifestCode -FormHtml $html -Port $Port + if (-not $code) { + Write-Error 'No code received from GitHub — app creation was not confirmed.' + return 1 + } + + # Exchange the temporary code (valid ~1 h, single use) for the credentials. + $app = gh api --method POST "app-manifests/$code/conversions" 2>$null | ConvertFrom-Json + if ($LASTEXITCODE -ne 0 -or -not $app) { + Write-Error 'Exchanging the manifest code failed — the code may have expired; rerun the script.' + return 1 + } + + $keyPath = Save-AppCredential -App $app -OutputDirectory $OutputDirectory + Write-Information @" +App created: $($app.slug) (App ID $($app.id)) +Private key: $keyPath — move it to your secret manager; never commit it. +Next steps: + 1. Install it on THIS repo only: $($app.html_url)/installations/new + 2. Mint an installation token for the agent's shell (see setup/AI-Maintainer-Identity.adoc). + 3. Verify from that shell: pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 +"@ -InformationAction Continue + return 0 +} + +# Execute only when run directly, not when dot-sourced by a test. +if ($MyInvocation.InvocationName -ne '.') { + exit (Invoke-AppCreation -AppName $AppName -Organization:$Organization -Port $Port -OutputDirectory $OutputDirectory) +} diff --git a/setup/Protect-MainBranch.Tests.ps1 b/setup/Protect-MainBranch.Tests.ps1 deleted file mode 100644 index a280e3d..0000000 --- a/setup/Protect-MainBranch.Tests.ps1 +++ /dev/null @@ -1,83 +0,0 @@ -#Requires -Version 7.0 -Set-StrictMode -Version Latest - -BeforeAll { - # Dot-source the SUT: defines its functions; the run guard skips execution. - . (Join-Path $PSScriptRoot 'Protect-MainBranch.ps1') -} - -Describe 'Get-ProtectionRuleset' -Tag 'Fast' { - BeforeAll { - $script:R = Get-ProtectionRuleset -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' - } - - It 'targets the default branch' { - $script:R.conditions.ref_name.include | Should -Contain '~DEFAULT_BRANCH' - } - - It 'requires a pull request' { - ($script:R.rules | Where-Object type -EQ 'pull_request') | Should -Not -BeNullOrEmpty - } - - It 'requires the given status check' { - $rsc = ($script:R.rules | Where-Object type -EQ 'required_status_checks').parameters.required_status_checks - $rsc.context | Should -Contain 'lint' - } - - It 'blocks force-push and deletion' { - $script:R.rules.type | Should -Contain 'non_fast_forward' - $script:R.rules.type | Should -Contain 'deletion' - } - - It 'routes merges through a merge queue' { - $script:R.rules.type | Should -Contain 'merge_queue' - } - - It 'requires review threads to be resolved (holds auto-merge on open comments)' { - ($script:R.rules | Where-Object type -EQ 'pull_request').parameters.required_review_thread_resolution | - Should -BeTrue - } -} - -Describe 'Get-StrictLayerRuleset' -Tag 'Fast' { - BeforeAll { - $script:L = Get-StrictLayerRuleset - } - - It 'exempts the repository admin role' { - $script:L.bypass_actors.actor_type | Should -Contain 'RepositoryRole' - $script:L.bypass_actors.bypass_mode | Should -Contain 'exempt' - } - - It 'requires one code-owner approval given after the last push' { - $p = ($script:L.rules | Where-Object type -EQ 'pull_request').parameters - $p.required_approving_review_count | Should -Be 1 - $p.require_code_owner_review | Should -BeTrue - $p.require_last_push_approval | Should -BeTrue - $p.dismiss_stale_reviews_on_push | Should -BeTrue - } - - It 'targets the default branch under a distinct name' { - $script:L.conditions.ref_name.include | Should -Contain '~DEFAULT_BRANCH' - $script:L.name | Should -Not -Be 'main-protection' - } -} - -Describe 'Get-MergeFlowSetting' -Tag 'Fast' { - It 'enables auto-merge and branch cleanup, and touches nothing else' { - $s = Get-MergeFlowSetting - $s.allow_auto_merge | Should -BeTrue - $s.delete_branch_on_merge | Should -BeTrue - # No merge-method keys: a repo-wide disable would also bind bypass actors. - $s.Keys | Where-Object { $_ -match 'allow_.*_merge$' -and $_ -ne 'allow_auto_merge' } | - Should -BeNullOrEmpty - } -} - -Describe 'Invoke-BranchProtection' -Tag 'Fast' { - It 'returns non-zero when gh is not installed' { - Mock Test-GhCli { $false } - Invoke-BranchProtection -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' 2>$null | - Should -Be 1 - } -} diff --git a/setup/Protect-MainBranch.ps1 b/setup/Protect-MainBranch.ps1 index 55926e3..7eff667 100644 --- a/setup/Protect-MainBranch.ps1 +++ b/setup/Protect-MainBranch.ps1 @@ -37,53 +37,90 @@ Set-StrictMode -Version Latest function Test-GhCli { [bool](Get-Command gh -ErrorAction SilentlyContinue) } function Get-ProtectionRuleset { - <# Build the rulesets-API payload (pure; no side effects). #> + <# Build the rulesets-API payloads (pure; no side effects) — one ruleset per + rule type, posted as separate API calls. GitHub rejects the whole request + if ANY rule in a ruleset is invalid for the repo (e.g. 'merge_queue' on a + personal/user-owned repo, which only supports it on org-owned repos or + Enterprise Cloud) — splitting per-rule means that one incompatible rule + can't block the rest of the protections from landing. #> param( [Parameter(Mandatory)][string]$CheckName, [Parameter(Mandatory)][int]$RequiredApprovals, [Parameter(Mandatory)][string]$MergeMethod ) - @{ - name = 'main-protection' - target = 'branch' - enforcement = 'active' - conditions = @{ ref_name = @{ include = @('~DEFAULT_BRANCH'); exclude = @() } } - rules = @( - @{ type = 'deletion' } - @{ type = 'non_fast_forward' } - @{ - type = 'pull_request' - parameters = @{ - required_approving_review_count = $RequiredApprovals - dismiss_stale_reviews_on_push = $false - require_code_owner_review = $false - require_last_push_approval = $false - # Unresolved review threads block the merge — with auto-merge - # on, a review comment holds an agent PR until it is resolved. - required_review_thread_resolution = $true + $target = @{ ref_name = @{ include = @('~DEFAULT_BRANCH'); exclude = @() } } + @( + @{ + name = 'main-protection-deletion' + target = 'branch' + enforcement = 'active' + conditions = $target + rules = @(@{ type = 'deletion' }) + } + @{ + name = 'main-protection-non-fast-forward' + target = 'branch' + enforcement = 'active' + conditions = $target + rules = @(@{ type = 'non_fast_forward' }) + } + @{ + name = 'main-protection-pull-request' + target = 'branch' + enforcement = 'active' + conditions = $target + rules = @( + @{ + type = 'pull_request' + parameters = @{ + required_approving_review_count = $RequiredApprovals + dismiss_stale_reviews_on_push = $false + require_code_owner_review = $false + require_last_push_approval = $false + # Unresolved review threads block the merge — with + # auto-merge on, a review comment holds an agent PR + # until it is resolved. + required_review_thread_resolution = $true + } } - } - @{ - type = 'required_status_checks' - parameters = @{ - required_status_checks = @(@{ context = $CheckName }) - strict_required_status_checks_policy = $false + ) + } + @{ + name = 'main-protection-required-status-checks' + target = 'branch' + enforcement = 'active' + conditions = $target + rules = @( + @{ + type = 'required_status_checks' + parameters = @{ + required_status_checks = @(@{ context = $CheckName }) + strict_required_status_checks_policy = $false + } } - } - @{ - type = 'merge_queue' - parameters = @{ - merge_method = $MergeMethod - grouping_strategy = 'ALLGREEN' - max_entries_to_build = 5 - min_entries_to_merge = 1 - max_entries_to_merge = 5 - min_entries_to_merge_wait_minutes = 5 - check_response_timeout_minutes = 60 + ) + } + @{ + name = 'main-protection-merge-queue' + target = 'branch' + enforcement = 'active' + conditions = $target + rules = @( + @{ + type = 'merge_queue' + parameters = @{ + merge_method = $MergeMethod + grouping_strategy = 'ALLGREEN' + max_entries_to_build = 5 + min_entries_to_merge = 1 + max_entries_to_merge = 5 + min_entries_to_merge_wait_minutes = 1 + check_response_timeout_minutes = 60 + } } - } - ) - } + ) + } + ) } function Get-StrictLayerRuleset { @@ -148,13 +185,29 @@ function Invoke-BranchProtection { return 1 } - $json = Get-ProtectionRuleset -CheckName $CheckName -RequiredApprovals $RequiredApprovals -MergeMethod $MergeMethod | - ConvertTo-Json -Depth 8 - $json | gh api --method POST "repos/$repo/rulesets" --input - - if ($LASTEXITCODE -ne 0) { - Write-Error 'Failed to create ruleset (it may already exist, or you lack admin on the repo).' + $rulesets = Get-ProtectionRuleset -CheckName $CheckName -RequiredApprovals $RequiredApprovals -MergeMethod $MergeMethod + $failed = @() + foreach ($ruleset in $rulesets) { + ($ruleset | ConvertTo-Json -Depth 8) | gh api --method POST "repos/$repo/rulesets" --input - | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Warning "Ruleset '$($ruleset.name)' failed (it may already exist, be unsupported on this repo/plan, or you lack admin) — continuing with the rest." + $failed += $ruleset.name + } + } + $optionalRulesets = @('main-protection-merge-queue') + $requiredFailures = @($failed | Where-Object { $_ -notin $optionalRulesets }) + + if ($failed.Count -eq $rulesets.Count) { + Write-Error 'All branch-protection rulesets failed; nothing was applied.' return 1 } + if ($requiredFailures.Count -gt 0) { + Write-Error "Required branch-protection rulesets failed: $($requiredFailures -join ', ')" + return 1 + } + if ($failed.Count -gt 0) { + Write-Warning "Branch protection partially applied — skipped: $($failed -join ', ')" + } if ($WithStrictLayer) { (Get-StrictLayerRuleset | ConvertTo-Json -Depth 8) | gh api --method POST "repos/$repo/rulesets" --input - diff --git a/setup/README.md b/setup/README.md index 22fd276..85683ca 100644 --- a/setup/README.md +++ b/setup/README.md @@ -1,9 +1,38 @@ # One-time repo setup -Tooling to run **once** on a new repo created from this template — then delete -this whole folder. +Tooling to run **once** on a new repo created from this template, in this +order — the last step deletes this whole folder via its own PR. Tests for +these scripts live in [`tests/`](tests/): this folder reads as an ordered +runbook, so tests are kept out of its root rather than co-located beside each +script (the repo-wide default — see [`.config/scripts/README.md`](../.config/scripts/README.md)). +They still leave with the folder on teardown. [`CHANGELOG.md`](CHANGELOG.md) +has the version history; the current version is in [`version`](version). -## Protect the default branch +Bringing this tooling into a repo that *wasn't* created from this template +instead? Start with [`MIGRATION.md`](MIGRATION.md) — an executable runbook +that walks through every concept, not just this folder's scripts. + +All steps at a glance (details in the sections below): + +```pwsh +# 1. Protect the default branch +pwsh -NoProfile -File setup/Protect-MainBranch.ps1 + +# 2. Enable server-side security settings +pwsh -NoProfile -File setup/Enable-RepoSecurity.ps1 + +# 3. Set up the AI-maintainer identity, then verify it FROM THE AGENT'S SHELL +pwsh -NoProfile -File setup/New-AIMaintainerApp.ps1 +pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 + +# Optional: install the bundled AI skills before they are deleted with setup/ +Copy-Item -Recurse setup/optional-skills/skills/develop-with-tdd ~/.claude/skills/ + +# 4. Remove this folder and open it as the repo's first PR +pwsh -NoProfile -File setup/Complete-Setup.ps1 +``` + +## 1. Protect the default branch Needs the GitHub CLI, authenticated with admin on the repo: @@ -21,7 +50,14 @@ given after the last push). Run it only once a second identity — e.g. an AI-maintainer App/PAT per [`AI-Maintainer-Identity.adoc`](AI-Maintainer-Identity.adoc) — opens PRs; with no other identity it binds nobody. -## Enable server-side security settings +Each rule posts as its own ruleset, so one rejected rule can't block the rest. +In particular, **`merge_queue` only works on org-owned repos** (public or +private-with-GitHub-Enterprise-Cloud) or Enterprise Cloud generally — GitHub +rejects it outright on a personal/user-owned repo. The script warns and skips +it in that case; the other four protections (no force-push, no deletion, +required PR, required `lint` check) still land. + +## 2. Enable server-side security settings The security toggles that live in GitHub's API, not in repo files. Same requirements (admin + `gh`); every call is idempotent, so re-running is safe: @@ -36,17 +72,50 @@ default workflow token. Pass `-SkipWorkflowTokenReadOnly` if a workflow depends on the legacy read/write default. Settings that are plan-restricted or already set warn and are skipped; the script still applies the rest. -## Set up AI-maintainer identities +## 3. Set up and verify the AI-maintainer identity -[`AI-Maintainer-Identity.adoc`](AI-Maintainer-Identity.adoc) is the knowledge -base for giving each AI agent its own least-privilege identity (GitHub App or -fine-grained PAT) instead of running under the owner's admin account. +Agents must work under their own least-privilege identity, never under your +admin account. [`AI-Maintainer-Identity.adoc`](AI-Maintainer-Identity.adoc) is +the knowledge base; two scripts automate what can be automated: -## Remove this folder +```pwsh +# Create a least-privilege GitHub App (one browser confirmation — GitHub has +# no pure-API path for App creation). One App per locally-run agent. +pwsh -NoProfile -File setup/New-AIMaintainerApp.ps1 + +# From the AGENT's shell: verify its credential is a bot identity +# (installation token or fine-grained PAT) WITH repo write/push and WITHOUT +# admin. Fails closed. +pwsh -NoProfile -File setup/Test-AIMaintainerIdentity.ps1 +``` -Once protection is on, this folder has served its purpose: +Create Apps only for agents *you* run locally (Claude Code, Codex CLI, …). +Vendor-hosted agents — Copilot coding agent, Codex cloud — bring their own +GitHub App: install and repo-scope *theirs*; you can't substitute your own. + +## Optional: install the bundled AI skills + +[`optional-skills/`](optional-skills/) bundles two skills: `develop-with-tdd` +(plus the TDD knowledge base behind it) that this repo's docs reference, and +`changelog-entry` for adding high-quality `CHANGELOG.md` entries. If your +agent already has an equivalent, skip it; no need for a second copy. Otherwise +install what you want before the next step deletes the bundle, e.g. for Claude +Code: ```pwsh -git rm -r setup -git commit -m "Remove one-time setup tooling" +Copy-Item -Recurse setup/optional-skills/skills/develop-with-tdd ~/.claude/skills/ +Copy-Item -Recurse setup/optional-skills/skills/changelog-entry ~/.claude/skills/ ``` + +## 4. Remove this folder (its own first PR) + +Once the steps above succeeded, the folder has served its purpose: + +```pwsh +pwsh -NoProfile -File setup/Complete-Setup.ps1 +``` + +Branches, removes `setup/`, and opens a PR whose body fills the repo's PR +template. **Failsafe:** it refuses to run on a template repo (`isTemplate`), +so the template itself never loses its setup tooling — and on a dirty +worktree, so the removal PR contains nothing else. diff --git a/setup/Test-AIMaintainerIdentity.ps1 b/setup/Test-AIMaintainerIdentity.ps1 new file mode 100644 index 0000000..f9640e6 --- /dev/null +++ b/setup/Test-AIMaintainerIdentity.ps1 @@ -0,0 +1,114 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS + Verify that the credential in the current shell is a least-privilege + AI-maintainer identity — not the owner's admin account. + +.DESCRIPTION + Run this from the shell an AI agent will use (the agent inherits that + shell's gh/git credential). It classifies the active token and checks its + effective permission on this repo, then fails unless ALL hold: + + - the token is a GitHub App installation token (ghs_) or a fine-grained + PAT (github_pat_) — see setup/AI-Maintainer-Identity.adoc for how to + create one (New-AIMaintainerApp.ps1 automates the App path), and + - the credential does NOT have admin on the repo, and + - the credential has effective write access (`permissions.push=true`) on + this repository. + + Everything else fails closed: an OAuth user token (gho_, your interactive + `gh auth login` session), a classic PAT (ghp_, unscoped), or an + unrecognized format. Exit code 0 = safe to hand this shell to an agent. + Complete-Setup.ps1 runs this check before removing the setup/ folder. +#> +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest + +function Test-GhCli { [bool](Get-Command gh -ErrorAction SilentlyContinue) } + +function Get-TokenKind { + <# Classify a GitHub token by its documented prefix (pure; no side effects). + https://github.blog/engineering/behind-githubs-new-authentication-token-formats/ #> + param([Parameter(Mandatory)][string]$Token) + switch -Regex ($Token) { + '^ghs_' { 'Installation'; break } # App installation (server-to-server) + '^github_pat_' { 'FineGrainedPat'; break } + '^ghp_' { 'ClassicPat'; break } + '^gho_' { 'OAuthUser'; break } # interactive `gh auth login` + '^ghu_' { 'UserToServer'; break } # App acting AS the signed-in user + default { 'Unknown' } + } +} + +function Get-IdentityVerdict { + <# Apply the least-privilege rules to the gathered facts (pure; no side + effects). Fails closed: only the two identity mechanisms from + AI-Maintainer-Identity.adoc pass, never with admin, and only with + effective write permission on this repo. #> + param( + [Parameter(Mandatory)][string]$TokenKind, + [Parameter(Mandatory)][bool]$HasAdmin, + [Parameter(Mandatory)][bool]$HasWrite + ) + $findings = [System.Collections.Generic.List[string]]::new() + + if ($HasAdmin) { + $findings.Add('Credential has ADMIN on this repo — an agent could disable the branch ruleset. Use a write-only identity.') + } + if (-not $HasWrite) { + $findings.Add('Credential lacks write/push permission on this repo — setup cannot proceed with a read-only token.') + } + switch ($TokenKind) { + 'Installation' { } + 'FineGrainedPat' { } + 'OAuthUser' { $findings.Add('Token is an OAuth user token — the owner''s interactive gh session, not a separate agent identity. Actions would be attributed to your own account.') } + 'UserToServer' { $findings.Add('Token is a user-to-server App token — it acts as the signed-in user, so agent actions are attributed to you.') } + 'ClassicPat' { $findings.Add('Token is a classic PAT — it cannot be scoped per-permission or per-repo. Replace it with a fine-grained PAT or an App installation token.') } + default { $findings.Add("Token format not recognized ($TokenKind) — cannot verify least privilege; failing closed.") } + } + + @{ Pass = ($findings.Count -eq 0); Findings = $findings } +} + +function Invoke-IdentityCheck { + # Return codes are the contract; keep Write-Error non-terminating. + $ErrorActionPreference = 'Continue' + + if (-not (Test-GhCli)) { + Write-Error 'GitHub CLI (gh) is not installed. https://cli.github.com/' + return 1 + } + $repo = gh repo view --json nameWithOwner --jq '.nameWithOwner' 2>$null + if (-not $repo) { + Write-Error 'Not a GitHub repository, or gh is not authenticated (`gh auth login`).' + return 1 + } + $token = gh auth token 2>$null + if (-not $token) { + Write-Error 'No token available in this shell (`gh auth token` returned nothing).' + return 1 + } + + # Effective permissions of THIS credential on THIS repo; missing/erroring + # fields fail closed. + $admin = gh api "repos/$repo" --jq '.permissions.admin' 2>$null + $push = gh api "repos/$repo" --jq '.permissions.push' 2>$null + $hasAdmin = "$admin" -eq 'true' + $hasWrite = "$push" -eq 'true' + + $verdict = Get-IdentityVerdict -TokenKind (Get-TokenKind -Token $token) -HasAdmin $hasAdmin -HasWrite $hasWrite + if (-not $verdict.Pass) { + $verdict.Findings | ForEach-Object { Write-Error $_ } + Write-Error "This shell's credential is NOT a least-privilege AI-maintainer identity for $repo. See setup/AI-Maintainer-Identity.adoc." + return 1 + } + Write-Information "Credential is a least-privilege AI-maintainer identity for $repo." -InformationAction Continue + return 0 +} + +# Execute only when run directly, not when dot-sourced by a test. +if ($MyInvocation.InvocationName -ne '.') { + exit (Invoke-IdentityCheck) +} diff --git a/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md b/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md new file mode 100644 index 0000000..300bfac --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/ASSESSMENT_BRIEF.md @@ -0,0 +1,133 @@ +# TDD documentation and skill assessment brief + +Use this brief when asking an AI agent to review the TDD knowledge base and the +`develop-with-tdd` skill. + +The goal is not a generic writing review. The goal is to determine whether the +guide teaches the intended TDD practice clearly, and whether the skill applies +that practice reliably while changing code. + +## Review targets + +Assess these targets in this order: + +1. `knowledge/tdd/guide/` +2. `knowledge/tdd/research-findings.md` +3. `knowledge/tdd/traceability.md` +4. `skills/develop-with-tdd/` + +Treat the guide as the human teaching authority. + +Treat the skill as an execution artifact that should operationalize the same +doctrine without needing to load the full guide at runtime. + +## Intended audience + +The human guide must work for: + +- beginners who do not yet know why TDD exists; +- practitioners who have used tests but may not have used TDD deliberately; +- skeptics who associate TDD with dogma, slow tests, brittle mocks, or coverage + theater; +- developers working in both greenfield and legacy code. + +Do not assess the guide as if every reader already understands test levels, +mocks, characterization tests, BDD, CI, or refactoring constraints. + +## Core doctrine to preserve + +The material should teach TDD as a fast-feedback design discipline, not as a +coverage ritual. + +The central rule is: + +> Prefer the shortest trustworthy feedback loop that faithfully exercises the +> current risk. + +The review should preserve these ideas: + +- coverage is a by-product and diagnostic signal, not the primary goal; +- the first value of TDD is thinking about desired behavior before + implementation; +- a TDD cycle clarifies a contract, produces a small design decision, and leaves + executable regression evidence behind; +- feedback speed includes relevance, fidelity, repeatability, diagnostic value, + and decision timing, not only runtime; +- test level and execution cadence are related but independent; +- a slower faithful test can be better than a fast irrelevant test; +- BDD, ATDD, and Specification by Example are related practices, not simple + replacements or synonyms for TDD; +- legacy work may need characterization, minimal behavior-preserving seams, and + smaller entry strategies before normal TDD becomes possible. + +## Required review questions + +Answer these questions with concrete file or section references: + +1. Does the guide have a clear learning journey from "why TDD" to "how to + practise it" to "how to adapt it without dogma"? +2. Are important terms introduced before the text relies on them? +3. Does the guide explain why Red matters, including why an AI agent must not + infer that a test would fail? +4. Does the guide distinguish Red, Green, Refactor, characterization, pure + refactoring, exploration, and intentional contract changes? +5. Does it explain feedback lanes clearly enough for a team to choose fast, + standard, and thorough suites? +6. Does it cover focused, integration, contract, acceptance, and E2E evidence + without implying one universal test pyramid? +7. Does it address common misconceptions and failure modes without turning into + a defensive rant? +8. Does it help both greenfield and legacy development? +9. Does the skill preserve the same invariants as the guide? +10. Are any examples misleading, incomplete, over-fitted, or too abstract for a + beginner? + +## Skill-specific checks + +When reviewing `skills/develop-with-tdd/`, check whether it instructs an AI +agent to: + +- establish a known baseline before attributing failures; +- choose the shortest trustworthy evidence for the current risk; +- write or revise evidence before implementing new or corrected behavior; +- run and observe a meaningful Red instead of assuming Red; +- distinguish harness, compile, environment, and behavior failures; +- reach Green without weakening the test or changing the agreed contract; +- refactor only under passing evidence; +- report what was run, what was omitted, and what uncertainty remains; +- use nonstandard branches honestly, such as characterization-first legacy work, + pure refactor, exploration without an oracle, or intentional contract change. + +Also check whether references are loaded conditionally. The skill should stay +compact by default and use deeper references only when the task needs them. + +## Output format for an AI reviewer + +Use this structure: + +1. Findings by severity +2. Missing or underdeveloped angles +3. Skill and guide synchronization issues +4. Suggested concrete edits +5. Residual risks or open questions + +For each finding, include: + +- location; +- why it matters; +- suggested correction. + +Avoid vague praise. A clean assessment should still mention residual risks, +scope limits, and the strongest remaining objections. + +## Non-goals + +Do not require the guide to become a complete testing textbook. + +Do not add every adjacent practice into the core learning path. Security, +accessibility, performance, compliance, observability, mutation testing, and +formal methods belong in adjacent or appendix material unless they directly +change the TDD practice being taught. + +Do not optimize the skill for maximum coverage. Optimize it for trustworthy, +timely evidence while preserving the Red-Green-Refactor discipline. diff --git a/setup/optional-skills/knowledge/tdd/MAINTENANCE.md b/setup/optional-skills/knowledge/tdd/MAINTENANCE.md new file mode 100644 index 0000000..3db7729 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/MAINTENANCE.md @@ -0,0 +1,42 @@ +# Maintenance model + +The knowledge base has three authorities with different jobs: + +1. **Evidence authority:** `research-findings.md` owns sourced historical, empirical, and definitional findings. +2. **Teaching authority:** `human/**/*.md` owns explanations, progression, examples, and reader-facing trade-offs. +3. **Execution authority:** `develop-with-tdd/SKILL.md` and its bundled reference own agent behavior. + +`analysis.md` records editorial judgments. `traceability.md` connects the authorities; it does not duplicate their full content. The original AsciiDoc draft remains provenance, not an active authority. + +## Change procedure + +1. Record new source evidence or a corrected interpretation in `research-findings.md`. +2. Update the affected row in `traceability.md`. +3. Update human explanation where reader understanding changes. +4. Update the skill only where runtime agent behavior changes. +5. Forward-test every changed skill branch with a task that does not reveal the intended answer. +6. Validate local links, skill metadata, language, feedback-lane commands, and matrix destinations. + +Record forward-test scenarios and results in `skill-evaluation.md`. Replace hypothetical scenarios with runnable fixtures when a future change depends on stack-specific execution rather than decision quality. + +Keep user-facing installation and learning guidance in the top-level `README.md`. The installable skill contains only `SKILL.md`, provider metadata, and runtime resources. + +## Scope rule + +Include a perspective when it materially changes at least one of: + +- what behavior or risk is tested; +- where the observation boundary belongs or whether a seam is needed; +- which oracle or test double is appropriate; +- how quickly and reliably feedback arrives; +- which feedback lane should run during the cycle or before a risky transition; +- whether another assurance technique is better; +- legal, regulatory, safety, or operational obligations. + +Record deliberately excluded detail in the relevant human chapter. Do not add a method merely to make the catalogue exhaustive. + +Place a topic in the adjacent-practices appendix when it helps a reader recognize TDD’s boundary but does not change the ordinary Red–Green–Refactor procedure. + +## Duplication rule + +Repeat a short core principle only when an audience must act without loading another artifact. Keep evidence in research, explanations in human chapters, and imperatives in the skill. Link instead of copying examples or source discussions into the skill. diff --git a/setup/optional-skills/knowledge/tdd/README.md b/setup/optional-skills/knowledge/tdd/README.md new file mode 100644 index 0000000..1452196 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/README.md @@ -0,0 +1,75 @@ +# TDD knowledge base + +This directory separates a human learning path from the compact instructions an AI agent needs while changing code. + +The human chapters are the teaching authority. The `develop-with-tdd` folder is a self-contained execution package; it does not load the human curriculum during development. + +## Choose a route + +| Reader or need | Start here | Continue with | +| -------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------- | +| New to TDD | [Why TDD exists](guide/01-why-tdd.md) | Practise -> misconceptions -> first worked example | +| Ready to try one cycle | [Practising TDD deliberately](guide/02-practising-tdd.md) | Greenfield example in Chapter 7 | +| Skeptical of TDD | [Applying TDD without dogma](guide/06-applying-tdd-without-dogma.md) | Research findings and worked examples | +| Changing legacy code | [Legacy guidance](guide/06-applying-tdd-without-dogma.md#starting-in-legacy-code) | Legacy worked example | +| Choosing focused, integration, contract, or E2E evidence | [Choosing evidence and feedback lanes](guide/04-choosing-evidence-and-feedback-lanes.md) | Decision examples and adjacent practices | +| Introducing TDD to a team | [Team adoption](guide/06-applying-tdd-without-dogma.md#introducing-tdd-to-a-team) | Feedback lanes and maintenance model | +| Asking an AI agent to develop through TDD | [`develop-with-tdd`](../../skills/develop-with-tdd/SKILL.md) | Installation notes below | +| Asking an AI agent to assess the material | [Assessment brief](ASSESSMENT_BRIEF.md) | Guide -> research -> traceability -> skill | + +## Core learning path + +These chapters form the main argument and should be read in order by a beginner: + +1. [Why test-driven development exists](guide/01-why-tdd.md) +2. [Practising TDD deliberately](guide/02-practising-tdd.md) +3. [Common misconceptions and failure modes](guide/03-common-misconceptions-and-failure-modes.md) +4. [Choosing evidence and feedback lanes](guide/04-choosing-evidence-and-feedback-lanes.md) +5. [BDD and collaborative discovery](guide/05-bdd-and-collaborative-discovery.md) + +## Applied guidance + +1. [Applying TDD without dogma](guide/06-applying-tdd-without-dogma.md) +2. [Worked examples](guide/07-worked-examples.md) + +## Reference and optional material + +- [Stack-specific notes](guide/appendices/stack-specific-notes.md) +- [Glossary](guide/appendices/glossary.md) +- [Adjacent practices and further reading](guide/appendices/adjacent-practices-and-further-reading.md) + +Optional material helps readers recognize when TDD needs another form of evidence. It is not prerequisite knowledge for attempting the core loop. + +## Install the AI skill + +The portable skill consists of `../../skills/develop-with-tdd/SKILL.md`, `../../skills/develop-with-tdd/references/`, and optional provider metadata in `../../skills/develop-with-tdd/agents/`. + +### Claude Code + +Copy the complete `develop-with-tdd` folder to one of: + +- project scope: `.claude/skills/develop-with-tdd/`; +- global scope: `~/.claude/skills/develop-with-tdd/`. + +The directory name must match the `name` in `SKILL.md`. Invoke it with `/develop-with-tdd` or allow its description to trigger it. + +To make TDD the default for behavioral code, add a matching rule to `CLAUDE.md` at the same scope: + +> When implementing or correcting behavior in production code, use the `develop-with-tdd` skill to establish Red, reach Green, and refactor under evidence. + +Keep throwaway diagnostics and exploratory spikes outside that blanket rule. + +### OpenAI + +Install the complete folder in the applicable skills directory. The `agents/openai.yaml` file supplies OpenAI UI metadata. + +Invoke the skill explicitly with `$develop-with-tdd` or allow model invocation when the environment supports it. + +## Analysis and provenance + +- [Primary-source research findings](research-findings.md) +- [Claim-to-deliverable traceability](traceability.md) +- [Maintenance model](MAINTENANCE.md) +- [Assessment brief for AI reviewers](ASSESSMENT_BRIEF.md) + +Superseded source material may remain in the old working area as provenance. It is not active documentation for this package. diff --git a/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md b/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md new file mode 100644 index 0000000..3740fc2 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/01-why-tdd.md @@ -0,0 +1,213 @@ +# Why test-driven development exists + +This chapter assumes that you can read a small automated test. It does not assume that you have practised test-driven development or agree that it is useful. + +## TDD in one minute + +Test-driven development (TDD) is a way to develop one observable behavior at a time. + +A behavior is a response that matters to a caller or user. Before implementing it, the developer expresses the intended response as an executable example and runs that example. + +For new behavior or a defect correction, the example must show that the current system does not yet satisfy the intended behavior. This observed, relevant failure is **Red**. + +The developer then changes the production code until the example passes. This is **Green**. While the evidence remains Green, the developer may improve the structure without changing behavior. This is **Refactor**. + +Together, these steps form **Red–Green–Refactor**. + +The immediate value is feedback while requirements and design decisions are still cheap to change. The retained examples later provide evidence against regressions. + +TDD is therefore both a development workflow and a way to produce tests. Having automated tests does not mean that the code was developed through TDD. + +## TDD strengthens a familiar way of working + +Many developers already work iteratively: write a small part of the code, run the application, inspect the result, and continue when it appears to work. + +That is a useful feedback loop. TDD does not replace the instinct to check work. It makes the loop explicit, repeatable, and available before implementation settles the answer. + +An informal loop often starts with code and checks it through a broad manual or end-to-end path. That evidence may be realistic, but it can be slow to repeat and hard to diagnose. + +The result may also disappear when the developer moves on. Unless the scenario is recorded, later changes depend on someone remembering and repeating it. + +TDD strengthens this loop in three ways: + +- state the expected behavior before implementing it; +- use the shortest trustworthy feedback loop that faithfully exercises the current risk; +- retain the example so it can be rerun after later changes. + +Each new behavior is checked alongside previously recorded behavior. This reduces the chance that an attempted improvement silently damages something that already worked. + +German has a useful word for that outcome: **Verschlimmbesserung**—an attempted improvement that makes something worse. + +Tests can expose such regressions only when they describe the affected behavior with a useful expectation. Incorrectly specified, unstable, or untested behavior can still regress. + +## The problem begins before code is written + +Most software changes sound simple: add a rule, correct a defect, or alter an existing behavior. + +The first risk is not syntax. It is building the wrong outcome, overlooking an important case, or damaging behavior that should remain unchanged. + +Consider the request “orders of €50 or more receive free shipping.” Before code is written, several questions appear: + +- Does exactly €50 qualify? +- Is the threshold applied before or after discounts? +- Do taxes or shipping fees contribute? +- What happens in another currency? + +Code can answer these questions accidentally. The first implementation may silently choose one interpretation. Tests derived from that code may then repeat the same assumptions. + +TDD asks for a concrete expected outcome first. This does not guarantee the right requirement, but it makes the current interpretation visible and reviewable before implementation hides it. + +## What TDD tries to improve + +TDD aims to shorten several costly delays: + +- the delay before an ambiguous requirement becomes visible; +- the delay before an incorrect implementation is detected; +- the delay before an awkward interface is experienced; +- the delay before a later change reveals a regression. + +It does this through several modest mechanisms: + +- **Requirement clarity:** an example makes an expected outcome concrete. +- **Interface feedback:** using an interface before implementing it exposes awkward inputs, outputs, and responsibilities. +- **Localized feedback:** small changes make a new failure easier to attribute. +- **Incremental design:** each example can challenge the current design before more structure accumulates. +- **Change safety:** retained examples can reveal regressions during later modification and refactoring. +- **External memory:** the suite records cases that would otherwise depend on developer memory. + +These are mechanisms, not guaranteed project outcomes. Their value depends on the problem, the examples, the chosen test boundary, and the discipline of the developer. + +## Why test-first and test-after provide different feedback + +A test written after implementation can still verify behavior, document a contract, and detect later regressions. + +It cannot influence design decisions already made. Existing code also anchors attention: the test may demonstrate what the implementation does instead of challenging what it should do. + +That is the relevant difference between **test-first** and **test-after**. It concerns when feedback becomes available, not whether one test is automatically good and the other bad. + +Test-first can still produce a weak test. It may assert private details or derive the expected result from the same flawed reasoning used in production. + +Test timing creates an opportunity for design feedback. Test quality determines whether that opportunity is used well. Chapter 2 develops the criteria for trustworthy evidence. + +## Fast feedback means more than a fast test + +TDD is built around fast feedback, but test runtime is only one part of feedback speed. + +The governing rule is: + +> Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. + +A useful loop is quick enough to guide the next decision, relevant to the behavior, repeatable, and diagnostic when it fails. + +A millisecond test that bypasses the database cannot answer a question about database semantics. A slower test through the real database may be the correct Red when its semantics are the risk. + +Start as narrowly as the risk permits, not as narrowly as the test framework permits. Broader evidence can then answer broader questions without forcing every inner cycle through the whole system. + +The chapter on choosing evidence and feedback lanes develops this trade-off. + +## What TDD does not promise + +TDD does not prove that requirements are correct or complete. A precise example of the wrong requirement is still wrong. + +It does not automatically create good design or a sound architecture. Tests coupled to imagined internals can encourage unnecessary interfaces, dependency injection, and brittle simulations. + +It does not eliminate integration, deployment, security, usability, or operational failures. A focused test can prove a rule without proving that the assembled system works. + +It is not guaranteed to make every team faster or reduce every defect rate. Research reports mixed results across different settings. + +The defensible claim is narrower: TDD changes when certain questions are asked and how quickly some mistakes become visible. + +Whether that feedback is worth its cost must be judged for the code and risk at hand. The chapter on applying TDD without dogma examines evidence, objections, and limits. + +## One observed cycle + +Suppose the agreed rule is: an order qualifies for free shipping when its discounted subtotal is at least €50. + +### Establish a known baseline + +Run the relevant existing tests first. If they pass, a later failure can be attributed to the new example. If they do not, record the existing failures before changing anything. + +### Red: demonstrate one missing behavior + +The first example defines one boundary case: + +```python +def test_fifty_euros_qualifies_for_free_shipping(): + order = Order(discounted_subtotal=Decimal("50.00")) + + assert order.qualifies_for_free_shipping() is True +``` + +Run the test. Do not infer that it would fail, even when the change seems obvious. + +The useful Red is a reproducible failure caused by the missing shipping rule. A broken fixture, wrong import, or unrelated failure does not establish that the example detects the intended problem. + +If a missing method prevents the assertion from running, add only enough structure to reach the behavior. Then rerun and observe the expected behavioral failure. + +This requirement applies equally to human and AI developers. Neither confidence nor generated reasoning is evidence that the test can fail. + +### Green: satisfy the demonstrated behavior + +Make the example pass with the smallest coherent production change: + +```python +class Order: + def __init__(self, discounted_subtotal): + self.discounted_subtotal = discounted_subtotal + + def qualifies_for_free_shipping(self): + return self.discounted_subtotal >= Decimal("50.00") +``` + +Run the same test and observe Green. Do not obtain Green by weakening the assertion, skipping the test, swallowing the failure, or silently changing the requested contract. + +“Smallest” does not mean deliberately poor code. It means avoiding unrelated features, speculative abstractions, and cases that no current example or clear requirement demands. + +### Refactor: improve structure without changing behavior + +With the test passing, inspect production and test code. Improve names, remove duplication, or clarify responsibilities while rerunning the relevant tests. + +Perhaps the threshold deserves a named policy. Perhaps no refactoring is justified yet. Refactor is an opportunity to improve structure, not a demand to invent an abstraction. + +An intentional behavior change is not refactoring. It begins another cycle with a new or revised expectation. + +### Continue with an informative example + +A useful next example might show that €49.99 does not qualify. It examines the other side of the boundary instead of repeating the same fact with another large value. + +Another example could clarify whether discounts apply before the threshold. Each example should resolve uncertainty or put useful pressure on the current design. + +Finish this cycle at Green before starting the next behavior. That keeps failures attributable to a small, recent change. + +## Why coverage is not the goal + +Coverage reports which code a suite executed. It does not show that expectations were correct, assertions were meaningful, or important faults would be detected. + +A test can execute every line without checking a result. High coverage and low confidence can therefore coexist. + +Coverage remains useful as a diagnostic. An uncovered branch may reveal a missed case, and a sudden drop may reveal a configuration error. + +The distortion begins when a percentage becomes the objective. People optimize for the visible number while harder questions about risk and fault detection receive less attention. + +TDD often produces coverage because each new behavior is exercised by its example. Coverage is a by-product of the feedback loop, not its purpose. + +Mutation testing can probe whether tests detect selected changes, but its score is also evidence to interpret rather than a universal target. + +## What the retained tests become + +During development, an example helps make one decision. Afterward, the retained test may serve three related purposes: + +- **Specification:** it records behavior the system is expected to preserve. +- **Verification:** it supplies evidence that the current implementation satisfies that behavior. +- **Regression detection:** it warns when a later change violates the recorded behavior. + +Design feedback is different because it occurs while an interface or responsibility is still being formed. The same test may contribute to all these jobs at different times. + +The next chapter turns this first cycle into a deliberate practice and explains how to choose behavior, evidence, expected results, and useful next examples. + +## Sources and further reading + +- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) +- Martin Fowler, [Test Coverage](https://martinfowler.com/bliki/TestCoverage.html) +- Rafique and Mišić, [The Effects of Test-Driven Development on External Quality and Productivity](https://doi.org/10.1109/TSE.2012.28) +- Munir et al., [Considering rigor and relevance when evaluating test driven development](https://doi.org/10.1016/j.infsof.2014.01.001) diff --git a/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md b/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md new file mode 100644 index 0000000..3e7c543 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/02-practising-tdd.md @@ -0,0 +1,319 @@ +# Practising TDD deliberately + +Chapter 1 showed one cycle. This chapter turns that outline into a repeatable practice. + +The difficult part is rarely writing an assertion. It is choosing the next behavior, obtaining faithful evidence, and taking a step small enough to learn from. + +## Start from behavior, not code inventory + +Do not begin by listing functions that need tests. Begin with a behavior a caller needs or a rule the system must preserve. + +A behavior connects a situation, an action, and observable outcomes. For example: given a discounted subtotal of €50, asking about shipping eligibility returns true. + +One behavior does not mean one method, one test class, or one assertion. Several values may need to be checked together to describe one coherent outcome. + +For example, accepting an order may return an identifier and record the same identifier in an event. Two assertions can describe that single commitment. + +The useful limit is one behavioral decision per cycle. If a failure could mean that several unrelated rules are wrong, split the example. + +## Make a provisional test list + +Before the first test, list a few examples that might clarify the behavior. For the shipping rule, the list could be: + +- exactly €50 qualifies; +- €49.99 does not qualify; +- the discounted subtotal determines eligibility; +- an unsupported currency is rejected. + +This is a thinking aid, not a commitment to write the whole suite first. The list will change as each completed cycle reveals more about the requirement and design. + +Choose one example that is important, uncertain, and small enough to complete. Leave the remaining items as reminders. + +Do not implement all listed tests before production code. Doing so commits to an imagined design and discards the feedback one completed cycle should provide to the next. + +## Establish a known baseline + +Run the smallest relevant existing test set before adding the example. + +If it passes, the new failure can be attributed to the new example. If it fails, record the commands and failures before changing anything. + +A pre-existing failure is not the Red for the new behavior. Either restore the baseline or isolate the new work without claiming that an unrelated failure proves anything about it. + +On a new project with no tests, the baseline may be a successful build and confirmation that the test runner starts cleanly. This prevents infrastructure problems from masquerading as Red. + +## Choose where to observe the behavior + +A test needs a route for supplying inputs and observing outcomes. That route is its **observation boundary**. + +For the shipping rule, the boundary could be an `Order` method, a pricing component, an HTTP endpoint, or a complete browser checkout. + +Choose the narrowest stable boundary that can answer the current question without hiding relevant semantics. + +A narrow boundary often gives faster, more diagnostic feedback. A broader boundary is necessary when a database, network, framework, or deployed configuration is part of the risk. + +Do not equate “narrow” with “one method.” Several functions or objects may form one coherent unit of behavior. + +In legacy-code literature, a **seam** is a place where behavior or a dependency can be varied without editing the code at that point. A seam can make a useful observation boundary controllable. + +## Derive the expected result independently + +The basis on which a test decides whether behavior is acceptable is its **oracle**. + +An oracle is weak when the test calculates its expected result with the same algorithm as production. The same mistake can then appear on both sides and the test will pass. + +Prefer evidence independent of the implementation: + +- an agreed example from a requirement or domain expert; +- a known literal such as the €50 boundary; +- an invariant such as “sorting preserves all input elements”; +- a trusted reference implementation; +- a reviewed baseline for complex existing output; +- a relationship between executions when an exact value is unavailable. + +If no reliable oracle exists, exact example-based TDD may not fit the uncertain part of the work. The exception paths later in this chapter explain what to do instead. + +## Observe a meaningful Red + +Write or revise the smallest example of behavior the system does not yet satisfy, then run it. + +Never infer that the test would fail. Human confidence and AI-generated reasoning are not evidence that the test can detect the missing or incorrect behavior. + +Observed Red answers the first question about a new test: can it fail before the production code satisfies it? + +A meaningful Red is: + +- **observed:** the test was actually executed; +- **reproducible:** the same state produces the same failure; +- **relevant:** the intended behavior caused the failure; +- **discriminating:** an important wrong implementation would be detected; +- **attributable:** the new example or changed expectation explains the failure. + +A broken fixture, wrong import, unavailable environment, or unrelated defect is not a behavioral Red. Repair the route to the behavior, return to the known baseline, and run again. + +For a defect, reproduce the defect first. Seeing the regression test fail against the current code shows that the test reaches the problem it claims to protect. + +### If the new test passes immediately + +Do not add production code merely to manufacture a cycle. Investigate why the behavior already appears Green. + +The behavior may already exist, another path may satisfy it accidentally, or the assertion may be insensitive. Verify the setup, action, and oracle. + +When safe, temporarily remove or perturb the relevant behavior and confirm that the test detects the change. Restore the original baseline afterward. + +This sensitivity check does not recreate the design feedback of genuine test-first work. + +If the behavior genuinely exists, retain the test only if its regression value justifies its cost. Then select a behavior that is actually missing. + +### If the test fails for the wrong reason + +Repair the test route or add only the skeletal production structure needed to reach the intended assertion. Do not implement the behavior yet. + +Re-establish the baseline and rerun until the failure reaches the intended rule. + +Do not count a sequence of infrastructure errors as progress through Red. It proves only that the behavior has not yet been observed. + +### If the suite was already failing + +Separate the known failure from the new evidence. Record its command and output, fix it when in scope, or use an agreed isolated baseline. + +Never present an inherited failure as if the new example exposed it. + +## Make the smallest coherent Green + +Change only enough production code to satisfy the current example while preserving existing behavior. + +Then run the same test and observe it pass. Run nearby relevant tests to make sure the local solution did not violate recorded behavior. + +Green must come from satisfying the intended contract. Do not weaken an assertion, skip a test, swallow an error, or silently reinterpret the requirement to obtain it. + +Three implementation strategies are useful: + +- **Obvious Implementation:** write the general solution directly when it is genuinely clear. +- **Fake It:** use a narrow result when a working example will help reveal the next step. +- **Triangulation:** add a contrasting example whose difference creates pressure for a more general rule. + +Fake It is a learning strategy, not a ritual. Hard-coding an answer when the correct implementation is obvious adds ceremony without insight. + +Triangulation is useful when one example permits several plausible rules. A second example supplies information the first did not. + +“Smallest” means the smallest coherent change. It does not require deliberately unreadable code or a solution known to violate an established requirement. + +## Refactor only under Green + +Green means the demonstrated behavior works. It does not mean the design is finished. + +Improve structure while preserving behavior: rename concepts, remove duplication, simplify control flow, move responsibilities, or introduce an abstraction justified by current pressure. + +Run relevant tests after each structural step. If they turn Red, restore Green before continuing. + +An intentional change in observable behavior is not refactoring. It starts another cycle with a new or revised expectation. + +An internal interface may change during refactoring when all callers can change with it. Compatibility of a published interface is observable behavior and must be handled deliberately. + +Refactoring is not mandatory motion. If the code already communicates the current design well, continue to the next behavior. + +## Select the next example for information + +The next test should establish something the current suite does not. + +Useful choices include: + +- the other side of a boundary; +- an empty, missing, or malformed input; +- a competing business rule; +- a failure observed in production; +- a state transition that should be rejected; +- a contrasting example that creates pressure to generalize; +- an invariant over a wider input space. + +Changing €60 to €70 rarely adds information if both values exercise the same rule. Prefer examples that distinguish plausible implementations. + +Finish the current cycle at Green before beginning another behavioral decision. Closed cycles keep failures attributable and make interruptions safer. + +## Change an existing contract deliberately + +Tests are not immutable. When the intended behavior changes, the executable specification must change with it. + +Begin from a known Green baseline. Revise or add an expectation that distinguishes the new contract from the old implementation, then run it and observe Red. + +A narrowed example of behavior that remains supported may stay Green. Keep it as evidence for the retained contract, but obtain Red from another expectation that the old implementation cannot satisfy. + +Change production code until the revised expectation and unaffected behavior are Green. Update documentation, consumers, and broader evidence when compatibility is part of the change. + +Sometimes backward compatibility requires adding a new behavior while preserving the old one. That is a product or interface decision, not something a test framework should decide implicitly. + +Do not rewrite a failing test merely because production code changed accidentally. Change a test only when the intended contract, evidence, or test design has deliberately changed. + +## What deserves a focused test? + +Treat testing as a risk decision, not a census of functions. + +Strong candidates include: + +- business rules and data transformations with meaningful partitions; +- validation, authorization, money, identity, time, and state transitions; +- parsers, serializers, calculations, and boundary conversions; +- concurrency, retry, idempotency, and failure recovery; +- public contracts between independently changing components; +- code that changes often or is difficult to diagnose through broader evidence. + +Data-transforming functions are good candidates because their inputs and outputs often provide a clear oracle. They are not automatically worth a separate test. + +A trivial transformation already exercised through stable public behavior may gain no diagnostic or design value from another test at the function boundary. + +Ask whether a focused test would clarify a contract, distinguish meaningful cases, improve diagnosis, or enable safer change. If none applies, broader evidence may be enough. + +## What makes a retained test useful? + +A useful behavior test normally has these properties: + +- **Relevant:** it protects behavior or risk worth maintaining. +- **Discriminating:** it can fail for an important incorrect implementation. +- **Readable:** its setup, action, and expected outcome communicate the rule. +- **Independent:** its oracle does not merely reproduce production logic. +- **Stable:** structural refactoring does not break it without a behavior change. +- **Diagnostic:** a failure points to a reasonably small question. +- **Repeatable:** unchanged code and environment produce the same result. +- **Economical:** its evidence justifies its runtime and maintenance cost. + +No single property proves quality. A fast, readable test of an irrelevant detail is still low value. + +## Depend on behavior, not replaceable structure + +A durable test depends on behavior at its chosen boundary. It should fail when that behavior changes and remain Green when only replaceable internals change. + +Warning signs include assertions on private state, incidental call order, framework query chains, or storage details hidden behind a public retrieval contract. + +Interaction assertions are valid when requesting an interaction is itself the contract. A checkout component may need to request a charge exactly once. + +That focused test proves only that the request was made. It does not prove that the provider accepted, settled, or reconciled the charge. + +The chapters on choosing evidence and collaborative discovery explain how other evidence answers those questions without forcing one test to prove everything. + +## Use feedback lanes without weakening Red + +Tests created during TDD do not remain a separate species. Once retained, they become part of the project’s verification portfolio. + +During a cycle, run the new or revised test and nearby fast tests. Before handing off a change, run the project’s normal relevant suite. Use broader, slower evidence before a suitably risky transition. + +If only a slower test through a real dependency or an assembled path can faithfully detect the current risk, that slower test is the correct Red. + +Do not replace faithful evidence with an irrelevant focused test merely to make the loop faster. Instead reduce setup, narrow the scenario, improve diagnostics, or create a safe controllable boundary. + +The chapter on [choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md) develops fast, standard, and thorough execution lanes. + +## Keep results trustworthy + +A flaky test damages the feedback loop because its failure no longer points reliably to changed behavior. Treat flakiness as a defect, not background noise. + +Do not rerun until Green and call that verification. Diagnose shared state, time, environment, concurrency, infrastructure, or oracle problems. + +A broad example of stakeholder-visible behavior may guide several focused inner cycles. Keep it branch-local, pending, or outside required gates while it describes unfinished behavior. + +At completion, report what was run, what passed, what could not run, and what uncertainty remains. Never claim verification that was not observed. + +## Explicit exception paths + +The normal Red requirement applies to new behavior and defect corrections. Other kinds of work begin from different evidence states. + +### Characterizing legacy behavior + +A characterization test records what the system currently does so a risky change can preserve selected behavior. It usually begins Green and does not prove that the behavior is correct. + +If no test can run, a minimal behavior-preserving seam may be needed first. After characterization, establish a meaningful Red for the intended change. + +### Pure refactoring + +Pure refactoring begins Green and remains Green because observable behavior is intentionally unchanged. If behavior must change, begin a new cycle. + +### Exploration + +A spike or experiment may be needed to discover feasibility, an interface, or an oracle. Keep exploratory code disposable, capture what was learned, then start deliberate development from an explicit behavior. + +### No useful executable oracle + +Some outcomes require human review, experiments, statistical evaluation, monitoring, or specialist analysis. Apply TDD to deterministic subproblems, but do not pretend an exact assertion proves what it cannot. + +The [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md) introduces these complementary forms of evidence. + +## The operating invariants + +For new behavior and defect corrections, preserve these invariants: + +1. **Known baseline:** existing relevant results are known before the new example. +2. **Explicit decision:** one behavior, risk, boundary, and expected outcome are stated. +3. **Independent oracle:** the expectation is not copied from production reasoning. +4. **Observed Red:** the test fails reproducibly for the expected behavioral reason. +5. **Faithful evidence:** the test can detect the risk it claims to address. +6. **Green integrity:** production satisfies the evidence without weakening it. +7. **Safe refactoring:** structure changes only while behavior remains Green. +8. **Trustworthy result:** flaky or unexplained outcomes are not accepted as evidence. +9. **Closed cycle:** the current behavioral decision returns to Green before the next begins. +10. **Honest completion:** unrun checks and remaining uncertainty are reported. + +These rules protect the meaning of the colors. They are not targets for assertion counts, method coverage, cycle duration, or use of simulated dependencies. + +## A compact working loop + +For each new behavior or defect correction: + +1. State the behavior and why it matters. +2. Make or revise a provisional test list. +3. Establish the relevant baseline. +4. Choose the narrowest boundary faithful to the risk. +5. Derive the expected result independently. +6. Add or revise one example and observe meaningful Red. +7. Make the smallest coherent production change. +8. Observe Green and run nearby relevant tests. +9. Refactor without changing observable behavior. +10. Close the cycle, then choose the next informative example. + +The next chapter turns recurring misunderstandings into concrete symptoms and recovery actions. + +## Sources and further reading + +- Kent Beck, [Canon TDD](https://newsletter.kentbeck.com/p/canon-tdd) +- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) +- Martin Fowler, [Is Changing Interfaces Refactoring?](https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html) +- Michael Feathers, *Working Effectively with Legacy Code* +- Gerard Meszaros, [xUnit Test Patterns](http://xunitpatterns.com/) diff --git a/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md b/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md new file mode 100644 index 0000000..a7dfdcb --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/03-common-misconceptions-and-failure-modes.md @@ -0,0 +1,339 @@ +# Common misconceptions and failure modes + +TDD can disappoint because it was misapplied, because its evidence did not match the risk, or because it was a poor fit for the work. + +Those explanations are not interchangeable. Calling every criticism a misunderstanding makes TDD unfalsifiable. Calling every failed adoption proof against TDD ignores how the practice was used. + +This chapter first diagnoses common misuse. It then separates those failure modes from serious objections that remain even when TDD is practised competently. + +Each diagnostic uses the same questions: + +- What is the misconception? +- What symptom appears in the work? +- Why does it weaken feedback or design? +- How can the team recover? +- Where is the deeper explanation? + +## Mistaking a by-product for the goal + +### “TDD means testing every method” + +**Misconception:** Each function, method, or class must receive its own test. + +**Symptom:** The suite mirrors the code inventory. Trivial accessors receive tests while important cross-component behavior remains uncertain. + +**Why it fails:** Program structure does not determine testing value. Structure-coupled tests add maintenance cost and break during harmless refactoring. + +**Recovery:** Start from behavior and risk. Add focused evidence when it clarifies a contract, distinguishes important cases, improves diagnosis, or enables safe change. + +**Read next:** “Start from behavior, not code inventory” and “What deserves a focused test” in [Practising TDD deliberately](02-practising-tdd.md). + +### “Coverage is the objective” + +**Misconception:** A high coverage percentage demonstrates test quality or successful TDD. + +**Symptom:** Tests execute lines without discriminating assertions. Teams avoid valuable refactoring or write low-value tests to protect a target. + +**Why it fails:** Coverage measures execution, not correctness of the oracle, fault detection, or importance of the behavior. + +**Recovery:** Use coverage to find unexamined code, then ask which risk matters and what evidence could detect a meaningful error. Do not treat the percentage as the product. + +**Read next:** “Why coverage is not the goal” in [Why test-driven development exists](01-why-tdd.md). + +### “Test-first automatically produces good tests” + +**Misconception:** Writing an assertion before production code makes the resulting test valuable. + +**Symptom:** A test is fast and test-first but checks an irrelevant detail, copies production reasoning, or cannot detect a plausible defect. + +**Why it fails:** Timing creates an opportunity for design feedback. Relevance, independence, sensitivity, stability, and fidelity determine the quality of the evidence. + +**Recovery:** Review the behavior, risk, boundary, and oracle. Confirm that an important wrong implementation would fail. + +**Read next:** “Derive the expected result independently” and “What makes a retained test useful” in [Practising TDD deliberately](02-practising-tdd.md). + +## Breaking the meaning of Red + +### “The change is obvious, so Red can be skipped” + +**Misconception:** A developer or AI can reason that the test would have failed without executing it. + +**Symptom:** Test and implementation appear together, followed only by a passing run. Nobody knows whether the test was ever sensitive to the missing behavior. + +**Why it fails:** The test may use the wrong setup, miss the production path, contain a tautology, or assert a result the system already produced. + +**Recovery:** Run the new or revised test against the unchanged behavior. Require a reproducible failure caused by the intended rule before implementation. + +**Read next:** “Observe a meaningful Red” in [Practising TDD deliberately](02-practising-tdd.md). + +### “Any failure counts as Red” + +**Misconception:** A compilation error, broken fixture, missing import, or unavailable service proves the behavioral example. + +**Symptom:** Production code changes while the test has not yet reached the behavior it claims to describe. + +**Why it fails:** The failure proves only that the execution path is broken. It does not show that the oracle distinguishes the current behavior from the intended behavior. + +**Recovery:** Repair the test route with the smallest necessary structure. Restore the known baseline, then rerun until the expected behavioral failure is visible. + +**Read next:** “If the test fails for the wrong reason” in [Practising TDD deliberately](02-practising-tdd.md). + +### “An existing suite failure can serve as the new Red” + +**Misconception:** Any Red result in the repository is enough to begin the production change. + +**Symptom:** A pre-existing or unrelated failure is later reported as proof that the new test detected the missing behavior. + +**Why it fails:** The change has no attributable baseline. Green may hide the old failure or leave the new behavior untested. + +**Recovery:** Record or repair existing failures first. Isolate the new example and observe its own relevant failure. + +**Read next:** “Establish a known baseline” in [Practising TDD deliberately](02-practising-tdd.md). + +### “A new test that passes immediately is close enough” + +**Misconception:** An immediately Green test can be accepted as evidence of a completed Red–Green cycle. + +**Symptom:** The team adds production code anyway or retains an assertion without checking whether it can detect a defect. + +**Why it fails:** The behavior may already exist, be satisfied accidentally, or never be exercised by the test. + +**Recovery:** Inspect setup, action, and oracle. When safe, perturb the relevant behavior to check sensitivity. If the behavior already exists, choose a genuinely missing behavior. + +**Read next:** “If the new test passes immediately” in [Practising TDD deliberately](02-practising-tdd.md). + +## Turning the cycle into ceremony + +### “Write the entire test suite first” + +**Misconception:** Test-first means specifying every case before any implementation. + +**Symptom:** Many tests fail at once and encode an interface imagined before the first design feedback arrives. + +**Why it fails:** Later tests cannot benefit from discoveries made while satisfying earlier examples. Failures also become harder to attribute. + +**Recovery:** Keep a provisional test list, select one informative example, and close its cycle before choosing the next. + +**Read next:** “Make a provisional test list” in [Practising TDD deliberately](02-practising-tdd.md). + +### “One behavior means one assertion” + +**Misconception:** Every test may contain only one assertion. + +**Symptom:** One coherent outcome is split across tests with duplicated setup, or related facts become difficult to read together. + +**Why it fails:** Assertion count is a syntactic metric. The useful boundary is one behavioral decision and one understandable reason for failure. + +**Recovery:** Keep assertions together when they describe one outcome. Split a test when it mixes independent rules or failures would be ambiguous. + +**Read next:** “Start from behavior, not code inventory” in [Practising TDD deliberately](02-practising-tdd.md). + +### “Minimal Green requires deliberately bad code” + +**Misconception:** Every example must be satisfied with a hard-coded or knowingly poor implementation. + +**Symptom:** Obvious solutions are delayed by ritual, or code violates already-known requirements merely to demonstrate Fake It. + +**Why it fails:** The goal is a small, attributable learning step, not artificial incompetence. + +**Recovery:** Use Obvious Implementation when the rule is clear. Use Fake It or triangulation when a narrow implementation will expose uncertainty or useful design pressure. + +**Read next:** “Make the smallest coherent Green” in [Practising TDD deliberately](02-practising-tdd.md). + +### “Refactor may add behavior” + +**Misconception:** Any cleanup performed after Green belongs to Refactor, even when observable behavior changes. + +**Symptom:** New rules appear without an observed Red, and a failing test during cleanup is dismissed as an expected side effect. + +**Why it fails:** The evidence no longer distinguishes structural change from behavioral change. + +**Recovery:** Refactor only while observable behavior remains Green. Begin a new cycle when the intended contract changes. + +**Read next:** “Refactor only under Green” and “Change an existing contract deliberately” in [Practising TDD deliberately](02-practising-tdd.md). + +## Choosing evidence by fashion + +Test **scope** describes how much of the system participates. A focused test observes a narrow coherent behavior. An integration test includes real collaborators or infrastructure. + +An end-to-end test observes an assembled path from an outside entry point. The next chapter defines these and other evidence labels in more detail. + +### “TDD means unit tests” + +**Misconception:** Red–Green–Refactor must always operate through an isolated unit test. + +**Symptom:** A fast test bypasses the database, framework, contract, or assembly behavior that creates the current risk. + +**Why it fails:** TDD determines when feedback enters development. It does not require one fixed test scope. + +**Recovery:** Use the narrowest boundary that faithfully answers the current question. A focused, integration, or end-to-end test can be the correct Red. + +**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). + +### “The narrowest test is always best” + +**Misconception:** A focused test is preferable even when it bypasses the component that creates the risk. + +**Symptom:** A suite is fast but cannot detect mapping, persistence, framework, configuration, or deployment defects. + +**Why it fails:** Raw speed is not useful when the evidence is unfaithful to the question. + +**Recovery:** Prefer the shortest trustworthy loop that faithfully exercises the current risk. Use a broader Red when the broader boundary is what can fail. + +**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). + +### “End-to-end tests are always better because they are realistic” + +**Misconception:** The broadest available boundary provides the strongest evidence for every question. + +**Symptom:** Inner development cycles become slow and failures are hard to localize. Many scenarios repeat the same expensive setup. + +**Why it fails:** Broad tests include more collaborating parts, but they may still use unrealistic data or simulated services. Scope, fidelity, speed, and diagnosis are separate qualities. + +**Recovery:** Use focused evidence for local rules and broader evidence for assembly, journeys, and deployment risks. Keep only scenarios whose additional scope answers a distinct question. + +**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). + +### “One test level proves the whole feature” + +**Misconception:** A focused, integration, or end-to-end test can answer every relevant question about one behavior. + +**Symptom:** A team treats a recorded request as proof of an external effect, or repeats every local rule through the browser. + +**Why it fails:** Different boundaries expose different failures. One focused interaction does not prove provider acceptance, and one broad journey may diagnose a local rule poorly. + +**Recovery:** Map each material risk to the smallest faithful evidence. Remove tests that duplicate another test’s question without adding useful confidence. + +**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md) and the [worked examples](07-worked-examples.md). + +## Confusing testability with mockability + +A **test double** is a controlled substitute for a collaborator. A **mock** is a double used to set or verify expected interactions. + +### “Every collaborator should be mocked” + +**Misconception:** TDD requires replacing every dependency with a test double. + +**Symptom:** Interfaces exist only for tests, assertions mirror internal call order, and refactoring breaks many tests without changing behavior. + +**Why it fails:** Doubles can improve control and diagnosis, but excessive isolation couples tests to imagined implementation structure. + +**Recovery:** First look for a stable behavioral boundary. Use a double when control or observation is difficult and the asserted interaction is a meaningful contract. + +**Read next:** “Testability rather than mockability” in the [worked examples](07-worked-examples.md). + +### “A mock proves the external effect” + +**Misconception:** Verifying that code requested a charge, message, or write proves that the real system completed it. + +**Symptom:** Focused tests pass while provider contracts, credentials, serialization, settlement, or reconciliation fail. + +**Why it fails:** The test proves only the interaction visible at its boundary. + +**Recovery:** Keep the focused interaction test when the request matters. Add compatibility, integration, end-to-end, or operational evidence for the external effect and its failure modes. + +**Read next:** [Choosing evidence and feedback lanes](04-choosing-evidence-and-feedback-lanes.md). + +## Misusing retained tests + +### “Tests must never change” + +**Misconception:** Changing an existing test is equivalent to weakening it. + +**Symptom:** The suite preserves obsolete requirements, or developers work around a test instead of updating an intentional contract change. + +**Why it fails:** Retained tests are executable specifications. A specification must change when the intended behavior changes. + +**Recovery:** Distinguish an intentional contract change from an accidental regression. Revise the expectation first, observe Red, update production behavior, and preserve unaffected evidence. + +**Read next:** “Change an existing contract deliberately” in [Practising TDD deliberately](02-practising-tdd.md). + +### “A characterization test proves legacy behavior is correct” + +**Misconception:** Recording current output means the output is desired. + +**Symptom:** Defects and historical accidents become protected as requirements without review. + +**Why it fails:** A characterization test establishes what happens now, not what should happen. + +**Recovery:** Name the behavior being preserved and why. Review suspicious output with a domain owner, then establish Red for any intentional correction. + +**Read next:** The legacy sequence in [Applying TDD without dogma](06-applying-tdd-without-dogma.md) and the [worked examples](07-worked-examples.md). + +### “A flaky test can be rerun until Green” + +**Misconception:** A passing rerun is adequate verification. + +**Symptom:** Failures are ignored when a later execution happens to pass. The suite’s colors no longer carry a stable meaning. + +**Why it fails:** An unexplained result cannot be attributed to code behavior. It may hide shared state, races, time, infrastructure faults, or a weak oracle. + +**Recovery:** Treat flakiness as a defect. Reproduce and remove its cause, or exclude the test from required gates while clearly reporting the lost evidence. + +**Read next:** “Keep results trustworthy” in [Practising TDD deliberately](02-practising-tdd.md). + +## Asking TDD to solve a different problem + +### “TDD replaces requirement discovery” + +**Misconception:** Writing examples eliminates the need to understand users, goals, constraints, and competing interpretations. + +**Symptom:** Precise tests encode the wrong behavior, and passing software still fails the user’s need. + +**Why it fails:** TDD checks an expressed expectation. It cannot decide whether that expectation is valuable or complete. + +**Recovery:** Use collaborative examples, product discovery, domain expertise, and review to choose behavior. Use TDD to develop the agreed behavior in small, observable steps. + +**Read next:** [Behavior-driven development (BDD) and collaborative discovery](05-bdd-and-collaborative-discovery.md). + +### “TDD grows the whole architecture automatically” + +**Misconception:** Local test-first decisions remove the need for architectural reasoning, experiments, integration, and operational feedback. + +**Symptom:** Local units are easy to test, but the assembled system has poor boundaries, delivery risks, or quality-attribute failures. + +**Why it fails:** A TDD cycle provides local design feedback at its observation boundary. It does not by itself compare system-level options or validate operation at scale. + +**Recovery:** Combine small cycles with a thin runnable system slice, disposable experiments, deliberate architecture, broader evidence, and production feedback. + +**Read next:** [Applying TDD without dogma](06-applying-tdd-without-dogma.md) and the [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md). + +### “Every valuable outcome has an exact oracle” + +**Misconception:** Every quality can be reduced to one deterministic expected value. + +**Symptom:** Subjective, statistical, emergent, or operational qualities receive misleading assertions because the workflow expects Red–Green. + +**Why it fails:** Some questions require ranges, properties, expert review, experiments, monitoring, or repeated evaluation. + +**Recovery:** Apply exact example-based TDD to deterministic subproblems. Use the form of evidence appropriate to the remaining uncertainty. + +**Read next:** “No useful executable oracle” in [Practising TDD deliberately](02-practising-tdd.md) and the [adjacent-practices appendix](appendices/adjacent-practices-and-further-reading.md). + +## Serious objections are not misconceptions + +The following concerns can remain valid even when no failure mode above is present. They should be evaluated against a concrete alternative, not dismissed as resistance to discipline. + +They include: + +- the cycle and retained suite may cost more than the feedback repays; +- isolation-heavy tests can damage production design; +- strict micro-cycles may fit schema, distributed, or cross-cutting changes poorly; +- empirical results vary by task, team, experience, and comparison process; +- local design feedback cannot choose or validate the whole system architecture. + +These are evaluation questions, not errors to “recover” from. [Applying TDD without dogma](06-applying-tdd-without-dogma.md) examines their evidence, trade-offs, and adaptations. + +## A recovery sequence + +When a TDD effort feels slow or unhelpful: + +1. State the behavior and current risk again. +2. Check whether the oracle is independent and meaningful. +3. Confirm that Red was observed for the expected reason. +4. Check whether the boundary faithfully exercises the risk. +5. Remove structural coupling and unnecessary setup. +6. Restore trustworthy Green before starting another behavior. +7. Choose different evidence when TDD is not the right mechanism. + +The goal is not ritual compliance. It is early, trustworthy information that improves the next development decision. diff --git a/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md b/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md new file mode 100644 index 0000000..c3426a6 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/04-choosing-evidence-and-feedback-lanes.md @@ -0,0 +1,152 @@ +# Choosing evidence and feedback lanes + +The TDD loop tells you when to seek feedback. It does not tell you which parts of the system must participate in each check. + +A pricing rule can be driven through a focused test. A transaction rule may need a real database. A deployment route may need an assembled system. Each is compatible with test-first work. + +The governing rule is: + +> Prefer the shortest trustworthy feedback loop that faithfully exercises the current risk. + +Fast feedback matters only when it is relevant, repeatable, diagnostic, and capable of detecting the failure under discussion. + +## Workflow, scope, and fidelity are separate decisions + +TDD is a workflow: express unsupported behavior, observe a meaningful failure, make the behavior work, and improve the design under passing evidence. + +Test scope asks how much code participates. Fidelity asks which real semantics, dependencies, data, configuration, and runtime conditions participate. + +These dimensions are related, but they are not interchangeable. A broad test built on substitutes may be less faithful to database behavior than a narrow integration test using the production database engine. + +Nor does broad scope guarantee realism. A test can traverse many layers while using invented data, simplified infrastructure, and configuration unlike production. + +Choose the observation boundary from the question and risk. Do not choose it from a preferred label. + +## What the common labels usually ask + +Teams use test labels differently. Agree on local meanings, but preserve the distinct questions behind them. + +| Evidence | Main question | Common limitation | +| --------------- | --------------------------------------------------------------------- | --------------------------------------------------- | +| Focused or unit | Does one rule or coherent unit of behavior hold at a narrow boundary? | May omit infrastructure and assembly semantics | +| Component | Does a substantial part work through its public boundary? | May replace external systems or deployment wiring | +| Integration | Do real components or infrastructure collaborate correctly? | Setup and diagnosis often cost more | +| Contract | Do independently changing parties agree on exercised interactions? | Does not prove full provider behavior or deployment | +| Acceptance | Does a stakeholder-visible capability satisfy agreed examples? | May not use a fully deployed production-like path | +| E2E or system | Does an assembled path work from an outside observer's view? | Usually slower and harder to diagnose | +| Smoke | Is the deployed system basically reachable and operable? | Intentionally shallow | + +A focused test need not mean one method or class. Its unit can be any coherent behavior that is fast to control, observe, and diagnose. + +A component test commonly exercises a substantial owned part through a public boundary. An integration test focuses on collaboration with another real component or infrastructure. + +A contract test may be consumer-driven, provider-owned, or schema-based. Its defining concern is compatibility between parties that can change independently. + +An acceptance test concerns agreed capability. An E2E test concerns assembled operation. One test may satisfy both descriptions, but the questions remain useful when deciding what its result proves. + +## One behavior can need distinct evidence + +Suppose checkout calculates a total, stores an order, requests payment, and shows confirmation. + +A useful evidence set might contain: + +1. focused examples for pricing and order-state rules; +2. a database integration test for constraints and transaction behavior; +3. a contract test for the payment request and expected response; +4. an acceptance example for the stakeholder-visible outcome; +5. one E2E check for deployed checkout wiring; +6. monitoring and reconciliation for real settlement failures. + +No single item proves the whole behavior. The mock that records a charge request cannot prove settlement. The E2E path need not repeat every pricing partition already established at a cheaper boundary. + +Keep evidence when it answers a unique, material question. Remove duplication when a cheaper check answers the same question and the broader check covers no additional risk. + +## Evidence fidelity follows the risk + +Use real infrastructure when its semantics are part of the claim. Examples include transactions, SQL collation, serialization, filesystem behavior, queues, framework wiring, and deployed configuration. + +Use a narrower boundary when the risk is a domain rule and broader machinery would only add setup, delay, and ambiguous failures. + +If only an integration or E2E test can demonstrate the missing behavior, that slower test is the correct Red. Do not replace faithful evidence with an irrelevant fast test. + +You may later add a faster companion test for diagnosis or design feedback. It complements the broader evidence; it does not retroactively prove the broader risk. + +## Feedback lanes are execution cadences + +Retained tests become part of the project's verification portfolio. They are not permanently a separate class called “TDD tests.” + +How often a check runs is another decision. A focused test can be slow, and an integration test can be fast. Assign lanes from measured cost, reliability, diagnostic value, environment needs, and risk. + +### Fast or local lane + +This lane supports the immediate development loop. It normally includes the new or revised test, nearby deterministic checks, and other tests that return useful local feedback quickly. + +Its purpose is not to prove the entire change safe. It keeps each behavioral and structural decision small enough to diagnose. + +### Standard or change-validation lane + +This lane establishes confidence in the affected area before handoff or integration. It can include the changed module, relevant components, integrations, and contracts. + +Its composition should follow change impact. “Standard” means the team's normal evidence for a change, not every check in the repository. + +### Thorough or system-validation lane + +This lane supplies broader evidence before a risky transition. It may include the full relevant suite, E2E paths, environment-dependent checks, and specialized quality evaluation. + +Typical triggers are merge, release, infrastructure change, high-impact migration, or a scheduled verification run. The trigger depends on project risk and delivery design. + +## Do not impose universal time limits + +A five-second suite may feel slow in one inner loop and excellent in another system. A ten-minute suite may be acceptable before merge but unusable after every edit. + +Measure actual duration and feedback delay. Then give each lane a project-specific budget that preserves its purpose. + +Revisit the classification as tests, infrastructure, and risks change. A lane is an operating agreement, not an intrinsic property stored forever in a test name. + +## Cadence through a TDD change + +Use the lanes without weakening the Red–Green–Refactor invariants: + +1. Before Red, run the smallest relevant baseline or record known failures. +2. For Red, run the new or revised test and inspect its expected failure. +3. For Green, run that test plus nearby fast regressions. +4. During refactoring, rerun the fast lane after each meaningful structural step. +5. Before handoff or integration, run the standard lane. +6. Before the applicable risky transition, run the thorough lane. + +If the faithful Red is slow, run it when the behavior changes and use faster supporting checks between those runs. Never claim the slow risk is verified when only its companion checks ran. + +Report which lanes and commands ran, their results, which broader evidence did not run, and the uncertainty that remains. + +## Continuous integration extends the feedback loop + +Local lanes help one developer. Continuous integration tests whether the combined work still agrees at the team boundary. + +CI commands should have stable meanings. A required lane should mean that its known evidence passes, not that some failures are expected and ignored. + +Many teams run fast and standard lanes for each proposed change, then schedule or gate thorough checks according to cost and risk. This is a pattern, not a universal prescription. + +An unfinished outer example can remain pending, branch-local, or excluded from required gates. A permanently Red required suite destroys its value as a signal. + +Do not rerun a flaky lane until it happens to pass. Diagnose or explicitly isolate the unreliable check, record the lost evidence, and restore trust promptly. + +## A practical selection checklist + +Before adding or assigning a check, ask: + +- What incorrect behavior or risk must this evidence detect? +- At which boundary can that outcome be controlled and observed? +- Which real semantics must participate for the result to be faithful? +- Is a cheaper check already answering the same question? +- How quickly and reliably does this check return diagnostic feedback? +- In which lane will its result influence a useful decision? + +The next chapter addresses an earlier question: how a team discovers and expresses the behavior that these checks should evaluate. + +## Sources and further reading + +- Martin Fowler, [Test-Driven Development](https://martinfowler.com/bliki/TestDrivenDevelopment.html) +- Martin Fowler, [The Practical Test Pyramid](https://martinfowler.com/articles/practical-test-pyramid.html) +- Pact, [Introduction to contract testing](https://docs.pact.io/) +- Jez Humble, [Continuous Delivery](https://continuousdelivery.com/) +- Kent Beck, [TDD is Kanban for Code](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code) diff --git a/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md b/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md new file mode 100644 index 0000000..272eccd --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/05-bdd-and-collaborative-discovery.md @@ -0,0 +1,151 @@ +# BDD and collaborative discovery + +The earlier chapters assumed that one behavior could be stated clearly enough to drive a cycle. That is a useful way to learn TDD, but real product work often begins before such clarity exists. + +This chapter steps back from executing evidence to ask how a team decides which behavior matters and what an acceptable outcome means. + +TDD can make a developer's interpretation precise. It cannot, by itself, prove that the interpretation reflects a stakeholder's need. + +Behavior-Driven Development and related practices improve the input to the development loop through shared language, concrete examples, and collaboration. + +## Why BDD entered the conversation + +Dan North developed BDD while looking for better ways to teach and practise TDD. Learners struggled with where to start, what to test, how much to test at once, and how to name tests. + +Talking about behavior, examples, and scenarios redirected attention from test structure toward the outcome a caller needs. + +BDD later grew beyond a teaching reformulation. Its practice commonly includes discovery, formulation, automation, domain language, and outside-in thinking. + +It is therefore misleading to call BDD either wholly separate from TDD or merely TDD with renamed tests. They overlap, while BDD places stronger emphasis on collaborative discovery and shared understanding. + +## Behavior belongs to a caller + +The caller of a behavior need not be a person using a browser. + +- A customer may use a web application. +- Another service may use an API. +- Application code may use a library. +- An operator may use a command or alert. +- A reporting system may use a data product. + +Describe the result in that caller's vocabulary and at a boundary where the caller can observe it. + +“The customer receives an order confirmation” describes behavior. “The handler invokes repository method X and sets field Y” usually describes a current implementation. + +The caller perspective does not forbid interaction checks. Requesting a charge exactly once may be part of a component's contract. The check proves the request, not that an external provider settled it. + +## Given–When–Then clarifies an example + +BDD examples often use **Given–When–Then**: + +- **Given:** the relevant starting situation; +- **When:** the event or action under discussion; +- **Then:** the outcome visible at the chosen boundary. + +This structure separates context, trigger, and outcome. It can expose an example that combines several behaviors or observes private structure instead of a useful result. + +It does not make an example correct. The team still needs a trustworthy oracle, representative conditions, and agreement that the outcome matters. + +Gherkin and tools such as Cucumber are optional. They earn their cost when scenarios are genuinely read, discussed, or co-authored across roles. + +A normal test framework is often clearer when developers are the only audience. Tool choice does not determine whether the team is practising BDD. + +## Discovery precedes automation + +Automation preserves an interpretation. Collaborative discovery tests that interpretation before code makes it expensive. + +Useful conversations examine: + +- the business rule and its purpose; +- concrete examples and counterexamples; +- boundaries, exceptions, and failure outcomes; +- vocabulary that means the same thing to each participant; +- unanswered questions and assumptions; +- evidence needed to know the capability works. + +The practice often called **Three Amigos** brings product or domain, development, and testing perspectives together. The name describes viewpoints, not a required meeting size or set of job titles. + +Each viewpoint asks different questions. Product clarifies value and rules, development exposes technical consequences, and testing searches for ambiguity, boundaries, and missing risks. + +Example Mapping is one lightweight way to organize the conversation into a story, rules, examples, and questions. The cards are disposable; the shared understanding is the product. + +An executable scenario is most valuable after this conversation. Automating an unresolved example only makes uncertainty repeatable. + +## Related practices overlap without becoming synonyms + +**BDD** is a family of discovery, formulation, and automation practices centered on behavior in shared language. + +**Acceptance Test-Driven Development (ATDD)** clarifies acceptance examples collaboratively before implementation and uses them to guide development. + +**Specification by Example** discovers and documents requirements through concrete examples, often retaining selected examples as living specifications. + +All three can produce executable examples and improve the outer input to TDD. Their histories, communities, and emphases differ, so treating them as identical loses useful distinctions. + +None requires every conversational example to become an automated test. Automate examples when repeated evaluation and maintenance provide continuing value. + +## Outer examples and inner TDD cycles + +A stakeholder-visible example may be too broad to provide feedback after every code edit. It can still guide a series of smaller cycles. + +This pattern is called **double-loop TDD** or an outer acceptance loop with inner TDD cycles. + +**Outside-in TDD** is a broader design direction that begins at a caller-visible boundary and discovers collaborators inward. It can use double loops, but the terms are not synonyms. + +The outer example states a capability at a caller-visible boundary. It helps select the next thin slice and shows when the assembled capability is complete. + +Inner Red–Green–Refactor cycles implement one observable behavior at a narrower faithful boundary. Each cycle supplies faster design and diagnostic feedback. + +The loops answer different questions: + +| Loop | Main question | Normal cadence | +| ----- | --------------------------------------------------------------------------------- | -------------------------------------- | +| Outer | Does the stakeholder-visible capability work through the selected broad boundary? | At meaningful slice or assembly points | +| Inner | Does the next focused behavior work, and can its design be improved safely? | Repeated during implementation | + +The outer example does not replace focused evidence. Inner checks do not prove assembly. Use both only when each answers a distinct risk. + +## Keep the outer loop honest + +An outer example that represents unfinished behavior should not make the required shared suite permanently Red. + +Keep it pending, branch-local, or outside required gates until it is expected to pass. Its state must be explicit so nobody mistakes excluded evidence for verified behavior. + +When the outer example turns Green, run the standard lane and any thorough evidence warranted by the risk. One scenario does not establish unmentioned rules, quality attributes, or production outcomes. + +If an outer scenario takes days to complete, the inner loop still needs frequent closed cycles. A long-lived target is a planning aid, not permission to leave ordinary TDD cycles Red. + +## A bounded collaborative flow + +For a new capability: + +1. Identify the caller, desired outcome, and business purpose. +2. Discover rules, examples, counterexamples, and unanswered questions together. +3. Formulate one clear outer example when it will guide or verify the work. +4. Select a thin stakeholder-visible slice. +5. Implement it through closed inner Red–Green–Refactor cycles. +6. Revisit the outer example and any questions exposed by implementation. +7. Retain only evidence whose continuing value justifies its cost. + +This flow is not mandatory ceremony. A clear, low-risk rule may need only a short conversation and a focused example. + +## What BDD does not replace + +BDD does not replace product discovery, user research, architecture, exploratory testing, or specialized quality evaluation. + +Concrete examples can create false confidence when stakeholders are missing, vocabulary is unresolved, or the examples cover only the happy path. + +Living specifications also require maintenance. If scenarios duplicate lower-level checks, describe obsolete behavior, or serve no reader, they become expensive noise. + +Use collaboration where misunderstanding is a material risk. Use executable scenarios where repeated evaluation makes that understanding safer to change. + +The applied chapters show how these ideas behave in greenfield, legacy, and organizational settings. The appendix points to broader practices that complement TDD without belonging to its core loop. + +## Sources and further reading + +- Dan North, [Introducing BDD](https://dannorth.net/blog/introducing-bdd/) +- Dan North, [BDD is like TDD if...](https://dannorth.net/blog/bdd-is-like-tdd-if/) +- Cucumber, [Behaviour-Driven Development](https://cucumber.io/docs/bdd/) +- Agile Alliance, [Behavior-Driven Development](https://agilealliance.org/glossary/bdd/) +- Agile Alliance, [Acceptance Test-Driven Development](https://agilealliance.org/glossary/atdd/) +- Cucumber, [Example Mapping](https://cucumber.io/docs/bdd/example-mapping/) +- Steve Freeman and Nat Pryce, [Growing Object-Oriented Software, Guided by Tests](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/) diff --git a/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md b/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md new file mode 100644 index 0000000..8cec6ad --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/06-applying-tdd-without-dogma.md @@ -0,0 +1,231 @@ +# Applying TDD without dogma + +TDD exchanges effort now for earlier design feedback and reusable regression evidence. Whether that exchange is worthwhile depends on the behavior, risk, and cost of obtaining trustworthy feedback. + +This chapter examines those limits, then shows how entry into TDD differs for greenfield and legacy work. + +## Where TDD tends to earn its cost + +TDD has leverage when an executable oracle can express the next behavior, the code is likely to change, and early feedback can still influence its interface or responsibilities. + +Typical candidates include business rules, transformations, state machines, validation, parsers, authorization, calculations, and recovery logic. + +The case becomes stronger when mistakes are expensive, discovered cases are easy to forget, or debugging through a broader system is slow. + +TDD can also help when the final design is unclear but the next concrete behavior is known. The example constrains one decision without pretending to settle the whole architecture. + +## Where another mode may be better + +TDD supplies less leverage when the immediate goal is learning rather than stable behavior. + +Throwaway spikes, diagnostic scripts, visual exploration, and early research may need experiments or human review before an executable contract is useful. + +Pure wiring may contain little behavior beyond framework configuration. A mock-heavy focused test may only restate calls, while an integration test can answer the real question directly. + +Stable code that is not changing may not justify retrospective tests. Coverage is not a moral debt. + +Separate exploration from productionization. Explore cheaply, then preserve the parts that will survive behind explicit contracts and appropriate evidence. + +## Serious objections and what survives them + +The practical question is not whether TDD is always good. It is which mechanisms help in this context, compared with the way the work would otherwise be performed. + +### “The empirical evidence is mixed” + +Studies report varied effects on productivity and defects. Results depend on task, experience, cycle discipline, study design, and especially the comparison process. + +Do not claim that TDD is proven to make every team faster or every product better. + +The narrower claim is mechanistic. Executable examples can clarify outcomes, small batches can localize failures, and retained behavior tests can expose later regressions. + +Those mechanisms can be observed during work. Whether they repay their cost remains contextual. + +### “TDD cannot design the architecture” + +Focused examples can reveal awkward interfaces and misplaced responsibilities. They cannot select a sound system architecture by themselves. + +Architecture also needs product discovery, quality-attribute analysis, integration evidence, experiments, and deliberate decisions about boundaries and deployment. + +A walking skeleton can probe an initial architecture. Its result remains provisional rather than proof that the larger design is correct. + +### “TDD damages design” + +Tests can encourage interfaces, dependency injection, indirection, and fragmented responsibilities that the production problem never required. + +This happens when design is optimized for isolated mockability instead of useful controllability, observability, and domain boundaries. + +Writing a test first does not require mocking every collaborator. Run fast deterministic owned code together and introduce substitution where a real boundary justifies it. + +The test should make a production interface easier to use. It should not make every internal function replaceable. + +### “Tests can also be wrong” + +A passing test proves only that the observed outcome matched its oracle. The requirement, example, test data, boundary, or implementation can still be wrong. + +Derive expected results independently, review consequential examples, and use several forms of evidence when one oracle cannot cover the risk. + +Delete obsolete tests and revise tests when the intended contract changes. Preserving an incorrect specification is not safety. + +### “TDD slows obvious work down” + +On an obvious, low-risk, short-lived change, formalizing every small step may cost more than it reveals. + +Use Obvious Implementation when the solution is clear. TDD still requires an observed behavioral Red, but it does not require artificial hard-coding or unnecessary triangulation. + +The trade is effort now for possible savings in misunderstanding, debugging, regression detection, and future change. Reduce the ceremony when those benefits are unlikely. + +### “Strict micro-cycles do not fit every change” + +Schema migrations, distributed protocols, and cross-cutting changes may need a coherent slice before any useful test can execute. + +Keep one behavioral decision in flight, but let the smallest coherent step match the system. “Small” is a feedback property, not a fixed line count or time limit. + +### “Tests calcify the code” + +A suite coupled to private methods, call order, framework details, or storage structure makes harmless refactoring expensive. + +That suite is a liability whether its tests were written before or after the implementation. + +Behavior-focused tests reduce this problem but do not remove maintenance cost. Intended behavior changes still require test changes. + +### “TDD does not fit my domain” + +Some domains lack stable exact outcomes. Aesthetic judgment, model quality, emergent behavior, and exploratory analysis may require review, experiments, datasets, or monitoring. + +Scope TDD to parts with useful executable oracles. These may be examples, invariants, statistical thresholds, or controlled properties rather than exact outputs. + +An ML system may use ordinary tests for schemas and transformations while model quality uses dataset evaluation and production monitoring. + +The claim is never “test-drive everything.” + +### “TDD gets credit for ordinary engineering practices” + +Test-first timing, small batches, automation, regression tests, and refactoring are separable mechanisms. Teams can use some without adopting canonical TDD. + +TDD packages them into a disciplined feedback sequence. Its value should be judged against the actual alternative, not against an imaginary process with no checking or design thought. + +## Starting a new project + +Begin with a **walking skeleton**: the thinnest end-to-end slice that proves the build, test runner, delivery path, and runtime can work together. + +The first behavior may be a service answering a health request. This removes delivery uncertainty before application logic accumulates. + +Treat the skeleton as a provisional architecture probe. It demonstrates connection and deployability, not business correctness or long-term architectural fitness. + +Then choose one thin stakeholder-visible capability. An outer acceptance example can give it a destination; focused inner cycles develop the rules and interfaces needed to reach it. + +Do not design a complete acceptance suite in advance. Let one completed slice teach the team about vocabulary, architecture, infrastructure, and feedback cost. + +Keep required gates trustworthy. An unfinished outer example may remain pending, branch-local, or excluded from required gates until it represents behavior expected to pass. + +The greenfield sequence is therefore: + +1. establish the smallest runnable build, test, and delivery skeleton; +2. select one stakeholder-visible capability; +3. frame one concrete behavior and its oracle; +4. drive its inner behavior through observed Red–Green–Refactor cycles; +5. verify the assembled slice with the appropriate outer evidence; +6. use what was learned to select the next slice. + +Chapter 7 follows this sequence in a complete example. + +## Starting in legacy code + +Legacy code may not permit a clean Red–Green–Refactor cycle immediately. Hard-wired dependencies and unknown behavior can prevent safe observation. + +Begin with the risk of the intended change, not a blanket coverage campaign. + +When possible, characterize behavior that must remain stable before restructuring. A characterization test records current behavior; it does not declare that behavior desirable. + +Sometimes no useful test can run until a dependency is broken. In that case, introduce the smallest behavior-preserving seam needed to gain control or observation. + +That preparatory edit is risky because it lacks a test. Keep it mechanical, review it carefully, and obtain the best available external evidence before proceeding. + +The legacy sequence is: + +1. identify the intended change and the behavior at risk; +2. observe the current system through the best available boundary; +3. introduce a minimal seam first when no test can otherwise run; +4. characterize only behavior that must remain stable; +5. establish an observed Red for the new or corrected behavior; +6. make the smallest coherent Green change; +7. refactor under the improved evidence. + +Questionable existing behavior needs domain review before it becomes a long-term specification. Approval testing can help capture complex output for that review. + +Adoption is incremental. Each touched area should leave behind clearer boundaries and more useful evidence than it had before. + +## Coverage without coverage theatre + +Coverage can reveal code the suite never executes. It cannot establish requirement completeness, oracle strength, or fault detection by itself. + +Use it to ask questions: + +- Why is this changed branch unexercised? +- Is the code unreachable, low risk, or missing an example? +- Did configuration omit part of the suite? +- Does a regulatory objective require structural evidence? + +Avoid using a percentage as an individual performance goal. Once rewards depend on the number, weak assertions and low-value tests become rational responses. + +Regulated or safety-critical work may require structural coverage objectives. Meet them while preserving the distinction between executed structure and demonstrated correctness. + +Mutation testing can reveal assertions that miss selected faults. Inspect meaningful survivors instead of replacing one gamed score with another. + +## Signals worth discussing + +No single metric represents test quality or engineering confidence. + +Useful observations include: + +- time from a change to a trustworthy diagnosis; +- flaky-test rate and time spent investigating noise; +- escaped-defect themes and recurrence; +- change-failure and recovery patterns; +- mutation survivors in critical logic; +- maintenance cost of important suites; +- ease of changing a business rule without unrelated breakage. + +Use these signals to investigate the system, not to rank individual developers. + +## Test-suite health + +A retained suite is part of the production feedback system and requires maintenance. + +Treat flaky tests as defects. Classify shared state, timing, nondeterminism, environment, infrastructure, and oracle failures. + +Temporary quarantine needs an owner, reason, issue, and repair path. Permanent quarantine is deletion with extra noise. + +Broader tests should retain diagnostic artifacts. Without logs, identifiers, traces, or screenshots, realism often produces only expensive uncertainty. + +Delete redundant tests when cheaper evidence answers the same question. Rewrite tests that resist legitimate refactoring. Update examples when intended behavior changes. + +## Introducing TDD to a team + +Adoption needs practice and feedback, not only a policy. + +Start with a bounded pilot where behavior has a useful oracle and the cost of regression is visible. Pair or review early cycles so the team can discuss Red quality, boundaries, and test coupling. + +Retrospectives should examine whether the loop reduced uncertainty and how much maintenance it created. Adjust the practice rather than enforcing ritual. + +A practical team policy is: + +- state the behavior and relevant risk before implementation; +- observe Red for new or corrected behavior with an executable oracle; +- select the shortest trustworthy feedback loop faithful to that risk; +- use broader evidence for infrastructure, compatibility, and deployed behavior; +- introduce doubles and seams only when control or observation justifies them; +- treat coverage and mutation results as prompts for investigation; +- maintain fast, standard, and thorough feedback lanes; +- reconsider tests whose maintenance cost exceeds the evidence they provide. + +## Sources and further reading + +- David Heinemeier Hansson, [TDD is dead. Long live testing.](https://dhh.dk/2014/tdd-is-dead-long-live-testing.html) +- Kent Beck, [Canon TDD](https://newsletter.kentbeck.com/p/canon-tdd) +- Michael Feathers, *Working Effectively with Legacy Code* +- Michael Feathers, [The Seam Model](https://www.informit.com/articles/article.aspx?p=359417&seqNum=2) +- Martin Fowler, [Test Coverage](https://martinfowler.com/bliki/TestCoverage.html) +- Freeman and Pryce, [Growing Object-Oriented Software, Guided by Tests](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/) +- Rafique and Mišić, [The Effects of Test-Driven Development on External Quality and Productivity](https://doi.org/10.1109/TSE.2012.28) +- Munir et al., [Considering rigor and relevance when evaluating test driven development](https://doi.org/10.1016/j.infsof.2014.01.001) diff --git a/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md b/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md new file mode 100644 index 0000000..815983c --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/07-worked-examples.md @@ -0,0 +1,506 @@ +# Worked examples + +These examples show decisions unfolding over time. The code is illustrative Python, but the sequence applies across stacks. + +Each complete example records the baseline, expected behavior, observed Red, coherent Green, refactoring decision, and broader verification. + +Terms are defined in the [glossary](appendices/glossary.md). + +The example repository uses illustrative lane commands: + +```text +fast/local: pytest tests/unit -q +standard/change-validation: pytest tests/unit tests/component tests/integration -q +thorough/system-validation: pytest -q +``` + +Real projects should use their own measured commands and gates. + +## Complete example 1: a greenfield delivery quote + +A new service has a build and test command but no proven runtime or delivery path. + +### Establish the walking skeleton + +The team first verifies the known starting state: + +```text +$ python -m compileall -q src +$ pytest --version +pytest 9.x +``` + +The first component example requests `/health` and expects status 200. It initially receives 404, showing that the request reaches the application but the health capability is absent. + +The smallest implementation adds the route. The component example passes, and the same request succeeds after deployment to a test environment. + +The skeleton now proves build, test, request routing, deployment, and runtime connection. It remains a provisional architecture probe, not evidence for any delivery-quote rule. + +The first stakeholder-visible slice is a delivery quote. Orders at or above EUR 50 receive free standard delivery. The price below that threshold is still an open policy question. + +### Frame the slice + +The team writes a provisional test list: + +- EUR 50 receives free delivery; +- determine the price below EUR 50; +- an amount above EUR 50 receives free delivery; +- invalid negative totals are rejected; +- the HTTP endpoint returns the calculated quote. + +Only the first item becomes executable. The others remain reminders that may change after each cycle. + +The outer HTTP example is recorded as pending while the domain rule is developed through faster inner cycles: + +```python +@pytest.mark.skip(reason="delivery quote is in progress") +def test_quote_endpoint_returns_free_delivery_at_threshold(client): + response = client.post("/quotes", json={"order_total": "50.00"}) + + assert response.status_code == 200 + assert response.json() == {"standard_delivery_fee": "0.00"} +``` + +### Establish the baseline + +The health and test-runner checks pass: + +```text +$ pytest -q +1 passed, 1 skipped +``` + +There is no pre-existing failure that could be confused with the new Red. + +### First Red + +The first focused example states the inclusive threshold: + +```python +def test_standard_delivery_is_free_at_fifty_euros(): + assert standard_delivery_fee(Decimal("50.00")) == Decimal("0.00") +``` + +A missing symbol is only an intermediate Red. After adding the callable surface without the behavior, the test reaches the contract failure: + +```text +FAILED: expected Decimal("0.00"), received None +``` + +The failure is reproducible, attributable to the missing rule, and demonstrates that the assertion can detect its absence. + +### First Green + +The implementation supports the agreed range and refuses to invent the unresolved price: + +```python +def standard_delivery_fee(order_total): + if order_total >= Decimal("50.00"): + return Decimal("0.00") + raise QuotePolicyUndefined() +``` + +The focused test passes. This implementation is intentionally narrow, but it does not violate or silently invent an established requirement. + +### Second Red + +The product owner now decides that smaller orders cost EUR 5. The next example makes that newly agreed behavior executable: + +```python +def test_standard_delivery_costs_five_euros_below_threshold(): + assert standard_delivery_fee(Decimal("49.99")) == Decimal("5.00") +``` + +The test fails with `QuotePolicyUndefined`. The new example now justifies behavior below the threshold. + +### Second Green + +```python +def standard_delivery_fee(order_total): + if order_total >= Decimal("50.00"): + return Decimal("0.00") + return Decimal("5.00") +``` + +Both examples pass. The code handles only behavior supported by the current examples. + +The pending outer example can now include the newly agreed paid-delivery outcome. It is revised while still pending: + +```python +@pytest.mark.skip(reason="delivery quote is in progress") +def test_quote_endpoint_returns_standard_delivery_fees(client): + free = client.post("/quotes", json={"order_total": "50.00"}) + paid = client.post("/quotes", json={"order_total": "12.34"}) + + assert free.status_code == paid.status_code == 200 + assert free.json() == {"standard_delivery_fee": "0.00"} + assert paid.json() == {"standard_delivery_fee": "5.00"} +``` + +### Review the remaining examples + +An example above the threshold passes immediately because the current comparison already supports it. The team confirms that it reaches production code and retains it because the inclusive range is part of the contract. + +This is useful regression evidence, but it is not presented as a Red–Green cycle. + +The team then agrees that negative totals are invalid: + +```python +def test_standard_delivery_rejects_negative_total(): + with pytest.raises(InvalidOrderTotal): + standard_delivery_fee(Decimal("-0.01")) +``` + +The test fails because the current implementation returns EUR 5. That observed failure establishes the next Red. + +The smallest coherent Green adds the validation guard. All threshold and invalid-input examples pass. + +### Refactor while Green + +The literals now carry domain meaning, so they are named without changing behavior: + +```python +FREE_DELIVERY_THRESHOLD = Decimal("50.00") +STANDARD_DELIVERY_FEE = Decimal("5.00") + + +def standard_delivery_fee(order_total): + if order_total < Decimal("0.00"): + raise InvalidOrderTotal() + if order_total >= FREE_DELIVERY_THRESHOLD: + return Decimal("0.00") + return STANDARD_DELIVERY_FEE +``` + +The focused tests remain Green after the structural change. + +### Complete the vertical slice + +The pending marker is removed. The outer example now runs and receives 404 because the quote route does not exist. + +The route is added as a thin adapter over the tested rule: + +```text +$ pytest tests/component/test_quotes_api.py -q +1 passed +``` + +The contrasting requests confirm parsing, routing, serialization, and results consistent with the domain rule. They do not prove whether the adapter delegates or duplicates that rule. + +The outer check does not repeat every threshold and invalid-input partition through HTTP. + +Before handoff, the standard command shown above passes. The thorough `pytest -q` lane also passes before merge because the first slice changed delivery wiring. + +The next slice can now be selected from what the team learned. + +## Complete example 2: correcting an existing defect + +An existing percentage parser accepts negative discounts. Its current positive and zero-value tests pass. + +### Establish the baseline + +```text +$ pytest tests/unit/test_discounts.py -q +6 passed +``` + +### Establish Red + +```python +def test_rejects_negative_percentage(): + with pytest.raises(InvalidDiscountError): + parse_discount("-10%") +``` + +The observed result is: + +```text +FAILED: did not raise InvalidDiscountError +``` + +This is the intended Red. A parsing error, missing fixture, or unrelated failure would not demonstrate the defect. + +### Reach Green + +```python +def parse_discount(text): + value = parse_percentage(text) + if value < 0: + raise InvalidDiscountError("discount must not be negative") + return value +``` + +The new test and the neighboring parser tests pass. + +### Refactor and verify + +Validation already exists in another percentage-based input. Extracting a shared domain validator removes that duplication while all examples remain Green. + +The developer runs `pytest tests/unit/test_discounts.py tests/component/pricing -q` as the affected standard lane. + +No browser test is added because browser behavior was not part of the defect. + +The correction is test-first even though the original parser was not developed through TDD. + +## Complete example 3: changing an existing contract + +An order may currently be reopened at any time after cancellation. The clarified requirement permits reopening through exactly ten minutes, but not later. + +This is an intentional behavior change. Existing tests are specifications to revise, not immutable obstacles. + +### Establish the baseline + +The existing reopening examples pass. The team confirms the new time limit with the product owner and chooses the public `reopen` operation as the observation boundary. + +### Revise retained evidence and establish Red + +The former unconditional example is narrowed to the still-supported behavior: + +```python +def test_cancelled_order_can_be_reopened_at_exactly_ten_minutes(): + order = cancelled_order(cancelled_at=instant("10:00")) + + order.reopen(now=instant("10:10")) + + assert order.status == OrderStatus.OPEN +``` + +This example already passes because it describes a subset of existing behavior. It clarifies the retained contract but does not establish Red. + +The next example expresses the unsupported restriction: + +```python +def test_cancelled_order_cannot_be_reopened_after_ten_minutes(): + order = cancelled_order(cancelled_at=instant("10:00")) + + with pytest.raises(ReopenWindowExpired): + order.reopen(now=instant("10:10:00.001")) + + assert order.status == OrderStatus.CANCELLED +``` + +The test fails because no exception is raised. That observed failure establishes Red for the changed contract. + +### Reach Green + +```python +def reopen(self, now): + if now - self.cancelled_at > timedelta(minutes=10): + raise ReopenWindowExpired() + self.status = OrderStatus.OPEN +``` + +Both the retained and restricted examples pass. + +### Refactor and verify + +The ten-minute duration becomes a named policy value. Cancellation behavior unrelated to reopening remains unchanged. + +Focused order tests and the standard order-workflow suite pass. Published clients are reviewed because compatibility is part of the observable contract. + +This example shows why the runtime instruction must say “add or revise a test,” rather than only “add a test.” + +## Complete example 4: entering difficult legacy code + +A legacy invoice service constructs a tax client internally. Local tests cannot substitute it, and the real client requires credentials available only in the deployed test environment. + +The requested change is to exempt zero-value invoices from the tax request. Existing non-zero invoice behavior must remain stable. + +### Establish the best available baseline + +The focused local test cannot run because construction reaches the credentialed provider. That limitation is recorded instead of being called Red. + +An existing smoke check runs in the credentialed deployed test environment and passes: + +```text +$ pytest tests/system/test_invoice_smoke.py -q +1 passed +``` + +The team will rerun this check immediately after the unprotected dependency break. + +### Identify the risk + +No useful characterization test can run while construction is hard-wired. The first step is therefore a minimal dependency break, not a fictional Red. + +The constructor is changed to accept an optional tax client while preserving the production default: + +```python +class InvoiceService: + def __init__(self, tax_client=None): + self.tax_client = ( + tax_client if tax_client is not None else RealTaxClient() + ) +``` + +This edit is mechanical but not protected by a new test. It receives careful review, then the existing system smoke check passes again. + +### Characterize behavior to preserve + +A deterministic fake now makes current non-zero behavior observable: + +```python +def test_non_zero_invoice_requests_current_tax_amount(): + taxes = RecordingTaxClient(amount=Decimal("2.00")) + service = InvoiceService(tax_client=taxes) + + invoice = service.create_invoice(subtotal=Decimal("10.00")) + + assert invoice.total == Decimal("12.00") + assert taxes.requested_subtotals == [Decimal("10.00")] +``` + +The characterization test passes. It records the behavior at risk; it is not the Red for the requested change. + +### Establish Red for the change + +```python +def test_zero_value_invoice_does_not_request_tax(): + taxes = RecordingTaxClient(amount=Decimal("0.00")) + service = InvoiceService(tax_client=taxes) + + invoice = service.create_invoice(subtotal=Decimal("0.00")) + + assert invoice.subtotal == Decimal("0.00") + assert invoice.tax == Decimal("0.00") + assert invoice.total == Decimal("0.00") + assert taxes.requested_subtotals == [] +``` + +The test fails because the fake records a request for `Decimal("0.00")`. + +### Reach Green + +The coherent early return still creates the required invoice: + +```python +def create_invoice(self, subtotal): + if subtotal == Decimal("0.00"): + return Invoice(subtotal=subtotal, tax=Decimal("0.00")) + + tax = self.tax_client.calculate(subtotal) + return Invoice(subtotal=subtotal, tax=tax) +``` + +The outcome assertions prevent an implementation that merely skips all work. The interaction assertion proves that zero-value invoices avoid the external request. + +### Refactor and verify + +Only then is tax-request construction extracted into a small private operation. Public invoice behavior remains unchanged while the tests stay Green. + +The developer runs `pytest tests/unit/invoicing tests/contract/tax_provider -q`. The fake proves what the service requested; the contract check covers provider compatibility, not real settlement. + +## Complete example 5: when integration evidence is the correct Red + +A SQL Server application requires usernames to be unique under its production collation. An in-memory collection cannot supply that oracle. + +The test uses a disposable SQL Server instance with production-relevant schema and collation. + +### Establish the baseline + +The existing registration integration tests pass against the disposable engine: + +```text +$ pytest tests/integration/test_registration_sqlserver.py -q +4 passed +``` + +### Establish Red + +```python +def test_username_constraint_uses_production_collation(sql_user_repository): + sql_user_repository.insert(username="Alice") + + with pytest.raises(UsernameConstraintViolation): + sql_user_repository.insert(username="alice") + + assert sql_user_repository.count_matching("alice") == 1 +``` + +Both inserts succeed and the count becomes two. The new test is the only failure, and it observes the database constraint and persisted state rather than only an application return value. + +### Reach Green + +A production-relevant unique index is added. A separate application integration example verifies translation of that constraint violation into the duplicate-user result. + +The new example and neighboring SQL Server tests pass. + +### Refactor and verify + +The constraint-to-domain translation is extracted behind one named mapping operation while the integration suite remains Green. + +The developer runs: + +```text +$ pytest tests/unit/registration tests/integration/test_registration_sqlserver.py -q +18 passed +``` + +The migration test also verifies clean installation and upgrade from the previous schema. A focused test may support diagnosis, but it cannot replace the database evidence. + +This slower test is the correct Red because only it faithfully exercises the risk. Runtime speed is not allowed to select an irrelevant boundary. + +## Compact decision examples + +The remaining examples illustrate choices rather than complete development sequences. + +### Requesting an external effect + +A focused test can prove that checkout requested a charge: + +```python +payment_gateway.charge.assert_called_once_with(order.total) +``` + +It does not prove settlement. Provider contracts, sandbox integration, and operational reconciliation answer different questions. + +### Consumer-driven contract evidence + +A consumer records the requests and responses it relies on. Provider verification replays those interactions against the provider build. + +This detects exercised incompatibility without proving authorization, database behavior, unrelated endpoints, or production configuration. + +Use it when independently changing consumers and providers need compatibility feedback before release. + +### E2E evidence for a deployed path + +One checkout E2E test can cover routing, browser behavior, deployed configuration, and service wiring. + +Keep discount, tax, and validation partitions in faster tests. Capture traces, scoped logs, and screenshots so a broad failure remains diagnosable. + +### Controlled nondeterminism + +A retry policy with jitter can accept a seeded random source and expose bounds: + +```python +delay = retry_delay(attempt=3, random_source=seeded_random) +assert minimum <= delay <= maximum +``` + +Distribution quality still needs repeated statistical evaluation. One exact example is not a sufficient oracle. + +### Test-after evidence + +A black-box test written after implementation can still provide useful specification and regression evidence. + +It did not supply design feedback during the original implementation. Test timing and test quality are separate judgments. + +### Testability rather than mockability + +Code that constructs a payment SDK, reads the system clock, and returns nothing is hard to control and observe. + +Move SDK construction to the application’s startup wiring, often called the composition root. Inject the clock and payment capability, and expose a caller-relevant outcome. + +Keep deterministic owned helpers together when that gives clearer behavior tests. The number of interfaces or mocks is not a quality measure. + +## What the examples demonstrate + +The examples differ in scope, setup, and speed, but keep the same evidence discipline: + +- know the starting state; +- choose a faithful boundary and independent oracle; +- observe meaningful Red for new or corrected behavior; +- reach Green without weakening the evidence; +- refactor only while behavior remains Green; +- run broader evidence according to risk; +- report what remains unverified. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md b/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md new file mode 100644 index 0000000..12600a2 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/adjacent-practices-and-further-reading.md @@ -0,0 +1,191 @@ +# Adjacent practices and further reading + +TDD is one feedback workflow, not a complete engineering system. This appendix points to practices that answer nearby questions but are too broad for the core reading path. + +Each entry explains its question, connection to TDD, and trigger for deeper study. The links are starting points, not a required curriculum. + +## Test doubles and testability + +**Question:** How can a test control inputs or observe outcomes when a collaborator is slow, nondeterministic, destructive, or outside the team's control? + +**Relationship to TDD:** Dummies, stubs, spies, mocks, and fakes can make a boundary controllable. They do not prove the real external effect, and testability is broader than mockability. + +**Investigate when:** Time, randomness, networks, costly infrastructure, or separately governed systems obstruct useful feedback. Also investigate when interaction-heavy tests make safe refactoring difficult. + +**Start with:** Gerard Meszaros, [xUnit Test Patterns](https://xunitpatterns.com/); Martin Fowler, [Mocks Aren't Stubs](https://martinfowler.com/articles/mocksArentStubs.html). + +## TDD schools and working styles + +**Question:** Where should design pressure enter, and when should tests use real owned collaborators rather than substitutes? + +**Relationship to TDD:** Classicist TDD commonly uses real owned collaborators. Mockist TDD uses interaction expectations to discover roles. Inside-out and outside-in describe another, related axis. + +**Investigate when:** A team disagrees about isolation, test boundaries, or whether interfaces discovered through mocks clarify the domain. Judge outcomes, not school labels. + +**Start with:** Fowler, [Mocks Aren't Stubs](https://martinfowler.com/articles/mocksArentStubs.html). + +Also see Freeman and Pryce, [Growing Object-Oriented Software](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/). + +## Test portfolio heuristics + +**Question:** How should a project distribute focused, integration, and E2E evidence so feedback remains useful and affordable? + +**Relationship to TDD:** The pyramid, trophy, and honeycomb visualize cost or fidelity preferences. They do not determine the correct Red or replace risk analysis. + +**Investigate when:** The suite is slow, fragile, duplicative, or blind to important integration risks. Use measurements and unique questions rather than target proportions. + +**Start with:** Fowler, [The Practical Test Pyramid](https://martinfowler.com/articles/practical-test-pyramid.html); Kent C. Dodds, [The Testing Trophy](https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications). + +## Risk-based testing + +**Question:** Which behaviors and conditions deserve the most evidence when time and attention are limited? + +**Relationship to TDD:** Risk helps select the next behavior, observation boundary, and verification depth. Neither every function nor every data transformation deserves equal treatment. + +**Investigate when:** Impact, failure probability, change frequency, complexity, poor observability, or regulatory exposure varies materially across the system. + +**Start with:** ISO/IEC/IEEE 29119-2, [Test processes](https://www.iso.org/standard/79428.html); ISTQB, [Advanced Test Management](https://www.istqb.org/certifications/certified-tester-advanced-level-test-management-ctal-tm-v3-0/). + +## Exploratory testing + +**Question:** What important behavior or risk has nobody encoded as an expectation yet? + +**Relationship to TDD:** Automated examples check known claims repeatedly. Exploration combines learning, test design, and execution to discover surprises, usability problems, and new questions. + +**Investigate when:** The domain is unfamiliar, behavior is visual or experiential, incidents reveal unknown risks, or scripted checks pass while confidence remains low. + +**Start with:** Cem Kaner and James Bach, [The Nature of Exploratory Testing](https://kaner.com/pdfs/ETatQAI.pdf); James Bach, [Exploratory Testing Explained](https://www.satisfice.com/download/exploratory-testing-explained). + +## Property-based testing + +**Question:** Does an invariant hold across a generated range of inputs rather than only selected examples? + +**Relationship to TDD:** Properties can follow examples or drive a cycle when a trustworthy invariant is known. Generator and oracle quality still determine the evidence. + +**Investigate when:** Parsers, transformations, algebraic rules, protocols, or state machines have large input spaces and useful invariants. + +**Start with:** Koen Claessen and John Hughes, [QuickCheck](https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf). + +## Metamorphic testing + +**Question:** When an exact expected output is unavailable, should related executions still have a known relationship? + +**Relationship to TDD:** A metamorphic relation can serve as an oracle. It complements concrete examples when calculating one exact answer independently is difficult. + +**Investigate when:** Search, simulation, scientific computing, optimization, and ML systems have an oracle problem but support meaningful transformations or relations. + +**Start with:** Tsong Yueh Chen et al., [Metamorphic Testing: A Review of Challenges and Opportunities](https://doi.org/10.1145/3143561). + +## Model-based testing + +**Question:** Do generated action sequences and state transitions agree with an explicit behavioral model? + +**Relationship to TDD:** A model supplies an oracle across sequences that individual examples may miss. Creating and maintaining it is a separate design investment. + +**Investigate when:** Stateful workflows, protocols, devices, or user interfaces have many valid sequences and transition rules. + +**Start with:** Mark Utting, Alexander Pretschner, and Bruno Legeard, [A Taxonomy of Model-Based Testing Approaches](https://doi.org/10.1002/stvr.456). + +## Mutation testing + +**Question:** Would the retained tests detect selected plausible changes to production code? + +**Relationship to TDD:** Mutation probes the suite's ability to discriminate behavior. A mutation score is evidence to interpret, not a universal quality target. + +**Investigate when:** Coverage is high but assertions may be weak, critical rules deserve an evidence audit, or a team wants to find untested decision outcomes. + +**Start with:** Yue Jia and Mark Harman, [An Analysis and Survey of Mutation Testing](https://doi.org/10.1109/TSE.2010.62); Stryker, [Mutation testing elements](https://stryker-mutator.io/docs/mutation-testing-elements/). + +## Approval and snapshot testing + +**Question:** How can a team review and retain complex output when many small assertions would hide the meaningful whole? + +**Relationship to TDD:** An approved baseline can become an oracle for new or existing behavior. Snapshot storage is a mechanism; deliberate human review gives a change meaning. + +**Investigate when:** Reports, rendered output, serialized structures, compiler output, or legacy behavior are costly to express with focused assertions. + +**Start with:** ApprovalTests, [Approval Testing](https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html); Jest, [Snapshot Testing](https://jestjs.io/docs/snapshot-testing). + +## Static analysis, type systems, and formal methods + +**Question:** Which defects can be rejected or which critical properties can be justified without executing example-based tests? + +**Relationship to TDD:** Types and static checks shorten feedback for classes of invalid programs. Model checking and proof can establish properties that sampled executions cannot. + +**Investigate when:** Invalid states can be excluded by design, concurrency creates large state spaces, or failure impact justifies stronger assurance than examples provide. + +**Start with:** Benjamin Pierce, [Types and Programming Languages resources](https://www.cis.upenn.edu/~bcpierce/tapl/); Leslie Lamport, [The TLA+ Home Page](https://lamport.azurewebsites.net/tla/tla.html). + +## Quality attributes + +**Question:** Does the system remain acceptable under security, privacy, accessibility, performance, resilience, concurrency, compatibility, and resource constraints? + +**Relationship to TDD:** Some attributes can be driven by executable examples. Most also require threat models, workloads, fault injection, audits, specialist tools, or statistical evidence. + +**Investigate when:** The attribute affects users, legal duties, safety, cost, service objectives, or architecture. Define measurable quality scenarios rather than calling the concern “non-functional.” + +**Start with:** [OWASP ASVS](https://owasp.org/www-project-application-security-verification-standard/); [W3C WCAG](https://www.w3.org/TR/WCAG22/); Google, [Site Reliability Engineering](https://sre.google/books/). + +For privacy engineering, also see NIST, [Privacy Framework](https://www.nist.gov/privacy-framework). Select domain standards with qualified specialists where legal or safety obligations apply. + +## Monitoring, observability, and reconciliation + +**Question:** What can be learned only from the system operating with real traffic, dependencies, data, and failure modes? + +**Relationship to TDD:** Operational feedback detects outcomes pre-release checks cannot reproduce. It complements rather than excuses missing development evidence. + +**Investigate when:** Systems are distributed, depend on third parties, degrade gradually, process asynchronous work, or require confirmation of real business outcomes. + +**Start with:** Google SRE, [Monitoring Distributed Systems](https://sre.google/sre-book/monitoring-distributed-systems/); OpenTelemetry, [Observability primer](https://opentelemetry.io/docs/concepts/observability-primer/). + +## ML and LLM evaluation + +**Question:** How well does a probabilistic model behave across representative tasks, populations, adversarial inputs, and changing production conditions? + +**Relationship to TDD:** Test deterministic preprocessing, schemas, tool calls, and control flow normally. Model quality usually needs datasets, metrics, repeated trials, human judgment, and monitoring. + +**Investigate when:** Outputs vary, exact strings are not the real requirement, model or prompt changes shift behavior, or safety and fairness depend on a distribution of outcomes. + +**Start with:** NIST, [AI Risk Management Framework](https://www.nist.gov/itl/ai-risk-management-framework); NIST, [Generative AI Profile](https://doi.org/10.6028/NIST.AI.600-1). + +## Regulatory and safety structural coverage + +**Question:** Which prescribed evidence, traceability, independence, and structural coverage objectives apply to this product and assurance level? + +**Relationship to TDD:** TDD can contribute executable requirements and regression evidence. It does not replace mandated processes, reviews, traceability, or coverage analysis. + +**Investigate when:** Software affects aviation, automotive, medical, rail, industrial, financial, or other regulated and safety-critical domains. Engage assurance specialists early. + +**Start with:** FAA, [Advisory Circular 20-115D](https://www.faa.gov/documentLibrary/media/Advisory_Circular/AC_20-115D.pdf); ISO, [ISO 26262-6 software development](https://www.iso.org/standard/68388.html). + +Do not transfer a coverage target from one standard, assurance level, or jurisdiction to another. Determine the exact governing obligations for the system. + +## Continuous integration and delivery + +**Question:** Does work remain integrated, deployable, and evaluable as contributions and environments change? + +**Relationship to TDD:** CI extends the local feedback loop to the team. Delivery automation tests packaging and deployment paths that focused development checks cannot prove. + +**Investigate when:** More than one change stream exists, integration is delayed, releases are risky, or environment drift weakens confidence. + +**Start with:** Martin Fowler, [Continuous Integration](https://martinfowler.com/articles/continuousIntegration.html); Jez Humble, [Continuous Delivery](https://continuousdelivery.com/). + +## Architecture and walking skeletons + +**Question:** Which system boundaries, runtime responsibilities, and delivery paths must work together before local design choices can succeed? + +**Relationship to TDD:** Tests provide local design feedback but do not automatically grow a sound system architecture. A walking skeleton probes a provisional end-to-end structure early. + +**Investigate when:** Starting a system, crossing trust or deployment boundaries, making irreversible technology choices, or changing system-wide qualities. + +**Start with:** Freeman and Pryce, [Walking Skeleton](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html). + +Also see Michael Nygard, [Documenting Architecture Decisions](https://www.cognitect.com/blog/2011/11/15/documenting-architecture-decisions). + +Use spikes, prototypes, architecture decision records, and quality scenarios when uncertainty requires them. Keep the probe provisional; it is not permission for comprehensive design up front. + +## How to use this appendix + +Return to the core TDD loop when the current question is behavior that can be expressed through a trustworthy executable example. + +Follow an adjacent practice when it answers a different material question. Combine evidence deliberately, and state what each result does and does not establish. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md b/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md new file mode 100644 index 0000000..63aeb03 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/glossary.md @@ -0,0 +1,257 @@ +# Glossary + +This glossary defines terms as used in this documentation. Communities use some terms differently, especially “unit,” “mock,” “acceptance test,” and “contract test.” + +## Acceptance test + +A test of a stakeholder-visible capability against agreed examples or criteria. It may cross several technical components without exercising a production deployment. + +## Approval test + +A test that captures complex output, asks a person to review it, then compares later output with the approved baseline. + +## ATDD + +**Acceptance Test-Driven Development.** A collaborative practice in which acceptance examples are clarified before implementation and used to guide development. + +## BDD + +**Behavior-Driven Development.** Practices emphasizing behavior in domain language, outside-in discovery, concrete examples, and collaboration. Gherkin is optional. + +## Behavior + +An outcome a caller can observe under stated conditions. Behavior concerns what a system does at a chosen boundary rather than how its internals are arranged. + +## Characterization test + +A test that records what existing code currently does so it can be changed more safely. It does not imply that every captured behavior is desirable. + +## Classicist TDD + +A style that runs real owned collaborators together and uses doubles mainly at slow, nondeterministic, or external boundaries. Also called Detroit-style or sociable TDD. + +## Component test + +A test of a deployable or cohesive subsystem through its supported boundary, usually with external systems replaced or controlled. Its scope is broader than one focused rule. + +## Composition root + +The application startup location where concrete dependencies are constructed and connected. Keeping construction there can leave behavior code easier to control and observe. + +## Consumer-driven contract test + +A contract-testing workflow in which consumers publish the interactions they rely on and providers verify those interactions against their implementation. + +## Continuous integration + +The practice of integrating changes frequently and verifying the combined state automatically. It extends local feedback into feedback about the shared codebase. + +## Contract + +An agreement about observable inputs, outputs, errors, or interactions at a boundary. A contract may be published between systems or internal to one codebase. + +## Contract test + +A test of compatibility at a boundary between independently changing parties. Consumer-driven contract testing is one form; schema and protocol conformance are others. + +## Coverage + +A structural measure reporting which statements, branches, conditions, or paths executed. It does not show by itself that results were asserted or requirements were complete. + +## Double-loop TDD + +A development pattern in which an outer acceptance example gives a capability its destination while faster inner Red–Green–Refactor cycles develop the required parts. + +## Dummy + +A test double passed to satisfy an interface but not used by the behavior under test. + +## E2E test + +An **end-to-end test** that observes a deployed or fully assembled path. It can reveal wiring and configuration failures but is usually slower and harder to diagnose. + +## Executable example + +A concrete input, action, and expected outcome represented so a tool can evaluate it repeatedly. A TDD test begins as an executable example of desired behavior. + +## Fake + +A working implementation used in testing that takes shortcuts unsuitable for production, such as an in-memory repository with simplified semantics. + +## Fake It + +A TDD strategy that uses a narrow result when the general implementation is unclear. Later examples can require generalization; refactoring may only change structure. + +## Fast feedback lane + +The smallest project-defined checks used during Red, Green, and refactoring. Membership depends on measured feedback value, not on a test-level label. + +## Feedback lane + +A project-defined group of checks selected by cost, reliability, environment, and risk. This documentation uses fast/local, standard/change-validation, and thorough/system-validation lanes. + +## Feedback time + +The time from making a change to receiving a trustworthy diagnosis that can guide the next decision. It includes execution, setup, queuing, and investigation. + +## Flaky test + +A test that can pass and fail against the same relevant code because of uncontrolled state, timing, environment, concurrency, infrastructure, or nondeterminism. + +## Focused test + +A test at a narrow coherent behavior boundary, usually selected for diagnostic feedback. It may involve several functions or classes and is not guaranteed to be fast. + +## Gherkin + +A structured scenario language commonly using `Feature`, `Scenario`, `Given`, `When`, and `Then`. Using Gherkin does not by itself constitute BDD. + +## Given–When–Then + +A way to structure an example as relevant starting conditions, one triggering event, and observable outcomes. + +## Golden master + +A reviewed output retained as the comparison baseline for an approval or characterization test. + +## Green + +The phase in which production code is changed until the current example and relevant existing tests pass for the intended reason. + +## Green integrity + +The rule that Green is reached through a valid production change, not by weakening the oracle, skipping the test, swallowing failures, or silently changing the contract. + +## Integration test + +A test that exercises real collaboration between components or infrastructure, such as application code with the production database engine. + +## Mock + +A test double with interaction expectations. Terminology varies by framework; this documentation distinguishes mocks from stubs and spies by built-in expectations. + +## Mockist TDD + +A style that uses interaction expectations to discover collaborator roles from the outside in. Also called London-style or solitary TDD. + +## Mutation testing + +A technique that changes production code in selected ways and runs the suite to see whether tests detect those changes. + +## Observation boundary + +The interface through which a test supplies inputs and observes outcomes. It may be a function, component, API, message contract, command, or user interface. + +## Obvious Implementation + +A TDD strategy that writes the straightforward general implementation directly when the correct change is already clear. + +## Oracle + +The source of truth used to decide whether an outcome is acceptable. Examples include agreed literals, domain examples, invariants, trusted references, and reviewed baselines. + +## Property-based testing + +A technique that generates inputs to check general invariants and often shrinks a failure to a simpler counterexample. + +## Red + +The phase in which an executable example is observed failing because the intended behavior is absent or incorrect. + +This documentation treats compile and missing-symbol failures as intermediate Red states. When practical, continue until the behavioral expectation itself fails. + +## Red–Green–Refactor + +The recurring cycle: demonstrate one missing behavior, make it work, then improve structure without changing the demonstrated behavior. + +## Refactor + +Change the internal structure of code without intentionally changing observable behavior. An intended contract change is not refactoring. + +## Regression + +A previously working and still-required behavior that stops working after a change. + +## Seam + +A place where behavior or a dependency can be varied without editing the code at that point. Seams can make a legacy observation boundary controllable. + +## Smoke test + +A small test of basic operability after deployment or in a production-like environment. It is intentionally shallow and fast. + +## Snapshot test + +A baseline comparison that stores serialized or rendered output. It provides useful evidence only when changes are reviewed deliberately. + +## Specification by Example + +A collaborative practice that discovers and documents requirements through concrete examples, often retaining them as living specifications. + +## Standard feedback lane + +The project-defined checks used to validate the affected area before handoff or integration. It may contain focused, component, integration, and contract evidence. + +## Spy + +A test double that supplies behavior and records interactions for later assertions. + +## Stub + +A test double that returns controlled values or outcomes needed by the code under test. + +## Succession problem + +The problem of choosing and ordering examples so each provides useful information for the next implementation or design decision. + +## TDD + +**Test-Driven Development.** A workflow that develops one observable behavior at a time through Red–Green–Refactor cycles. + +## Test-after + +Writing a test after the relevant production behavior exists. The test may verify and document behavior but did not influence the preceding implementation decision. + +## Test double + +Any substitute used in place of a real collaborator during a test. Dummies, stubs, spies, mocks, and fakes serve different purposes. + +## Test-first + +Expressing intended behavior as an executable example before implementing it. Test-first enables early feedback but does not guarantee a useful test. + +## Test fidelity + +How faithfully a check exercises the semantics relevant to the current risk. Scope and fidelity are related but separate dimensions. + +## Test pyramid + +A portfolio heuristic favoring many narrow tests, fewer integration tests, and very few E2E tests because feedback cost often increases with scope. + +## Test trophy + +A portfolio heuristic emphasizing integration tests for applications whose main risks lie in component and framework collaboration. + +## Three Amigos + +A collaborative example-discovery practice involving product or domain, development, and testing perspectives. + +## Thorough feedback lane + +The broadest relevant project-defined checks, often including costly or environment-dependent evidence before merge, release, or deployment. + +## Triangulation + +A TDD strategy that adds a contrasting example when one example does not justify a useful general rule. The contrast creates pressure to generalize. + +## Unit test + +A test of a coherent unit of behavior at a narrow boundary. It is often fast, but “unit” does not determine cadence or mean exactly one method or class. + +## Verschlimmbesserung + +A German term for a change intended as an improvement that makes the result worse. Retained regression tests can expose this outcome for covered behavior. + +## Walking skeleton + +The thinnest end-to-end implementation proving that a new system’s build, test, delivery, and runtime parts connect. It is a provisional architecture probe. diff --git a/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md b/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md new file mode 100644 index 0000000..6b5a61d --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/guide/appendices/stack-specific-notes.md @@ -0,0 +1,54 @@ +# Stack-specific notes + +These examples cover stacks used in the surrounding project. They are not an exhaustive framework catalog. + +General TDD principles remain stable while framework details change. Follow repository conventions first, then use these notes when a runtime affects test behavior or design. + +Read the core chapters before treating stack mechanics as instructions. A framework does not decide which behavior matters or which evidence answers its risk. + +## PowerShell and Pester + +- PowerShell’s success stream can turn an expected scalar into an array. Assert exact type and value when output shape is part of the contract. +- Set strict error behavior in test setup so unexpected non-terminating errors cannot masquerade as useful output. +- Pester mocks replace commands within scope and retain call history. Prefer an owned boundary when cmdlet mocking would couple behavior tests to command decomposition. +- Keep modules with business logic distinct from one-off diagnostic scripts. + +See the official [Pester Mock command](https://pester.dev/docs/commands/Mock) reference. + +## Python and pytest + +- Match the existing framework. Pytest is a common default for new Python work, not a universal mandate. +- Use plain assertions and independently derived expected values. +- Default mutable fixtures to function scope. Broader fixture scope requires explicit reset or immutability. +- Patch where a name is looked up. Prefer an owned boundary when patching a third-party client would expose incidental protocol details. +- Use properties for parsers, serializers, round trips, and transformations with large input spaces. +- Test async code with the project’s async plugin and async-aware doubles. + +See the official [pytest fixture](https://docs.pytest.org/en/stable/how-to/fixtures.html) and [monkeypatch](https://docs.pytest.org/en/stable/how-to/monkeypatch.html) guides. + +## SQL and database-backed code + +- Use the production engine when dialect, query translation, transactions, constraints, migrations, locking, or concurrency participate in the risk. +- A fake database is appropriate only when its semantic differences cannot affect the behavior under test. +- Isolate each test’s data through rollback, schema or database isolation, or deterministic cleanup. +- Treat migrations as executable compatibility paths. Test clean installation and representative upgrades. +- Separate correctness tests from performance experiments. Performance needs representative data and a controlled environment. + +Microsoft’s [EF Core testing strategy](https://learn.microsoft.com/en-us/ef/core/testing/choosing-a-testing-strategy) describes SQLite, in-memory-provider, and query differences. + +Testcontainers provides [database testing guides](https://testcontainers.com/guides/getting-started-with-testcontainers-for-dotnet/) for disposable real engines. + +## C# and .NET + +- Follow the repository’s xUnit, NUnit, or MSTest convention and its assertion library. +- Await asynchronous operations. Blocking with `.Result` or `.Wait()` changes failure and deadlock behavior. +- Use `WebApplicationFactory` when the ASP.NET Core request pipeline is the relevant observation boundary. +- Treat EF Core’s InMemory provider and SQLite as fakes with different semantics, not proof of production-database behavior. +- Inject `TimeProvider` or another clock at time-sensitive boundaries. +- Set culture explicitly where formatting or parsing behavior depends on it. + +See Microsoft’s [ASP.NET Core integration-test guidance](https://learn.microsoft.com/en-us/aspnet/core/test/integration-tests) and [EF Core testing overview](https://learn.microsoft.com/en-us/ef/core/testing/). + +## Why these notes remain an appendix + +Stack guidance is human reference, not a hard-coded skill policy. The skill inspects the repository because framework versions and project conventions change faster than the core cycle. diff --git a/setup/optional-skills/knowledge/tdd/research-findings.md b/setup/optional-skills/knowledge/tdd/research-findings.md new file mode 100644 index 0000000..a91089c --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/research-findings.md @@ -0,0 +1,164 @@ +# Technical Review of the TDD Foundation + +Date: 2026-07-13 +Reviewed file: `tools/tdd/notNededAnymore/testing-methodologies-foundation.adoc` + +## Executive Assessment + +The document presents a strong, largely internally consistent **classicist TDD position**. Its core claims about short feedback loops, refactoring, the limited meaning of coverage, BDD collaboration, and complementary testing techniques are fundamentally sound. However, it is not yet a neutral “foundation”: definitions, rules of experience, and author preferences are repeatedly presented as universally applicable rules. Some claims are factually incorrect or too absolute; the most significant problems concern BDD/ATDD, the meaning of “Red,” test levels, mocks, and permanently red acceptance tests. + +## High-Priority Corrections + +### 1. “Red” is not universally limited to assertion failures + +The sections *What Counts as Red?* and the AI rule “Red means assertion failure, not compilation failure” present a disputed preference as a definition. Robert C. Martin’s second “Law of TDD” explicitly treats a test that fails **or fails to compile** as a sufficient Red state. Martin also distinguishes the second-by-second Three Laws nano-cycle from the minute-by-minute Red–Green–Refactor cycle. The document may choose assertion-only Red as its own stricter operating rule, but it must label it accordingly and must not claim that compile-time Red is not TDD. + +Source: [Robert C. Martin, *The Cycles of TDD*](https://blog.cleancoder.com/uncle-bob/2014/12/17/TheCyclesOfTDD.html) + +### 2. BDD is now more than “TDD with different vocabulary” + +Historically, BDD emerged from Dan North’s difficulties teaching TDD; the original questions included where to start, what to test, how much to test at once, and how to name tests. North’s origin account does not support the claim that BDD was created specifically because TDD was equated with unit testing or because of coverage chasing. Current BDD practice includes Discovery, Formulation, and Automation, together with collaboratively developed shared domain understanding. Therefore, “BDD is therefore not a separate practice from TDD” and “a scenario written by one person ... is structurally indistinguishable from a well-named TDD test” are too reductive. + +Sources: [Dan North, *Introducing BDD*](https://dannorth.net/blog/introducing-bdd/), [Dan North, *BDD is like TDD if...*](https://dannorth.net/blog/bdd-is-like-tdd-if/), [Cucumber, *Behaviour-Driven Development*](https://cucumber.io/docs/bdd/), [Agile Alliance, *BDD*](https://agilealliance.org/glossary/bdd/) + +### 3. BDD and ATDD overlap but are not simply synonymous + +“ATDD is, in practice, the same activity as BDD” erases a useful distinction. ATDD refers to acceptance tests created collaboratively **before implementation** from customer, development, and testing perspectives. BDD synthesizes and extends practices from TDD and ATDD through outside-in thinking, business value, shared language, and application across several abstraction levels. Some communities use the terms interchangeably; that supports describing “strong overlap,” not identity. + +Sources: [Agile Alliance, *ATDD*](https://agilealliance.org/glossary/atdd/), [Agile Alliance, *BDD*](https://agilealliance.org/glossary/bdd/) + +### 4. The document contradicts itself about TDD and test levels + +Early on, it correctly states that TDD is a workflow that can drive unit, integration, acceptance, and E2E tests. Later, it states that “TDD's primary domain is unit-level testing,” says TDD/BDD do not answer whether the integrated system works, and equates TDD with the unit level and BDD with the acceptance level in the comparison table. These are different models. The common definition usually describes TDD as tightly interwoven programming, **unit testing**, and refactoring; outside-in and acceptance-test-driven variants also exist. Recommendation: declare the chosen conceptual framework instead of equating a method with a level. + +Sources: [Martin Fowler, *Test-Driven Development*](https://martinfowler.com/bliki/TestDrivenDevelopment.html), [Agile Alliance, *TDD*](https://agilealliance.org/glossary/tdd/), [Cucumber, *BDD – Automation*](https://cucumber.io/docs/bdd/) + +### 5. Permanently red acceptance tests do not belong in a normally evaluated CI suite + +The recommendation to add unfinished acceptance tests “red from day one” to a slow suite in which red tests are expected destroys that suite’s signal. An intentionally red test may serve as a local target or an explicitly quarantined/pending test; when run, a CI suite should be reliably green or technically distinguish a known pending state. Feature branches, tags, or disabled pending scenarios are more appropriate. Likewise, “days or weeks” as a normal Red–Green cycle is not a general TDD rule and weakens the rapid feedback emphasized by primary descriptions. + +Sources: [Martin Fowler, *Test-Driven Development*](https://martinfowler.com/bliki/TestDrivenDevelopment.html), [Cucumber, *Automation*](https://cucumber.io/docs/bdd/) + +### 6. Mock tests do not prove a real external effect + +“If the interaction is the thing the user cares about (a payment was actually charged, an email was actually sent), use a mock” is inaccurately worded. A mock can prove that the code sent the expected request to a controlled abstraction; it specifically does **not** prove that the payment provider made the charge or the email was delivered. That requires contract, integration, sandbox, or E2E verification. In addition, a mock in Meszaros’ taxonomy is not simply “a spy with pre-programmed expectations,” but a double used for behavior verification; spies and mocks are related but distinct patterns. + +Sources: [Martin Fowler, *Mocks Aren't Stubs*](https://martinfowler.com/articles/mocksArentStubs.html), [Gerard Meszaros, *xUnit Test Patterns*](https://xunitpatterns.com/), [Pact, *What is contract testing?*](https://docs.pact.io/) + +### 7. Interface changes can be refactorings + +The rule “Any change to that [external] interface would belong in a new red step” is too absolute. An interface change is a refactoring when all controlled callers are changed with it and the system’s observable behavior is preserved. This is considerably riskier for published APIs, but it is not excluded by definition. + +Source: [Martin Fowler, *Is Changing Interfaces Refactoring?*](https://martinfowler.com/bliki/IsChangingInterfacesRefactoring.html) + +### 8. Coverage is neither the goal nor merely a worthless by-product + +The criticism of percentage chasing is correct: high line coverage guarantees neither strong oracles nor relevant cases. However, “do not propose coverage percentage targets” and the portrayal of coverage as merely a by-product are too absolute. Coverage is a useful **gap and risk signal**; low coverage reveals unexecuted areas, and meaningful thresholds depend on criticality, change frequency, expected lifetime, and domain. The claim that TDD produces high coverage “by definition” applies only to the path currently driven by a test; refactoring can introduce new branches, and a project may use TDD only partially. In safety-critical or regulated contexts, structural coverage objectives may also be normatively required. + +Source: [Google Testing Blog, *Code Coverage Best Practices*](https://testing.googleblog.com/2020/08/code-coverage-best-practices.html) + +There is also a minor error in the example: a test without an explicit assertion does fail if `calculate_total` raises an exception. It implicitly verifies “does not throw,” but not the return value. + +The AI rule that generally rejects tests without assertions or tests that only verify the absence of exceptions is also too absolute: “this operation does not throw for valid inputs” can itself be the required observable contract. The correct rule is that every test needs an effective oracle for **the behavior it claims to verify**. + +## Clarifications for Complementary Methods + +- **Mutation Testing:** The basic description is correct. “A much more honest quality measure” and direct comparisons between arbitrary mutation scores are too strong: score and cost depend on mutation operators, equivalent mutants, timeout/error classification, and the code under analysis. Even Stryker cannot reliably eliminate equivalent mutants automatically; 100% is not a meaningful universal target. Sources: [Stryker, *Equivalent mutants*](https://stryker-mutator.io/docs/mutation-testing-elements/equivalent-mutants/), [Stryker, *Mutant states and metrics*](https://stryker-mutator.io/docs/mutation-testing-elements/mutant-states-and-metrics/). + +- **Property-based Testing:** The definition and typical use cases are correct. “Do not suggest for ... UI” and the exclusion of glue code are acceptable heuristics but not prohibitions: stateful models can verify event sequences, protocols, and UI states. PBT does not replace examples; generator and oracle quality remain decisive. Source: [Claessen/Hughes, *QuickCheck: A Lightweight Tool for Random Testing of Haskell Programs*](https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf). + +- **Contract Testing:** The Pact description is correct for consumer-driven contract testing, but that is not the only form of contract testing. “At every inter-service boundary” is too absolute; the technique is especially valuable for independently deployed consumer/provider teams and systems with many integrations. Contract tests verify message compatibility, not the provider’s business correctness. Source: [Pact, *Introduction*](https://docs.pact.io/). + +- **Approval, Golden Master, Characterization, Snapshot:** These techniques overlap but are not stable synonyms. “Approval” describes a human-approved oracle/baseline workflow; “characterization” describes the purpose of preserving current legacy behavior; “snapshot” usually describes the stored comparison form. An approval test can verify new behavior, and a characterization test can use ordinary assertions. Sources: [ApprovalTests, *ApprovalTesting concept*](https://approvaltestscpp.readthedocs.io/en/latest/generated_docs/ApprovalTestingConcept.html), [Jest, *Snapshot Testing*](https://jestjs.io/docs/snapshot-testing). + +- **Testing Pyramid/Trophy:** Cohn popularized the pyramid; it is a cost heuristic, not a universal test-level standard. The claim that “each level is about an order of magnitude slower and more fragile” is unsupported and should be removed or clearly labeled a local rule of thumb. Dodds’ Trophy argues for ROI and tests that resemble real use, especially in the JavaScript/UI context; “integration tests should numerically outnumber unit tests” is one possible interpretation, not a general definition. Sources: [Martin Fowler, *Test Pyramid*](https://martinfowler.com/bliki/TestPyramid.html), [Kent C. Dodds, *The Testing Trophy and Testing Classifications*](https://kentcdodds.com/blog/the-testing-trophy-and-testing-classifications). + +- **Given–When–Then:** The context/event/outcome description is correct. “Exactly one `When`; split when it contains `and`” is a useful readability rule, but neither Gherkin syntax nor a universal BDD definition. Cucumber permits multiple steps and keyword sequences. Source: [Cucumber, *Gherkin Reference*](https://cucumber.io/docs/gherkin/reference/). + +- **Test Levels:** “Unit / Integration / E2E / Smoke” is useful but incomplete, and definitions vary between organizations. ISO/IEC/IEEE 29119 distinguishes test processes, test levels, and test types and includes system, acceptance, performance, usability, and reliability testing. Source: [ISO/IEC/IEEE 29119 series overview](https://committee.iso.org/sites/jtc1sc7/home/projects/flagship-standards/isoiecieee-29119-series.html). + +## Empirical Evidence + +The cautious core claim “do not sell it as empirically proven” is correct. It should not, however, slide into “the evidence supports no benefit”: + +- Four industrial case studies at Microsoft/IBM reported 40–90% lower pre-release defect density with a subjectively assessed 15–35% increase in initial development time. This is positive but observational and confounded evidence, not a general causal proof. Source: [Nagappan et al., 2008, IBM Research](https://research.ibm.com/publications/realizing-quality-improvement-through-test-driven-development-results-and-experiences-of-four-industrial-teams). +- A process dissection found that small, steady steps and test granularity may explain outcomes better than test-first ordering alone. Source: [Fucci et al., *A Dissection of the TDD Process*](https://arxiv.org/abs/1611.05994). +- A larger family of experiments with TDD novices found slightly higher quality under iterative test-last than under TDD and points to training and study-duration effects. Source: [Santos et al., *A Family of Experiments on TDD*](https://arxiv.org/abs/2011.11942). + +A defensible formulation is: **The empirical results are mixed and strongly dependent on context, skill, and operationalization. There are positive and negative findings; neither side justifies universal productivity or quality promises.** The “mechanistic” benefits are also plausible mechanisms, not guaranteed outcomes: a test written first can cement a poor interface, and a test safety net makes refactoring less risky, not automatically safe. + +## Missing or Underrepresented Perspectives + +A foundation should at least mention and clearly distinguish: + +1. **Inside-out vs. outside-in TDD** and **classicist/sociable vs. mockist/solitary TDD**. The document mentions London/classicist styles, but partly characterizes the London style inconsistently (“never mock what you own; mock only external dependencies” is not the same as “mock all interesting collaborators”). Source: [Fowler, *Mocks Aren't Stubs*](https://martinfowler.com/articles/mocksArentStubs.html). +2. **Specification by Example and Example Mapping** as collaborative requirements practices; BDD is not merely a test-writing style. +3. **Exploratory Testing and human/oracle questions**. Automated checks answer only explicitly encoded expectations; learning, unexpected risks, usability, and visual quality require human exploration. +4. **Risk-based Testing:** Neither “every function” nor “every transformation” is the right unit. Priority follows impact, probability, change frequency, complexity, and observability. Data transformations are often good candidates, but they are not a definitional TDD boundary. +5. **Non-functional quality attributes:** Security, performance, reliability/resilience, accessibility, compatibility, data quality, and observability require their own test designs and do not fit neatly into Unit/Integration/E2E. +6. **Static and formal techniques:** Type checking, linting, reviews, static analysis, model checking, and formal verification complement dynamic tests; the “foundation” focuses almost entirely on executable examples. +7. **Regulated/safety-critical development:** Coverage can be a required structural criterion in such contexts; “never use coverage targets” is therefore unsuitable as a universal AI rule. +8. **Test maintenance and economics:** Behavior-focused tests can also break under legitimate requirement changes and create maintenance costs. “Tests should not break when implementation changes” applies only when the observed interface and semantics remain unchanged. +9. **Fuzzing, metamorphic, and statistical testing:** TDD is not limited to fully deterministic logic. Randomness and concurrency can be tested through controlled seeds/seams and properties; ML/LLM behavior requires datasets, distributions, thresholds, and repeated evaluations rather than isolated deterministic assertions. +10. **Flaky-test management:** Quarantine, reproducibility, ownership, and repair budgets are missing even though unstable tests substantially damage the feedback signal, especially at integration/E2E levels. + +## Claims That Can Be Retained + +- The commonly accepted core of TDD is a small test/code/refactoring cycle; refactoring is essential. [Fowler](https://martinfowler.com/bliki/TestDrivenDevelopment.html) +- Test-first directs attention to the interface early; self-testing code and interface feedback are both important benefit mechanisms. [Fowler](https://martinfowler.com/bliki/TestDrivenDevelopment.html) +- High line coverage guarantees neither relevant paths nor effective assertions; coverage should be used contextually as a signal. [Google Testing Blog](https://testing.googleblog.com/2020/08/code-coverage-best-practices.html) +- BDD is not synonymous with Cucumber/Gherkin; collaboration and concrete examples are central. [Cucumber](https://cucumber.io/docs/bdd/) +- The pyramid and trophy are context-dependent heuristics, not competing laws of nature. [Fowler](https://martinfowler.com/articles/2021-test-shapes.html) +- Mockist and classicist TDD have real, documented trade-offs; a deliberate choice is better than unmarked mixing. [Fowler](https://martinfowler.com/articles/mocksArentStubs.html) + +## Implications for the AI-Condensed Version + +The AI rules should use four labels: + +- **Definition:** A broadly accepted core that is non-negotiable within the term. +- **Default:** A preferred working method that may be overridden by stronger contextual signals. +- **Heuristic:** A diagnostic prompt that helps choose an action but is not a compliance rule. +- **Project Decision:** For example, assertion-only Red, classicist default, coverage gates, real database vs. fake, or testing pyramid vs. trophy. + +Absolute terms such as “never,” “always,” “at every boundary,” and “does not apply” should remain only when they protect technical correctness or safety. Most current absolutes are valuable defaults, not universal facts. + +## Propagation Status + +The second iteration applies the research through a claim-to-deliverable matrix in `traceability.md`. The important changes are no longer confined to this report: + +- disputed claims about Red are labeled as conventions rather than definitions; +- BDD, ATDD, Specification by Example, and double-loop TDD remain overlapping but distinct; +- acceptance examples cannot silently keep an evaluated CI suite red; +- refactoring is governed by observable behavior, not an absolute ban on interface changes; +- mocks verify requested interactions, while real effects require broader evidence; +- coverage is retained as a diagnostic and possible regulatory objective, not a quality verdict; +- classicist and London/mockist schools are defined consistently; +- companion techniques and non-functional qualities are selected from their assurance question; +- AI rules are labeled Definition, Default, Heuristic, or Project decision. + +The human chapters cite primary or authoritative sources near historical and empirical claims. The self-contained skill stores its operating reference under `develop-with-tdd/references/`. + +## Public Skill Survey + +The separate `external-skill-survey.md` compares public TDD, testing-strategy, contract-testing, and E2E skills through their source repositories and skills.sh entries. + +No surveyed skill supplied a stronger general TDD foundation. The useful additions were practical: contrastive examples, mutation-sensitive data, factories, provider-state contract lifecycle, E2E diagnostics, flake ownership, and gate placement. + +The local skill adopts those ideas through conditional references. It rejects fixed pyramid ratios, universal mocking rules, mandatory coverage or mutation targets, one-assertion rigidity, and framework-specific recipes in the core workflow. + +## Targeted Follow-up: Feedback, Beginners, Criticism, Greenfield, and Legacy Work + +Date: 2026-07-14 +Scope: the current `human/01`–`07` chapters and all earlier findings in this file. + +The foundation is already strong on all five topics. It needs two concrete additions and three small emphases, not another broad chapter. + +| Topic | Finding and disposition | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Fast feedback** | **Emphasize the mechanism.** Fast feedback is produced by one small item in flight, a prompt relevant test run, a trustworthy result, and a design signal while the decision is still cheap—not by “unit” scope or mocking alone. Beck describes seconds-scale logic and design feedback; a study of 39 professionals found fine-grained, uniform steps positively associated with quality and productivity, while test-first sequencing had no important influence. Keep “seconds” as an aspiration, not a fixed SLA, and retain the current rule that the narrowest *relevant* boundary wins. Sources: [Beck, *TDD is Kanban for Code*](https://newsletter.kentbeck.com/p/tdd-is-kanban-for-code); [Fucci et al., *A Dissection of the TDD Process*](https://doi.org/10.1109/TSE.2016.2616877). | +| **Beginner practice** | **Add a short guided practice box.** Make the test list explicit: list behavioral variants, turn exactly one item into a concrete runnable test, complete its cycle, then revise and choose again. This fills a real gap between the current compact loop and worked examples without concretizing a whole speculative suite. A controlled experiment with 48 graduate novices found better functional quality with finer-grained task descriptions; a much smaller expert/novice comparison found experts used shorter, less variable cycles. These studies justify scaffolding and coaching, not a productivity promise. Sources: [Beck, *Canon TDD*](https://newsletter.kentbeck.com/p/canon-tdd); [Karac, Turhan, and Juristo, novice task-granularity experiment](https://doi.org/10.1109/TSE.2019.2920377); [Müller and Höfer, expert/novice process study](https://ps.ipd.kit.edu/downloads/za_2007_effect_experience_test_driven_development.pdf). | +| **Skeptical criticism** | **Cross-link and sharpen; do not add another debate chapter.** State once that TDD is neither a professionalism test nor inseparable from mock-heavy unit isolation. DHH’s primary objection is to test-first fundamentalism and isolation-driven indirection, while still endorsing automated regression testing; Beck likewise says there is no “gold star” for canonical conformance. Chapters 3 and 4 already answer the substantive issues by separating workflow, scope, and classicist/mockist choices. Sources: [DHH, *TDD is dead. Long live testing.*](https://dhh.dk/2014/tdd-is-dead-long-live-testing.html); [Beck, *Canon TDD*](https://newsletter.kentbeck.com/p/canon-tdd). | +| **Greenfield work** | **Keep the walking skeleton; add one limiting sentence.** It is a provisional end-to-end architecture probe, used to validate a broad-brush structure before focused inner cycles, not permission for comprehensive up-front architecture. The current health-path example and first stakeholder slice are otherwise sufficient. Source: [Freeman and Pryce, *Growing Object-Oriented Software, Guided by Tests*, Chapter 10](https://www.oreilly.com/library/view/growing-object-oriented-software/9780321574442/ch10.html). | +| **Legacy adjustment** | **Correct the apparent ordering constraint.** Characterization and seams are already covered well, but the current numbered sequence can imply that a characterization test must always precede a seam. Sometimes a dependency prevents any test from running. Permit the smallest behavior-preserving dependency break needed to create a test point, then characterize only the behavior around the intended change, add the failing new-behavior test, and resume normal cycles. Feathers explicitly describes dependency-breaking refactorings performed without tests *in the service of putting tests in place*, and his seam chapter explains using just enough substitution to get code under test. Sources: [Feathers, *Working Effectively with Legacy Code* sample](https://www.informit.com/content/images/9780131177055/samplepages/0131177052.pdf); [Feathers, *The Seam Model*](https://www.informit.com/articles/article.aspx?p=359417&seqNum=2). | + +**Scope boundary:** Do not add rigid cycle-time limits, test-count or coverage targets, a universal unit-first/mocking rule, or causal promises from short cycles. Also exclude a kata catalog, general learning theory, IDE/framework tutorials, a full CI/CD or architecture course, and Feathers’ complete dependency-breaking catalog. Broader integration, contract, E2E, operational, and non-functional testing already has enough context in Chapter 3; expanding it further here would dilute a curriculum whose distinctive subject is the TDD feedback workflow. diff --git a/setup/optional-skills/knowledge/tdd/traceability.md b/setup/optional-skills/knowledge/tdd/traceability.md new file mode 100644 index 0000000..253d720 --- /dev/null +++ b/setup/optional-skills/knowledge/tdd/traceability.md @@ -0,0 +1,49 @@ +# Claim-to-deliverable traceability + +This matrix connects material claims from the original draft and later reviews to their evidence, human explanation, and runtime instruction. + +“Research verdict” summarizes the relevant finding in `research-findings.md`. Human and skill destinations must remain aligned when a claim changes. + +| Claim or decision | Research verdict | Human destination | AI/skill destination | +| ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------- | ------------------------------ | --------------------------------------- | +| TDD is primarily a design-feedback discipline | Defensible thesis, not a universal definition | `human/01-why-tdd.md` | `SKILL.md`; operating reference | +| Prefer the shortest trustworthy loop faithful to the current risk | Retain; speed includes relevance, reliability, diagnosis, and decision timing | Human 01, 02, and 04 | Cycle invariants; feedback lanes | +| Establish a known baseline | Retain as an attribution requirement | Human 01 and 02 | Cycle invariants; step 1 | +| New or corrected behavior requires an observed Red | Retain as the local operating default; confidence is not evidence | Human 01–03 and 07 | Cycle invariants; step 3 | +| Compile or harness failures are not automatically behavioral Red | Continue to the intended contract failure when practical | Human 01–03 | Step 3; test-design examples | +| Green must preserve evidence sensitivity and the agreed contract | Retain as Green integrity | Human 02–03 and 07 | Cycle invariants; step 4 | +| Refactoring preserves behavior and runs under Green | Retain; intentional behavior change starts another cycle | Human 01–03 and 07 | Cycle invariants; step 5 | +| Coverage is not the goal | Retain with diagnostic and regulatory nuance | Human 01 and 06 | Operating reference | +| BDD arose partly from TDD teaching difficulties | Retain; reject a single-cause coverage story | Human 05 | Operating reference | +| BDD is merely renamed TDD | Reject as reductive | Human 05 | Operating reference | +| BDD, ATDD, and Specification by Example are synonyms | Reject; preserve overlap and different emphases | Human 05 | Operating reference | +| Test-first and test-after provide identical design feedback | Reject; post-hoc tests can still provide durable regression evidence | Human 01 and 03 | Test-design examples | +| Every transforming function needs a test | Retain only as a candidate heuristic | Human 02, 03, and 06 | Operating reference | +| Fake It may be generalized during refactoring | Reject when new inputs gain behavior; later examples justify generalization | Human 02, 03, and 07 | Step 4; operating reference | +| Published and internal interfaces have identical refactoring constraints | Reject; use the governing observation boundary and compatibility contract | Human 02 and 03 | Step 5; operating reference | +| An unfinished outer example may remain Red in required CI | Reject; keep it pending, branch-local, or outside required gates | Human 04–06 | Entry strategies | +| A test without an explicit assertion has no value | Replace with an effective-oracle rule | Human 02 and appendix | Test-design examples | +| Mutation score measures complete test quality | Treat as a fault-sensitivity proxy with limitations | Human 01, 06, and appendix | Operating reference | +| Contract tests prove provider correctness | Narrow to exercised compatibility | Human 04, 07, and glossary | Test-level examples; broader operations | +| Mocks prove payment or delivery occurred | Reject; mocks prove the visible requested interaction | Human 03, 04, 07, and appendix | Test-design examples | +| London TDD mocks only external code | Reject; it commonly mocks owned collaborators | Adjacent-practices appendix | Design-for-testability | +| A test pyramid prescribes universal counts | Replace with architecture, risk, and unique evidence questions | Human 04 and appendix | Test-level examples | +| SQLite is a safe universal database substitute | Reject; decide from semantic fidelity | Stack appendix and Human 07 | Test-level examples | +| Tests are the only assurance mechanism | Reject; include complementary evidence | Human 04–06 and appendix | Operating reference | +| TDD applies only to deterministic systems | Replace with the useful-executable-oracle criterion | Human 02, 03, 06, and appendix | Entry strategies | +| TDD is empirically proven superior | Reject universal causal guarantees | Human 01, 03, and 06 | Operating reference | +| One test level should prove the whole feature | Reject; assign each boundary a distinct assurance question | Human 03, 04, and 07 | Test-level examples | +| Mockability is the design objective | Reject; optimize controllability and observability while charging seams for indirection | Human 03, 07, and appendix | Design-for-testability | +| Greenfield TDD begins with a complete up-front test design | Reject; use a provisional walking skeleton and one thin slice | Human 06 and 07 | Entry strategies | +| Legacy characterization must always precede a seam | Reject; a minimal behavior-preserving seam may be needed before any test can run | Human 06 and 07 | Entry strategies | +| Characterization test Green is equivalent to a TDD Red | Reject; characterization records current behavior before the new-behavior cycle | Human 03, 06, and 07 | Entry strategies | +| Test level determines execution cadence | Reject; classify lanes from measured cost, reliability, environment, diagnosis, and risk | Human 04 and 07 | Feedback lanes | +| Fast, standard, and thorough lanes need universal durations | Reject; use project-specific budgets and repository commands | Human 04 | Feedback lanes | +| A fast irrelevant test is preferable to a slow faithful test | Reject | Human 01, 03, 04, and 07 | Cycle invariants; feedback lanes | +| Unrun checks may be implied by a Green result | Reject; report commands, omitted lanes, and remaining uncertainty | Human 02, 04, and 07 | Cycle invariants; step 7 | + +## Audit rule + +When a material claim changes, update its research verdict first, then every destination in that row. + +The change is complete only when human explanations, skill behavior, conditional references, examples, glossary terms, and this matrix agree. diff --git a/setup/optional-skills/skills/changelog-entry/SKILL.md b/setup/optional-skills/skills/changelog-entry/SKILL.md new file mode 100644 index 0000000..6c8abe0 --- /dev/null +++ b/setup/optional-skills/skills/changelog-entry/SKILL.md @@ -0,0 +1,111 @@ +--- +name: changelog-entry +description: Add a notable-change entry to a repository's CHANGELOG.md, written for the affected audience. Use when the user wants the changelog updated for a change, or a changelog bootstrapped for a repo that has none. +--- + +# Changelog entry + +Add a **notable**-change entry to a repository's `CHANGELOG.md`, written for +the **audience** it affects. The invariants below are non-negotiable; the +numbered steps are the process. + +Read [examples](references/examples.md) for a worked model of the repository +type in front of you — application, library/API, or infrastructure — or for +the bootstrap and negative examples. + +## Invariants + +Hold these across every run: + +- **Evidence, not subjects.** A commit subject says what the author typed; the + diff says what changed — derive entries from the change itself, not the log. +- **Notable only.** Record what the **audience** would act on. Exclude + internal refactors, formatting, test-only changes, and dependency churn — + unless they change behavior, compatibility, security, or operator/developer + experience. +- **Audience's perspective.** Each entry says what changed, who or what is + affected when relevant, and why it matters or what to do. +- **Never invent.** No fabricated version, tag, date, release, or claimed + impact. Unreleased work stays under `Unreleased`. When impact is genuinely + unknown, ask. +- **Preserve the format.** Match the existing changelog's structure, category + names, date format, link style, and line endings exactly. + +## 1. Inspect + +- Read repo guidance that governs docs: `AGENTS.md`, `CLAUDE.md`, + `CONTRIBUTING*`, any changelog policy in a README. +- Read `CHANGELOG.md` in full — its category scheme, date format, link style, + and where the newest entries go. +- Gather the change: `git diff`, `git log` for the range, the PR body, the + linked issue. + +Done when you know the repository's changelog convention (or its absence) and +hold the actual changed content. + +## 2. Fix the format and target section + +- **Convention exists** → follow it; format questions stop here. +- **No convention** → default to + [Keep a Changelog](https://keepachangelog.com/en/1.1.0/): `## [Unreleased]` + on top, releases below in reverse chronological order; categories `Added`, + `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security` (only those that + apply); ISO dates on released sections only. + +Done when the target format and the section to edit are fixed. + +## 3. Select and place each notable change + +- List candidates from the diff/commits/PR/issue; drop the non-notable. +- Identify the **audience** and write for it: application → end users; + library/service/API → API consumers; infrastructure → operators; contributor + tooling → contributors, only when the change materially affects the workflow. + Technical language fits a developer audience; raw implementation detail never + does — state the observable effect. +- Assign each entry a category. Make **breaking changes, removals, + deprecations, migrations, and security fixes** unmistakable: state the break + and the required action, never folded into a soft "Changed" line. + +Done when every surviving candidate has an audience, a category, and a stated +reason it matters. + +## 4. Write + +- Reverse chronological; `Unreleased` by default. +- One item per change: specific, independently understandable, in the repo's + format. Name the actual change — "various improvements" and "bug fixes" say + nothing. +- Rewrite from the change's effect, not from the commit message. +- Before adding, scan for an overlapping entry and update that one instead of + adding a second. + +Done when the entries are in the file, in the repository's format, with no +duplicate covering the same change. + +## 5. Show and validate + +- Show the entry (or the diff) to the user. +- Run the repo's Markdown validation (this repo: + `pre-commit run markdownlint-cli2 --files CHANGELOG.md`). +- Ask one focused question only when audience, release target, or user impact + can't be determined from the repository — otherwise state your default and + proceed. + +Done when the entries are placed, the format is intact, Markdown validates, and +nothing was invented. + +## Versioning and history + +- No releases yet → everything stays under `Unreleased`. +- On release, use the repo's scheme; if none exists, SemVer for versioned + packages/artifacts, date-based headings for continuously deployed products. +- Introducing a changelog mid-project: reconstruct history only from verifiable + commits, tags, releases, and PRs. Never imply a historical version that never + existed — group verified changes by date, or under a labeled `Pre-release + development` section. + +## Agent logging + +Agent actions belong in commits, PRs, and issues — not a changelog. Create or +update `AGENT_CHANGELOG.md` only when repository instructions explicitly require +one. diff --git a/setup/optional-skills/skills/changelog-entry/references/examples.md b/setup/optional-skills/skills/changelog-entry/references/examples.md new file mode 100644 index 0000000..d14bb78 --- /dev/null +++ b/setup/optional-skills/skills/changelog-entry/references/examples.md @@ -0,0 +1,125 @@ +# Changelog entry examples + +Worked models for each repository type, a bootstrap example, and negative +examples. All use the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) +default; when a repository has its own convention, follow that instead. + +The point of every example is the same: translate a raw change into an effect +on the repository's audience. + +## Application / product (audience: end users) + +Diff: scheduled CSV exports were built with the server's UTC clock; a config +field now carries the account timezone into the export job. + +```markdown +## [Unreleased] + +### Changed + +- Scheduled exports now use the account timezone instead of the server clock, + so daily reports arrive on the correct calendar day for accounts outside UTC. +``` + +Why it works: names the observable effect (reports on the right day) and who +is affected (non-UTC accounts). It does not mention the config field or the +job class — implementation the end user never sees. + +## Developer library / API (audience: API consumers) + +Diff: `parseDate()` gains a required `locale` parameter; calling it with one +argument no longer compiles. + +```markdown +## [Unreleased] + +### Changed + +- **BREAKING:** `parseDate()` now requires a `locale` argument. Calls with a + single argument no longer type-check. Pass the caller's locale, or + `Locale.ROOT` to keep the previous behavior. + +### Deprecated + +- `parseDateLegacy()` is deprecated and will be removed in the next major + release. Migrate to `parseDate(value, locale)`. +``` + +Why it works: technical language is correct because the audience is +developers, but the entry states the break and the exact migration action, +not the internal parser refactor that enabled it. + +## Infrastructure / deployment (audience: operators) + +Diff: Terraform module changes the default RDS instance class and enables +storage autoscaling. + +```markdown +## [Unreleased] + +### Changed + +- Default database instance class raised from `db.t3.medium` to + `db.r6g.large`. Applying this module to an existing environment triggers a + replacement — schedule a maintenance window before upgrading. + +### Added + +- Storage autoscaling on the database (up to 500 GB), removing the manual + disk-resize runbook step. +``` + +Why it works: written for whoever runs `terraform apply` — it flags the +destructive replacement and the required action (maintenance window), which a +raw "bump instance class" subject would hide. + +## Bootstrap: repo with commits but no releases + +A repository that has been developed for months with no tags introduces a +changelog. Reconstruct only from verifiable history and never imply releases +that never happened. + +```markdown +# Changelog + +Notable changes, following Keep a Changelog. No release has been tagged, so +current work sits under Unreleased and earlier history under Pre-release +development, grouped by commit date. + +## [Unreleased] + +### Added + +- Retry with exponential backoff on the payment webhook handler. + +## Pre-release development + +No versions, tags, or releases existed before this changelog was introduced. +Entries below are reconstructed from commit and PR history, grouped by date. + +### 2026-06-30 + +#### Added + +- Initial REST API for orders and a Postgres-backed persistence layer. +``` + +Why it works: the audience-facing changes are grouped by verifiable date under +an honestly labeled section; no `1.0.0` heading is invented to make the +history look released. + +## Negative examples — do NOT add an entry + +These changes are not notable on their own. Skip them unless they change +behavior, compatibility, security, or operator/developer experience. + +| Change | Why it is skipped | +| -------------------------------------------------------------- | ------------------------------------------------------------------ | +| Renamed a private helper `fetchData` → `loadData` | No observable effect on any audience. | +| Reformatted files with Prettier / ran the linter | Style only. | +| Added unit tests for existing behavior | No behavior change. | +| Bumped a transitive dev dependency with no API/behavior change | Dependency churn. | +| "Various improvements and bug fixes" | Vague filler — banned outright; name the actual change or drop it. | + +A dependency bump *does* earn an entry when it changes behavior, fixes a CVE +(→ `Security`), or alters a supported version — then say which, and why. diff --git a/setup/optional-skills/skills/develop-with-tdd/SKILL.md b/setup/optional-skills/skills/develop-with-tdd/SKILL.md new file mode 100644 index 0000000..25dfe72 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/SKILL.md @@ -0,0 +1,97 @@ +--- +name: develop-with-tdd +description: Develop and safely change code using trustworthy TDD feedback. Use when implementing behavior test-first, reproducing a defect before fixing it, evolving greenfield or legacy code, protecting a pure refactor with existing tests, or when another implementation workflow needs a red-green-refactor loop. +--- + +# Develop with TDD + +Work on one observable behavior at a time. Choose the shortest trustworthy feedback loop that faithfully exercises the current risk. A slower faithful test outranks a fast test that cannot detect that risk. + +Follow repository conventions for stacks, test commands, tags, and gates. Load a reference only when its condition applies: + +- For greenfield, defect, intentional contract change, legacy, pure-refactor, exploration, no-oracle, or immediately-passing-test work, read [entry strategies](references/entry-strategies.md). +- When an oracle, observation boundary, assertion, or test-data choice is uncertain, read [test-design examples](references/test-design-examples.md). +- When risk crosses a component, process, database, service, or deployed-system boundary, read [test-level examples](references/test-level-examples.md). +- When behavior is difficult to control or observe, or an interface, wrapper, or double is being considered, read [design for testability](references/design-for-testability.md). +- When suite cadence is unclear, feedback is too slow, or the repository lacks useful test lanes, read [feedback lanes](references/feedback-lanes.md). +- When creating or repairing environment-dependent, flaky, contract, integration, acceptance, or E2E tests, read [broader-test operations](references/broader-test-operations.md). +- When explaining, reviewing, or adapting this skill's policy, read [the operating reference](references/operating-reference.md). + +## Cycle invariants + +- **Faithful feedback:** Select evidence by risk, not by test count, coverage, or a preferred test level. +- **Known baseline:** Run relevant existing evidence first, or record why it cannot run and which failures already exist. +- **Explicit frame:** State one behavior, its risk, observation boundary, and an independently derived oracle before implementation. +- **Observed Red:** For new or corrected behavior, run the new or revised test and observe a relevant, reproducible failure attributable to that behavior. Never infer Red from confidence or inspection. +- **Green integrity:** Make the framed evidence pass by changing production behavior while preserving the test's sensitivity and the intended contract. +- **Behavior/refactor separation:** Add behavior under Red; change structure under Green while preserving observable behavior. +- **Trustworthy signal:** Diagnose flaky, environmental, or unexplained results before treating them as Red or Green. +- **Closed cycle:** Return to Green before starting the next behavioral decision. +- **Honest completion:** Claim only checks actually run; name omitted checks, unavailable environments, and remaining uncertainty. + +Characterization, pure refactoring, exploration, and work without an executable oracle use the explicit branches in [entry strategies](references/entry-strategies.md); they do not manufacture a Red. + +## 1. Orient and establish the baseline + +- Inspect requirements, public interfaces, nearby production code, existing tests, and repository test instructions. +- Classify the work. Load [entry strategies](references/entry-strategies.md) when it is not a straightforward new behavior in an already-tested area. +- Identify the smallest relevant existing test command and the broader commands expected before handoff. +- Run the smallest relevant baseline when possible. Record pre-existing failures and environmental limitations. + +Complete this step when the work type and repository conventions are known, and the baseline state or its absence is explicit. + +## 2. Frame one behavior + +- State the behavior in caller or domain language. +- Identify the failure risk the evidence must detect. +- Choose the narrowest stable observation boundary that reproduces that risk faithfully. +- Derive the expected result independently through a concrete example, property, model, trusted reference, or reviewed baseline. +- Choose the test level from the risk rather than the code's decomposition. + +Recover choices from requirements and repository evidence. Ask the user before settling a material ambiguity in product behavior, compatibility, oracle, observation boundary, or architecture that those sources cannot resolve. + +Complete this step when the behavior, risk, boundary, oracle, and test level are explicit. + +## 3. Establish Red + +- Add or revise one test that contributes new behavioral information. Revise an existing expectation when the requested contract intentionally changes. +- Make the test runnable, then execute the smallest faithful command. +- Trace the failure to the framed behavior. Treat unrelated compilation, fixture, setup, and environment failures as test-path problems; repair that path before claiming Red. A missing compile-time contract is Red only when that contract is the behavior under development. +- Reproduce the failure when nondeterminism or shared state could explain it. +- If the test passes immediately, follow the immediately-passing-test branch in [entry strategies](references/entry-strategies.md) before touching production code. + +Complete this step only after observing a relevant, reproducible, attributable Red for the expected reason. + +## 4. Reach Green + +- Make the smallest coherent production change that satisfies the example. +- Use an obvious implementation when the rule is clear. Use Fake It or Triangulation when examples have not yet revealed the general rule. +- Preserve assertion strength, test execution, error propagation, and the agreed contract. +- Run the formerly failing test, then relevant fast/local neighboring tests. + +Complete this step when the same evidence is Green for the intended reason, nearby fast feedback is Green, and no unrelated behavior was added. + +## 5. Refactor while Green + +- Improve names, duplication, responsibilities, and boundaries justified by behavior already known. +- Preserve observable behavior at the governing boundary; treat a behavior or compatibility change as a new cycle. +- Run relevant fast/local tests after each meaningful structural change. + +Complete this step when the structure is clear enough for the current behavior and all relevant fast/local evidence remains Green. + +## 6. Continue one behavior at a time + +- Select the next smallest example that adds information. +- Complete its Red, Green, and refactor stages before selecting another. +- Stop adding examples when the requested behavior is covered and additional cases would not change confidence or design. + +Complete this step when every requested behavior has appropriate evidence and every started cycle is closed at Green. + +## 7. Verify and report + +- Run repository-required formatting and static checks. +- Run the standard/change-validation lane before handoff. Run the thorough/system-validation lane when repository policy, risk, or the next transition requires it. +- Inspect the diff for implementation-coupled assertions, speculative code, weakened evidence, and accidental contract changes. +- Report behavior covered, exact commands and lanes run, results, checks not run, and remaining uncertainty. + +Complete the skill only when requested behavior is implemented, all required checks pass, every claimed result was observed, and residual risk is explicit. diff --git a/setup/optional-skills/skills/develop-with-tdd/agents/openai.yaml b/setup/optional-skills/skills/develop-with-tdd/agents/openai.yaml new file mode 100644 index 0000000..d8ab960 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Develop with TDD" + short_description: "Drive changes through trustworthy TDD cycles" + default_prompt: "Use $develop-with-tdd to implement this change one behavior at a time." diff --git a/setup/optional-skills/skills/develop-with-tdd/references/broader-test-operations.md b/setup/optional-skills/skills/develop-with-tdd/references/broader-test-operations.md new file mode 100644 index 0000000..da75643 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/broader-test-operations.md @@ -0,0 +1,61 @@ +# Broader-test operations + +Read this reference when adding or maintaining component, integration, contract, acceptance, E2E, smoke, or scheduled tests. + +Broader tests earn their cost only when their environment and diagnostics preserve a trustworthy signal. + +## Choose fidelity from the risk + +- Use the production engine when provider semantics are the behavior under test. +- Use realistic protocol adapters when serialization or compatibility is the risk. +- Use a fake when its differences cannot affect the asserted behavior. +- Record which production properties the environment intentionally does not represent. + +“More real” is not automatically better. Fidelity is valuable only along dimensions that can change the outcome. + +## Own state + +- Give each test deterministic setup and cleanup. +- Prefer unique data, transactional rollback, disposable resources, or isolated namespaces. +- Keep tests independent of execution order. +- Parallelize only when state ownership supports it; serial execution can conceal missing isolation. + +## Produce failure evidence + +A remote failure should include enough evidence to diagnose without rerunning blindly: + +- assertion and relevant domain inputs; +- application and dependency logs scoped to the test; +- request/response or message correlation identifiers; +- browser trace, screenshot, and console/network errors for UI paths; +- environment and version metadata when compatibility matters. + +Capture sensitive data safely and retain artifacts for an explicit period. + +## Handle flakiness as a defect + +Confirm a suspected flake through repetition or controlled reproduction. Classify its source: shared state, timing, environment, nondeterminism, infrastructure, or an unstable oracle. + +Quarantine only to restore suite signal. Record owner, reason, issue, expiry, and repair path. A quarantined test without ownership is a silently deleted test. + +Track flaky-test rate and repair time. Do not normalize rerunning until green as verification. + +## Operate consumer-driven contracts + +A complete contract lifecycle includes: + +1. consumer interaction generation from behavior the consumer actually uses; +2. explicit provider states that arrange required preconditions; +3. provider verification against the real provider interface; +4. contract publication and version association; +5. compatibility checks before release or deployment. + +Schema conformance and consumer-driven contracts answer different questions. A schema validates allowed shape; a consumer contract verifies that an exercised interaction remains supported. + +Neither proves the provider’s full business correctness or deployed network path. + +## Keep E2E narrow + +E2E means an external path across the assembled system. Its entry point may be a browser, API, CLI, message, or protocol. + +Keep critical paths and unique deployment risks at E2E. Move domain partitions to cheaper focused tests and infrastructure edge cases to integration tests. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/design-for-testability.md b/setup/optional-skills/skills/develop-with-tdd/references/design-for-testability.md new file mode 100644 index 0000000..2d2c8c5 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/design-for-testability.md @@ -0,0 +1,72 @@ +# Design for testability + +Read this reference when behavior is hard to observe or control, or when considering a new interface, wrapper, or test double. + +Testability means controlling relevant inputs and observing relevant outcomes. Mockability is one technique, not the design objective. + +## Expose outcomes + +Prefer a returned result or domain event over requiring tests to inspect logs or private state. + +```typescript +// Observable contract +const receipt = checkout(order); +expect(receipt.status).toBe("confirmed"); +``` + +Keep logs as operational evidence unless log content is itself a published contract. + +## Control difficult boundaries + +Inject time, randomness, network clients, file systems, schedulers, and external services when their real behavior would make the focused test slow, destructive, or nondeterministic. + +```typescript +function expiresAt(clock: Clock): Date { + return addMinutes(clock.now(), 15); +} +``` + +Pass the smallest capability needed. Avoid exposing an entire framework client when one domain operation is enough. + +## Prefer operation-shaped ports + +Specific: + +```typescript +interface CustomerDirectory { + findCustomer(id: CustomerId): Promise; + saveCustomer(customer: Customer): Promise; +} +``` + +Over-general: + +```typescript +interface RemoteClient { + request(operation: string, payload: unknown): Promise; +} +``` + +Operation-shaped ports reveal the contract and return one known shape. A generic dispatcher leaks protocol generality into callers and their doubles. + +## Respect the project’s TDD school + +In classicist/sociable TDD, let owned deterministic collaborators run together and substitute difficult boundaries. + +In London/mockist TDD, define collaborator protocols outside-in and verify messages. Mocking an owned collaborator is expected in that style. + +Do not silently mix the schools within one design area. Follow repository evidence or make the migration a project decision. + +## Charge every seam rent + +An interface, wrapper, or dependency-injection parameter adds concepts and maintenance. Introduce it when it creates a stable ownership boundary, controls a difficult dependency, centralizes policy, or enables meaningful substitution. + +Keep direct code when an indirection exists only to satisfy a mock framework. Test-induced design damage is a signal that isolation cost exceeds assurance value. + +## Keep deep modules + +A useful boundary hides substantial implementation choices behind a small contract. Avoid one-interface-per-class structures that expose the same decomposition in production and tests. + +## Verify the boundary itself + +A double verifies code on one side of a port. Add contract or integration evidence for the production adapter when compatibility or infrastructure semantics matter. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/entry-strategies.md b/setup/optional-skills/skills/develop-with-tdd/references/entry-strategies.md new file mode 100644 index 0000000..8f49e8c --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/entry-strategies.md @@ -0,0 +1,75 @@ +# TDD entry strategies + +Read only the branch that matches the work. Return to the core Red-Green-Refactor workflow when the branch establishes a testable behavioral decision. + +## Greenfield work + +1. Inspect repository conventions and establish the smallest test harness needed for the first behavior. +2. If the principal uncertainty is deployment or cross-component wiring, build a walking skeleton: one minimal path through the intended architecture and delivery mechanism. +3. Treat that skeleton as a provisional architecture probe, not proof of business correctness. +4. Select one stakeholder-visible slice and drive its inner behavior through normal cycles. +5. Use an outer acceptance example only when it clarifies the slice. Keep unfinished outer evidence branch-local or explicitly outside required gates until the feature is expected to pass. + +Do not design the complete test portfolio or architecture before the first useful slice. + +## Defect correction + +1. Establish the existing relevant baseline. +2. Reproduce the reported symptom at the narrowest boundary that still exercises the suspected risk. +3. Confirm that the failure is attributable to the defect rather than the fixture or environment. +4. Make the correction, run the reproducer, and run neighboring regression evidence. + +If the symptom cannot be reproduced, gather stronger diagnostics or clarify the report before making a speculative production change. + +## Intentional contract change + +1. Identify which existing expectations describe the old contract and which behavior remains unaffected. +2. Revise an obsolete expectation or add a new one to express the requested contract. +3. Observe Red against the old production behavior. +4. Implement the new contract, preserve unaffected behavior, and update other public contract artifacts when required. + +Changing a test is correct when the agreed behavior changed. Weakening a test to accommodate an unintended implementation change is not. + +## Untested or fragile legacy code + +1. Identify the exact change risk and behavior that must remain stable. +2. If no useful test can run, introduce the smallest behavior-preserving seam needed to control inputs or observe outcomes. Verify that preparatory edit with the best existing evidence and review available. +3. Add characterization evidence for behavior that must be preserved. Expect it to begin Green when it records current behavior. +4. Treat characterization as a record, not approval. Resolve whether questionable user-visible behavior is preserved or corrected. +5. Establish Red for the new or corrected behavior, then follow the normal cycle. + +Create only the seam required for the next safe change; avoid a preparatory redesign of the legacy area. + +## Pure refactoring + +1. Establish a Green baseline at the governing observation boundary. +2. Change structure without intentionally changing behavior or compatibility. +3. Run relevant fast/local evidence after each meaningful edit and broader evidence before handoff as risk requires. + +No new Red is required because no new behavior is being requested. If behavior must change, stop and frame a normal cycle. + +## Disposable exploration + +Use a time-boxed spike when the interface, feasibility, or expected behavior is not yet understood well enough to form a useful oracle. Keep the spike disposable. After learning: + +- discard it and drive the chosen behavior through TDD; or +- explicitly characterize retained behavior before treating the code as maintained production code. + +Do not present exploratory execution as Red-Green-Refactor evidence. + +## No useful executable oracle + +Use the strongest suitable alternative evidence when correctness cannot be expressed as a stable executable oracle. Examples include reviewed qualitative judgment, representative statistical evaluation, exploratory testing, formal analysis, production monitoring, or reconciliation. + +Report why TDD does not govern that part of the change, which evidence replaces it, and what uncertainty remains. Continue to use TDD for deterministic surrounding code where useful. + +## Test passes immediately + +Determine which condition applies before changing production code: + +- **Behavior already exists:** confirm the example exercises production code and retain it only if it adds durable regression value. +- **Insensitive oracle or data:** choose inputs and assertions that distinguish plausible wrong behavior. +- **Wrong path or boundary:** trace execution and move the observation boundary to the risk. +- **Duplicate evidence:** remove or consolidate it if it adds no information. + +When uncertainty remains, demonstrate sensitivity with a controlled mutation or alternate failing input, then restore the probe. If the requested behavior already exists, report that finding rather than manufacture a production change or an artificial Red. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/feedback-lanes.md b/setup/optional-skills/skills/develop-with-tdd/references/feedback-lanes.md new file mode 100644 index 0000000..bed29ae --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/feedback-lanes.md @@ -0,0 +1,55 @@ +# Feedback lanes + +Read this reference when selecting suite cadence, when feedback is too slow, or when the repository has no useful test grouping. + +Tests retained after TDD become part of the project's verification portfolio; they are not a separate species of “TDD tests.” Test level and execution cadence are independent: a focused test may be slow, and an integration test may be fast. + +## The three lanes + +| Lane | Decision it supports | Typical contents | +| ------------------------------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Fast/local** | Is the current small edit moving in the intended direction? | The new or revised test, relevant focused tests, and fast deterministic neighboring tests | +| **Standard/change validation** | Is the affected area safe enough to hand off? | Changed modules or components, relevant integrations and contracts, plus repository-required static checks | +| **Thorough/system validation** | Is the system safe enough for a risky transition? | The full relevant portfolio, critical E2E paths, representative environments, and specialized security, performance, resilience, compatibility, or statistical evaluation | + +Use repository names and commands when they already express these decisions. The lane names describe intent; they do not require three literal scripts. + +## Cadence through a change + +1. **Before Red:** run the smallest relevant baseline. +2. **At Red:** run the new or revised test through the smallest faithful command. +3. **At Green:** run that test and relevant fast/local neighbors. +4. **During refactoring:** rerun fast/local evidence after each meaningful structural edit. +5. **Before handoff:** run standard/change-validation evidence. +6. **Before merge, deployment, release, or another risky transition:** run thorough/system evidence when repository policy or risk requires it. + +A slower integration or E2E test is the correct Red when only that boundary can detect the risk. Shorten setup and diagnostics where possible; do not substitute a fast test that answers a different question. + +## Classify from evidence + +Classify tests using: + +- measured execution time rather than assumed test-level speed; +- reliability and state isolation; +- environment and service requirements; +- diagnostic quality; +- the risk uniquely covered; +- the decision or gate the result supports. + +Avoid universal duration thresholds. Re-measure periodically because suites and infrastructure change. + +## Establish or change lanes + +When useful lanes do not exist: + +1. inventory existing commands, tags, projects, and CI jobs; +2. measure representative execution times and flaky behavior; +3. map unique risks to the smallest faithful commands; +4. propose groupings that preserve required evidence and improve feedback; +5. treat new tags, scripts, and CI gate changes as project decisions with team ownership. + +Do not silently redefine required gates while implementing a feature. The skill may recommend a lane design; apply repository-wide gate changes only when the task or user authorizes them. + +## Report the signal + +Name the commands and lanes actually run, their results, broader lanes omitted, and the uncertainty each omission leaves. A Green fast/local lane is not a claim that the standard or thorough lane would pass. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/operating-reference.md b/setup/optional-skills/skills/develop-with-tdd/references/operating-reference.md new file mode 100644 index 0000000..8a29862 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/operating-reference.md @@ -0,0 +1,58 @@ +# TDD operating reference + +Read this reference when explaining, reviewing, or adapting the skill's policy. Ordinary TDD cycles use `SKILL.md` directly. + +## Guidance strength + +- **Definition:** shared meaning required for correct interpretation. +- **Default:** preferred action unless repository evidence or explicit requirements justify another choice. +- **Heuristic:** diagnostic prompt rather than a compliance rule. +- **Project decision:** choice governed by the repository, team, regulatory context, or user. + +Apply explicit requirements and repository-wide conventions before defaults. Ask for a decision only when requirements and repository evidence cannot resolve a material product, compatibility, oracle, observation-boundary, or architecture choice. + +## Conceptual boundaries + +- **Definition:** TDD interleaves executable behavioral evidence, implementation, and design improvement. Its immediate value is feedback during design; retained tests later provide regression evidence. +- **Definition:** Test-first timing, test level, and execution cadence are separate dimensions. TDD does not imply unit-only tests or one universal suite speed. +- **Definition:** An observation boundary is the interface through which a test supplies inputs and observes outcomes. +- **Definition:** A seam is a place where behavior or a dependency can be varied. A seam may make a boundary controllable, but the terms are not synonyms. +- **Definition:** Testability is the ability to control relevant inputs and observe relevant outcomes. Mockability is one technique, not the objective. +- **Definition:** Coverage records execution. It does not establish oracle strength, requirement completeness, fault detection, or design quality. +- **Definition:** Characterization records current behavior. It does not approve that behavior as correct or desirable. + +## Why the cycle gates matter + +- **Observed Red** demonstrates that the test can detect the missing or defective behavior. A claimed or inferred Red supplies no evidence of sensitivity. +- **Minimal coherent Green** limits the number of decisions made without feedback. It does not require deliberately poor code. +- **Refactoring under Green** separates structural decisions from behavioral decisions so regressions remain attributable. +- **One closed cycle at a time** keeps failures local and recent enough to diagnose. + +These mechanisms reduce uncertainty; they do not make either the implementation or its tests infallible. + +## Evidence decisions + +- **Default:** Select evidence from the risk and question it must answer, not from source-file, method, test-count, or coverage targets. +- **Default:** Prefer a stable focused boundary for domain rules and transformations when it faithfully exposes the risk. +- **Default:** Use real integration semantics for framework wiring, database behavior, serialization, deployment configuration, and cross-service behavior when substitutes could change the outcome. +- **Default:** Derive oracles independently through domain examples, known literals, properties, trusted references, metamorphic relations, models, or reviewed baselines. +- **Heuristic:** A test that breaks under behavior-preserving restructuring may be observing decomposition rather than behavior. +- **Project decision:** Accessibility, security, performance, resilience, privacy, concurrency, safety, operability, and probabilistic behavior require evidence appropriate to their risk and representative environment. + +## Bound the claims + +TDD supplies local feedback; it does not replace product discovery, architecture, code review, exploratory testing, static analysis, integration evidence, delivery verification, or production monitoring. Its effect depends on task, practitioner experience, test design, codebase, and comparison baseline. + +BDD, ATDD, and Specification by Example may improve the behavior and examples entering the cycle. They complement TDD rather than changing every test into an acceptance scenario. + +## Preserve project authority + +Treat these as project decisions: + +- classicist/sociable versus London/mockist collaboration style; +- published compatibility and migration policy; +- production-like environments and acceptable substitutes; +- required structural coverage or regulatory evidence; +- test tags, lane definitions, CI gates, and release criteria. + +Recommend changes when evidence shows a problem; do not silently replace repository-wide policy inside a feature change. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/test-design-examples.md b/setup/optional-skills/skills/develop-with-tdd/references/test-design-examples.md new file mode 100644 index 0000000..194d943 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/test-design-examples.md @@ -0,0 +1,151 @@ +# Test-design examples + +Read this reference when writing or reviewing a test, choosing an oracle, or deciding whether an assertion is coupled to structure. + +## Contents + +- [Behavior versus decomposition](#behavior-versus-decomposition) +- [Independent oracle versus tautology](#independent-oracle-versus-tautology) +- [Relevant Red versus harness failure](#relevant-red-versus-harness-failure) +- [Public observation versus back-door inspection](#public-observation-versus-back-door-inspection) +- [Requested effect versus real effect](#requested-effect-versus-real-effect) +- [Meaningful completion versus empty execution](#meaningful-completion-versus-empty-execution) +- [Test-first versus durable test-after](#test-first-versus-durable-test-after) +- [Discriminating test data](#discriminating-test-data) +- [Factories with meaningful defaults](#factories-with-meaningful-defaults) + +## Behavior versus decomposition + +Structure-coupled: + +```typescript +test("checkout calls calculateTax then saveOrder", async () => { + await checkout(cart); + expect(calculateTax).toHaveBeenCalledBefore(saveOrder); +}); +``` + +Behavior-coupled: + +```typescript +test("checkout confirms an order with the calculated total", async () => { + const result = await checkout(cartWithItems(10, 5)); + expect(result).toEqual({ status: "confirmed", total: 15 }); +}); +``` + +Use the second form in classicist code when callers care about the result. The first form can be valid in deliberate London/mockist code when collaborator protocol and ordering are the behavior being designed. + +## Independent oracle versus tautology + +Tautological: + +```typescript +const expected = items.reduce((sum, item) => sum + item.price, 0); +expect(calculateTotal(items)).toBe(expected); +``` + +Independent: + +```typescript +expect(calculateTotal([{ price: 10 }, { price: 5 }])).toBe(15); +``` + +The first test repeats the production algorithm. Prefer a worked domain example, known literal, property, trusted reference, or reviewed baseline. + +## Relevant Red versus harness failure + +A useful Red reaches the intended observation boundary and fails because the behavior is absent or wrong. For example: + +```text +Expected shipping price: 0.00 +Received shipping price: 4.99 +``` + +This failure demonstrates that the example can distinguish the current behavior from the requested rule. + +These failures do not yet establish that signal: + +```text +ModuleNotFoundError: test fixture package is missing +Connection refused before the request reached the service +SyntaxError in the test file +``` + +Repair the test path and run again. A missing type, symbol, or endpoint can be a relevant Red when introducing that public contract is the framed behavior; otherwise continue until the behavioral expectation itself fails. + +Observe the output rather than predicting it. If the test is already Green, investigate whether the behavior exists, the test misses production code, or its data and oracle cannot distinguish a plausible fault. + +## Public observation versus back-door inspection + +Usually coupled to storage: + +```typescript +await registerUser({ email: "a@example.com" }); +expect( + await db.query("select * from users where email = ?", ["a@example.com"]), +).toHaveLength(1); +``` + +Observed through the caller contract: + +```typescript +const user = await registerUser({ email: "a@example.com" }); +expect(await getUser(user.id)).toMatchObject({ email: "a@example.com" }); +``` + +Direct database observation is correct when the database artifact is the contract—for example a migration, reporting table, or downstream integration. Choose from the risk, not a blanket ban. + +## Requested effect versus real effect + +```typescript +await placeOrder(order); +expect(paymentGateway.charge).toHaveBeenCalledWith(order.total); +``` + +This proves that the application requested a charge through its boundary. It does not prove settlement. Add provider contract, sandbox integration, reconciliation, or operational evidence when the real effect matters. + +## Meaningful completion versus empty execution + +Weak unless non-throwing is the contract: + +```python +def test_import_runs(): + import_records(valid_rows) +``` + +Meaningful when successful completion is the behavior: + +```python +def test_empty_batch_is_accepted(): + import_records([]) # any exception fails the test +``` + +An explicit assertion is not mandatory; an effective oracle is. Name the non-throwing contract so the test cannot masquerade as stronger verification. + +## Test-first versus durable test-after + +A black-box test written after implementation can still be durable specification and regression evidence. It did not provide design feedback during the original implementation. Evaluate timing and coupling separately. + +## Discriminating test data + +Weak examples can let plausible faults survive: + +```python +assert multiply_price(10, 1) == 10 +``` + +The multiplier identity cannot distinguish multiplication from returning the original price. Use a value such as `3` when the rule must demonstrate scaling, then add boundary values that distinguish relevant alternatives. + +Ask which plausible implementation mistake each example would detect. Mutation tooling can support this reasoning, but it is not required in every cycle. + +## Factories with meaningful defaults + +Use a factory when repeated setup obscures behavior: + +```typescript +const order = orderFactory({ status: "cancelled" }); +expect(() => ship(order)).toThrow(InvalidTransition); +``` + +Defaults should form a valid, recognizable domain object. Override only fields relevant to the behavior. Random or unrealistic defaults make failures difficult to interpret. diff --git a/setup/optional-skills/skills/develop-with-tdd/references/test-level-examples.md b/setup/optional-skills/skills/develop-with-tdd/references/test-level-examples.md new file mode 100644 index 0000000..79d5ea9 --- /dev/null +++ b/setup/optional-skills/skills/develop-with-tdd/references/test-level-examples.md @@ -0,0 +1,83 @@ +# Test-level decision examples + +Read this reference when choosing between focused, component, integration, contract, acceptance, and E2E evidence. + +## Decision table + +| Risk or question | Smallest faithful evidence | What it does not prove | +| ----------------------------------------------------- | --------------------------------------------------- | -------------------------------------------------- | +| Shipping threshold and rounding | Focused domain test | Database or UI wiring | +| ORM translation, collation, transaction, or migration | Integration test against the relevant engine | Complete user workflow | +| Consumer request/response compatibility | Consumer-driven contract plus provider verification | Provider business correctness or deployment wiring | +| A component with real owned collaborators | Component test through its public API | Cross-service deployment | +| A critical deployed user journey | E2E test from an outside observer | Every business-rule partition | +| Parser invariants over large input space | Property-based focused test | Production integrations | +| Complex legacy output | Reviewed approval/characterization test | Whether historical behavior is desirable | + +## Focused example: domain boundary + +```python +def test_fifty_euros_qualifies_for_free_shipping(): + assert shipping_price(Decimal("50.00")) == Decimal("0.00") +``` + +This is the right level when the risk is the inclusive rule. Driving a browser to test `>=` adds cost without adding relevant evidence. + +## Integration example: database semantics + +```csharp +[Fact] +public async Task UsernamesAreUniqueUsingProductionCollation() +{ + await repository.Add(new User("Alice")); + var result = () => repository.Add(new User("alice")); + await result.Should().ThrowAsync(); +} +``` + +Run this against the production database engine or a disposable instance with the relevant schema and collation. An in-memory collection cannot certify provider translation or constraint behavior. + +## Contract example: compatibility + +A consumer contract may state: + +```json +{ + "request": { "method": "GET", "path": "/users/42" }, + "response": { "status": 200, "body": { "id": 42, "name": "Alice" } } +} +``` + +Consumer tests create the expectation; provider verification replays it against the provider. This catches incompatible messages without deploying the entire system. It does not prove authorization, persistence, or every provider rule. + +Complete consumer-driven testing also needs provider-state setup, contract publication/versioning, and a compatibility check before release. A JSON Schema or OpenAPI check validates shape but is not the same lifecycle. + +## E2E example: deployed capability + +```typescript +test("customer completes checkout", async ({ page }) => { + await page.goto("/cart"); + await page.getByRole("button", { name: "Checkout" }).click(); + await expect( + page.getByRole("heading", { name: "Order confirmed" }), + ).toBeVisible(); +}); +``` + +Use an E2E test for a small number of critical deployed paths. Prefer user-facing roles and outcomes, isolated test data, explicit environment ownership, and failure artifacts such as traces or screenshots. + +Keep edge partitions in faster lower-level tests. Do not reproduce the full business-rule matrix through the browser. + +E2E is not synonymous with browser testing. An external API, CLI, message, or protocol can be the entry point when that is how the assembled system is used. + +## Complementary evidence + +A feature may need several non-duplicating checks: + +1. focused examples for domain rules; +2. integration tests for infrastructure semantics; +3. contracts for independently deployed boundaries; +4. one E2E path for deployment and wiring; +5. monitoring or reconciliation for effects only production can reveal. + +Each check must have a distinct assurance question. Remove a slower test when a cheaper test provides the same evidence and no unique risk remains. diff --git a/setup/tests/Complete-Setup.Tests.ps1 b/setup/tests/Complete-Setup.Tests.ps1 new file mode 100644 index 0000000..f384fe9 --- /dev/null +++ b/setup/tests/Complete-Setup.Tests.ps1 @@ -0,0 +1,80 @@ +#Requires -Version 7.0 +Set-StrictMode -Version Latest + +BeforeAll { + # Dot-source the SUT: defines its functions; the run guard skips execution. + . (Join-Path $PSScriptRoot '..' 'Complete-Setup.ps1') +} + +Describe 'Test-RemovalAllowed' -Tag 'Fast' { + It 'blocks removal on a template repo, naming the failsafe' { + $v = Test-RemovalAllowed -RepoInfo @{ isTemplate = $true; nameWithOwner = 'DenWin/Template' } + $v.Allowed | Should -BeFalse + $v.Reason | Should -Match 'template' + } + + It 'allows removal on a repo created from the template' { + (Test-RemovalAllowed -RepoInfo @{ isTemplate = $false; nameWithOwner = 'DenWin/NewRepo' }).Allowed | + Should -BeTrue + } +} + +Describe 'Get-RemovalPullRequestBody' -Tag 'Fast' { + BeforeAll { + $script:Body = Get-RemovalPullRequestBody + } + + It 'fills every section the PR template demands' { + foreach ($section in '## Goal', '## Scope', '## Risk & rollback', '## Evidence') { + $script:Body | Should -Match ([regex]::Escape($section)) + } + } + + It 'cites the completed setup scripts as Evidence' { + $script:Body | Should -Match 'Protect-MainBranch' + $script:Body | Should -Match 'Enable-RepoSecurity' + } +} + +Describe 'Invoke-SetupRemoval' -Tag 'Fast' { + It 'returns non-zero when gh is not installed' { + Mock Test-GhCli { $false } + Invoke-SetupRemoval 2>$null | Should -Be 1 + } + + It 'refuses to touch a template repo — no git mutation happens' { + Mock Test-GhCli { $true } + Mock gh { '{"isTemplate":true,"nameWithOwner":"DenWin/Template"}' } + Mock git { $global:LASTEXITCODE = 0 } + Invoke-SetupRemoval 2>$null | Should -Be 1 + Should -Invoke git -Exactly -Times 0 + } + + It 'refuses on a dirty worktree so the PR holds only the removal' { + Mock Test-GhCli { $true } + Mock gh { '{"isTemplate":false,"nameWithOwner":"o/r"}' } + Mock git { + $global:LASTEXITCODE = 0 + if ($args -contains 'status') { return ' M some/file.ps1' } + } + Invoke-SetupRemoval 2>$null | Should -Be 1 + Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 0 + } + + It 'removes setup/, pushes a branch, and opens the PR on the happy path' { + Mock Test-GhCli { $true } + Mock gh { + $global:LASTEXITCODE = 0 + if ($args -contains 'view') { return '{"isTemplate":false,"nameWithOwner":"o/r"}' } + if ($args -contains 'create') { return 'https://github.com/o/r/pull/1' } + } + Mock git { + $global:LASTEXITCODE = 0 + if ($args -contains 'status') { return '' } + } + Invoke-SetupRemoval | Should -Be 0 + Should -Invoke git -ParameterFilter { $args -contains 'rm' } -Exactly -Times 1 + Should -Invoke git -ParameterFilter { $args -contains 'push' } -Exactly -Times 1 + Should -Invoke gh -ParameterFilter { $args -contains 'create' } -Exactly -Times 1 + } +} diff --git a/setup/Enable-RepoSecurity.Tests.ps1 b/setup/tests/Enable-RepoSecurity.Tests.ps1 similarity index 97% rename from setup/Enable-RepoSecurity.Tests.ps1 rename to setup/tests/Enable-RepoSecurity.Tests.ps1 index bf8ded0..417255d 100644 --- a/setup/Enable-RepoSecurity.Tests.ps1 +++ b/setup/tests/Enable-RepoSecurity.Tests.ps1 @@ -3,7 +3,7 @@ Set-StrictMode -Version Latest BeforeAll { # Dot-source the SUT: defines its functions; the run guard skips execution. - . (Join-Path $PSScriptRoot 'Enable-RepoSecurity.ps1') + . (Join-Path $PSScriptRoot '..' 'Enable-RepoSecurity.ps1') } Describe 'Get-SecuritySetting' -Tag 'Fast' { diff --git a/setup/tests/New-AIMaintainerApp.Tests.ps1 b/setup/tests/New-AIMaintainerApp.Tests.ps1 new file mode 100644 index 0000000..220bd3e --- /dev/null +++ b/setup/tests/New-AIMaintainerApp.Tests.ps1 @@ -0,0 +1,87 @@ +#Requires -Version 7.0 +Set-StrictMode -Version Latest + +BeforeAll { + # Dot-source the SUT: defines its functions; the run guard skips execution. + . (Join-Path $PSScriptRoot '..' 'New-AIMaintainerApp.ps1') +} + +Describe 'Get-AppManifest' -Tag 'Fast' { + BeforeAll { + $script:M = Get-AppManifest -AppName 'denwin-ai-maintainer' -RedirectUrl 'http://localhost:8712/callback' + } + + It 'grants exactly Contents RW, Pull requests RW, Metadata RO — nothing else' { + $p = $script:M.default_permissions + $p.contents | Should -Be 'write' + $p.pull_requests | Should -Be 'write' + $p.metadata | Should -Be 'read' + $p.Keys.Count | Should -Be 3 + } + + It 'grants no admin, workflow, or secrets permission' { + $keys = @($script:M.default_permissions.Keys) + $keys | Should -Not -Contain 'administration' + $keys | Should -Not -Contain 'workflows' + $keys | Should -Not -Contain 'secrets' + } + + It 'keeps the webhook inactive (a maintainer bot needs none)' { + $script:M.hook_attributes.active | Should -BeFalse + } + + It 'registers a private app under the given name with the callback redirect' { + $script:M.public | Should -BeFalse + $script:M.name | Should -Be 'denwin-ai-maintainer' + $script:M.redirect_url | Should -Be 'http://localhost:8712/callback' + } +} + +Describe 'Get-ManifestFormHtml' -Tag 'Fast' { + BeforeAll { + $manifest = Get-AppManifest -AppName 'my-bot' -RedirectUrl 'http://localhost:8712/callback' + $script:Html = Get-ManifestFormHtml -Manifest $manifest -Owner 'DenWin' + } + + It 'posts to the personal-account manifest endpoint by default' { + $script:Html | Should -Match 'action="https://github\.com/settings/apps/new' + } + + It 'posts to the organization endpoint when -Organization is set' { + $manifest = Get-AppManifest -AppName 'my-bot' -RedirectUrl 'http://localhost:8712/callback' + $orgHtml = Get-ManifestFormHtml -Manifest $manifest -Owner 'my-org' -Organization + $orgHtml | Should -Match 'action="https://github\.com/organizations/my-org/settings/apps/new' + } + + It 'embeds the manifest JSON and auto-submits' { + $script:Html | Should -Match 'my-bot' + $script:Html | Should -Match '\.submit\(\)' + } +} + +Describe 'Save-AppCredential' -Tag 'Fast' { + It 'writes the private key next to nothing else and returns its path' { + $dir = Join-Path $TestDrive 'keys' + $app = @{ id = 42; slug = 'my-bot'; pem = "-----BEGIN RSA PRIVATE KEY-----`nkey`n-----END RSA PRIVATE KEY-----" } + $path = Save-AppCredential -App $app -OutputDirectory $dir + $path | Should -Be (Join-Path $dir 'my-bot.private-key.pem') + Get-Content $path -Raw | Should -Match 'BEGIN RSA PRIVATE KEY' + } + + It 'hardens private-key permissions immediately after writing' { + Mock Protect-PrivateKeyFile {} + $dir = Join-Path $TestDrive 'keys' + $app = @{ id = 42; slug = 'my-bot'; pem = "-----BEGIN RSA PRIVATE KEY-----`nkey`n-----END RSA PRIVATE KEY-----" } + + $path = Save-AppCredential -App $app -OutputDirectory $dir + + Should -Invoke Protect-PrivateKeyFile -Times 1 -Exactly -ParameterFilter { $Path -eq $path } + } +} + +Describe 'Invoke-AppCreation' -Tag 'Fast' { + It 'returns non-zero when gh is not installed' { + Mock Test-GhCli { $false } + Invoke-AppCreation -AppName 'x' 2>$null | Should -Be 1 + } +} diff --git a/setup/tests/Protect-MainBranch.Tests.ps1 b/setup/tests/Protect-MainBranch.Tests.ps1 new file mode 100644 index 0000000..1f5961a --- /dev/null +++ b/setup/tests/Protect-MainBranch.Tests.ps1 @@ -0,0 +1,130 @@ +#Requires -Version 7.0 +Set-StrictMode -Version Latest + +BeforeAll { + # Dot-source the SUT: defines its functions; the run guard skips execution. + . (Join-Path $PSScriptRoot '..' 'Protect-MainBranch.ps1') +} + +Describe 'Get-ProtectionRuleset' -Tag 'Fast' { + BeforeAll { + $script:Rs = Get-ProtectionRuleset -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' + $script:AllRules = $script:Rs.rules + } + + It 'returns one ruleset per rule, each targeting the default branch' { + $script:Rs.Count | Should -Be 5 + $script:Rs | ForEach-Object { $_.conditions.ref_name.include | Should -Contain '~DEFAULT_BRANCH' } + } + + It 'gives every ruleset a unique name' { + ($script:Rs.name | Sort-Object -Unique).Count | Should -Be $script:Rs.Count + } + + It 'requires a pull request' { + ($script:AllRules | Where-Object type -EQ 'pull_request') | Should -Not -BeNullOrEmpty + } + + It 'requires the given status check' { + $rsc = ($script:AllRules | Where-Object type -EQ 'required_status_checks').parameters.required_status_checks + $rsc.context | Should -Contain 'lint' + } + + It 'blocks force-push and deletion' { + $script:AllRules.type | Should -Contain 'non_fast_forward' + $script:AllRules.type | Should -Contain 'deletion' + } + + It 'routes merges through a merge queue, isolated in its own ruleset' { + $script:AllRules.type | Should -Contain 'merge_queue' + ($script:Rs | Where-Object { $_.rules.type -contains 'merge_queue' }).rules.Count | Should -Be 1 + } + + It 'requires review threads to be resolved (holds auto-merge on open comments)' { + ($script:AllRules | Where-Object type -EQ 'pull_request').parameters.required_review_thread_resolution | + Should -BeTrue + } +} + +Describe 'Get-StrictLayerRuleset' -Tag 'Fast' { + BeforeAll { + $script:L = Get-StrictLayerRuleset + } + + It 'exempts the repository admin role' { + $script:L.bypass_actors.actor_type | Should -Contain 'RepositoryRole' + $script:L.bypass_actors.bypass_mode | Should -Contain 'exempt' + } + + It 'requires one code-owner approval given after the last push' { + $p = ($script:L.rules | Where-Object type -EQ 'pull_request').parameters + $p.required_approving_review_count | Should -Be 1 + $p.require_code_owner_review | Should -BeTrue + $p.require_last_push_approval | Should -BeTrue + $p.dismiss_stale_reviews_on_push | Should -BeTrue + } + + It 'targets the default branch under a distinct name' { + $script:L.conditions.ref_name.include | Should -Contain '~DEFAULT_BRANCH' + $script:L.name | Should -Not -Be 'main-protection' + } +} + +Describe 'Get-MergeFlowSetting' -Tag 'Fast' { + It 'enables auto-merge and branch cleanup, and touches nothing else' { + $s = Get-MergeFlowSetting + $s.allow_auto_merge | Should -BeTrue + $s.delete_branch_on_merge | Should -BeTrue + # No merge-method keys: a repo-wide disable would also bind bypass actors. + $s.Keys | Where-Object { $_ -match 'allow_.*_merge$' -and $_ -ne 'allow_auto_merge' } | + Should -BeNullOrEmpty + } +} + +Describe 'Invoke-BranchProtection' -Tag 'Fast' { + It 'returns non-zero when gh is not installed' { + Mock Test-GhCli { $false } + Invoke-BranchProtection -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' 2>$null | + Should -Be 1 + } + + It 'keeps applying the rest when one ruleset (e.g. merge_queue) is rejected' { + $script:RulesetPostCalls = 0 + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if (($args -join ' ') -match 'repos/.*/rulesets') { + $script:RulesetPostCalls++ + if ($script:RulesetPostCalls -eq 5) { $global:LASTEXITCODE = 1; return } + } + $global:LASTEXITCODE = 0 + } + Invoke-BranchProtection -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' 3>$null | + Should -Be 0 + } + + It 'returns non-zero only when every ruleset is rejected' { + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + $global:LASTEXITCODE = 1 + } + Invoke-BranchProtection -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' 2>$null 3>$null | + Should -Be 1 + } + + It 'returns non-zero when a required ruleset (not merge_queue) is rejected' { + $script:RulesetPostCalls = 0 + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if (($args -join ' ') -match 'repos/.*/rulesets') { + $script:RulesetPostCalls++ + if ($script:RulesetPostCalls -eq 3) { $global:LASTEXITCODE = 1; return } + } + $global:LASTEXITCODE = 0 + } + Invoke-BranchProtection -CheckName 'lint' -RequiredApprovals 0 -MergeMethod 'SQUASH' 2>$null 3>$null | + Should -Be 1 + } +} diff --git a/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 b/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 new file mode 100644 index 0000000..853c413 --- /dev/null +++ b/setup/tests/Test-AIMaintainerIdentity.Tests.ps1 @@ -0,0 +1,100 @@ +#Requires -Version 7.0 +Set-StrictMode -Version Latest + +BeforeAll { + # Dot-source the SUT: defines its functions; the run guard skips execution. + . (Join-Path $PSScriptRoot '..' 'Test-AIMaintainerIdentity.ps1') +} + +Describe 'Get-TokenKind' -Tag 'Fast' { + It 'classifies from its documented prefix' -ForEach @( + # Fixtures from GitHub's token-format docs, not real credentials. + @{ Token = 'ghs_16C7e42F292c6912E7710c838347Ae178B4a'; Kind = 'Installation' } # gitleaks:allow + @{ Token = 'github_pat_11ABC123_abcdef'; Kind = 'FineGrainedPat' } # gitleaks:allow + @{ Token = 'ghp_16C7e42F292c6912E7710c838347Ae178B4a'; Kind = 'ClassicPat' } # gitleaks:allow + @{ Token = 'gho_16C7e42F292c6912E7710c838347Ae178B4a'; Kind = 'OAuthUser' } # gitleaks:allow + @{ Token = 'ghu_16C7e42F292c6912E7710c838347Ae178B4a'; Kind = 'UserToServer' } # gitleaks:allow + @{ Token = 'something-else-entirely'; Kind = 'Unknown' } + ) { + Get-TokenKind -Token $Token | Should -Be $Kind + } +} + +Describe 'Get-IdentityVerdict' -Tag 'Fast' { + It 'passes an installation token without admin' { + (Get-IdentityVerdict -TokenKind 'Installation' -HasAdmin $false -HasWrite $true).Pass | Should -BeTrue + } + + It 'passes a fine-grained PAT without admin' { + (Get-IdentityVerdict -TokenKind 'FineGrainedPat' -HasAdmin $false -HasWrite $true).Pass | Should -BeTrue + } + + It 'fails any credential that has admin on the repo, naming the containment risk' { + $v = Get-IdentityVerdict -TokenKind 'Installation' -HasAdmin $true -HasWrite $true + $v.Pass | Should -BeFalse + $v.Findings -join ' ' | Should -Match 'admin' + } + + It 'fails an OAuth user token (the owner''s interactive gh session)' { + $v = Get-IdentityVerdict -TokenKind 'OAuthUser' -HasAdmin $false -HasWrite $true + $v.Pass | Should -BeFalse + $v.Findings -join ' ' | Should -Match 'session|interactive|own account' + } + + It 'fails a classic PAT (cannot be scoped per-permission)' { + (Get-IdentityVerdict -TokenKind 'ClassicPat' -HasAdmin $false -HasWrite $true).Pass | Should -BeFalse + } + + It 'fails closed on an unrecognized token format' { + (Get-IdentityVerdict -TokenKind 'Unknown' -HasAdmin $false -HasWrite $true).Pass | Should -BeFalse + } + + It 'fails when the credential cannot push to the repository' { + $v = Get-IdentityVerdict -TokenKind 'Installation' -HasAdmin $false -HasWrite $false + $v.Pass | Should -BeFalse + $v.Findings -join ' ' | Should -Match 'write|push' + } +} + +Describe 'Invoke-IdentityCheck' -Tag 'Fast' { + It 'returns non-zero when gh is not installed' { + Mock Test-GhCli { $false } + Invoke-IdentityCheck 2>$null | Should -Be 1 + } + + It 'returns 0 for a least-privilege bot credential' { + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if ($args -contains 'token') { return 'ghs_16C7e42F292c6912E7710c838347Ae178B4a' } # gitleaks:allow + if (($args -join ' ') -match 'permissions\.admin') { $global:LASTEXITCODE = 0; return 'false' } + if (($args -join ' ') -match 'permissions\.push') { $global:LASTEXITCODE = 0; return 'true' } + $global:LASTEXITCODE = 0 + } + Invoke-IdentityCheck | Should -Be 0 + } + + It 'returns non-zero when the credential is the owner''s interactive session' { + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if ($args -contains 'token') { return 'gho_16C7e42F292c6912E7710c838347Ae178B4a' } # gitleaks:allow + if (($args -join ' ') -match 'permissions\.admin') { $global:LASTEXITCODE = 0; return 'true' } + if (($args -join ' ') -match 'permissions\.push') { $global:LASTEXITCODE = 0; return 'true' } + $global:LASTEXITCODE = 0 + } + Invoke-IdentityCheck 2>$null | Should -Be 1 + } + + It 'returns non-zero when token kind is allowed but push/write is missing' { + Mock Test-GhCli { $true } + Mock gh { + if ($args -contains 'view') { return 'owner/repo' } + if ($args -contains 'token') { return 'github_pat_11ABC123_abcdef' } # gitleaks:allow + if (($args -join ' ') -match 'permissions\.admin') { $global:LASTEXITCODE = 0; return 'false' } + if (($args -join ' ') -match 'permissions\.push') { $global:LASTEXITCODE = 0; return 'false' } + $global:LASTEXITCODE = 0 + } + Invoke-IdentityCheck 2>$null | Should -Be 1 + } +} diff --git a/setup/version b/setup/version new file mode 100644 index 0000000..0d91a54 --- /dev/null +++ b/setup/version @@ -0,0 +1 @@ +0.3.0