QA: mutation to 6 modules (≥80), continuous deep-fuzz + load/SLO harnesses (closes #14)

.github/workflows/fuzz-deep.yml

@@ -0,0 +1,36 @@
+name: Deep Fuzz
+
+# The continuous, high-volume fuzz job (the committed fuzz.test.ts is the bounded per-PR smoke).
+# Up to 1,000,000 execs OR --minutes per target, feedback-driven, with crash/hang findings written to
+# qa/fuzz/findings.jsonl. Scheduled weekly + on demand; too slow to gate every PR.
+on:
+  schedule:
+    - cron: "0 5 * * 1" # Mondays 05:00 UTC
+  workflow_dispatch:
+    inputs:
+      minutes:
+        description: Minutes per target
+        default: "20"
+
+jobs:
+  fuzz:
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm build
+      - run: node qa/fuzz/deep-fuzz.mjs --minutes ${{ github.event.inputs.minutes || '20' }}
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: fuzz-corpus-and-findings
+          path: |
+            qa/fuzz/findings.jsonl
+            qa/corpus/
+          retention-days: 14

.github/workflows/load.yml

@@ -0,0 +1,31 @@
+name: Load
+
+# Throughput / latency / memory-leak SLO check for the local servers. Environment-sensitive, so it runs
+# weekly + on demand rather than gating every PR. Hard SLOs: zero errors under load and bounded post-GC
+# RSS growth (leak detection); latency/throughput are gated against generous, runner-tolerant caps.
+on:
+  schedule:
+    - cron: "0 6 * * 1" # Mondays 06:00 UTC
+  workflow_dispatch:
+    inputs:
+      seconds:
+        description: Soak duration (seconds)
+        default: "30"
+
+jobs:
+  load:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm build
+      - name: Mock server load + leak SLO
+        run: node --expose-gc qa/load/load-test.mjs --target mock --seconds ${{ github.event.inputs.seconds || '30' }} --concurrency 64 --rps-floor 200 --p95-cap-ms 250
+      - name: Web server load + leak SLO (static route)
+        run: node --expose-gc qa/load/load-test.mjs --target web --seconds ${{ github.event.inputs.seconds || '30' }} --concurrency 32 --rps-floor 100 --p95-cap-ms 500

.gitignore

@@ -23,3 +23,7 @@ qa/mutation-report.json
 # Playwright e2e artifacts
 test-results/
 playwright-report/
+
+# Transient QA outputs (regenerated)
+qa/fuzz/findings.jsonl
+qa/mut-*.json

package.json

@@ -18,7 +18,10 @@
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
     "docs:preview": "vitepress preview docs",
-    "test:e2e": "playwright test -c e2e/playwright.config.ts"
+    "test:e2e": "playwright test -c e2e/playwright.config.ts",
+    "fuzz:deep": "node qa/fuzz/deep-fuzz.mjs",
+    "load": "node --expose-gc qa/load/load-test.mjs",
+    "mutation": "stryker run"
   },
   "devDependencies": {
     "@axe-core/playwright": "^4.11.3",

packages/core/test/drift-mutation.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+import { computeDrift, refMatchesOp } from "../src/spec/drift";
+import type { SpecOperation } from "../src/spec/openapi";
+import type { CollectionOp } from "../src/spec/collection";
+
+const op = (o: Partial<SpecOperation>): SpecOperation => ({ key: "GET /x", method: "GET", path: "/x", operationId: undefined, parameters: [], requestBodyRequired: false, ...o }) as SpecOperation;
+const col = (o: Partial<CollectionOp> & { ref: CollectionOp["ref"] }): CollectionOp => ({ name: "r", queryParams: [], hasBody: false, ...o }) as CollectionOp;
+
+describe("drift — exact matching & diffing (mutation kills)", () => {
+  it("refMatchesOp: operationId match wins; mismatch is false", () => {
+    expect(refMatchesOp({ operationId: "getPet" }, op({ operationId: "getPet" }))).toBe(true);
+    expect(refMatchesOp({ operationId: "getPet" }, op({ operationId: "other" }))).toBe(false);
+    // when ref has no operationId, fall through to operation-key matching
+    expect(refMatchesOp({ operation: "GET /x" }, op({ key: "GET /x" }))).toBe(true);
+    expect(refMatchesOp({ operation: "GET /y" }, op({ key: "GET /x" }))).toBe(false);
+    // neither field → false
+    expect(refMatchesOp({}, op({ key: "GET /x" }))).toBe(false);
+  });
+
+  it("normalizeKey (via refMatchesOp): trims, uppercases method, collapses inner whitespace", () => {
+    expect(refMatchesOp({ operation: "  get   /a/b  " }, op({ key: "GET /a/b" }))).toBe(true); // trim + multi-space + upper
+    expect(refMatchesOp({ operation: "get /a/b" }, op({ key: "GET /a/b" }))).toBe(true);
+    // a single token (no path) is returned trimmed as-is, so it won't match a "METHOD path" key
+    expect(refMatchesOp({ operation: "GET" }, op({ key: "GET" }))).toBe(true);
+    expect(refMatchesOp({ operation: "GET" }, op({ key: "GET /a" }))).toBe(false);
+    // a 2-token op normalizes (boundary parts.length === 2)
+    expect(refMatchesOp({ operation: "post /a" }, op({ key: "POST /a" }))).toBe(true);
+  });
+
+  it("computeDrift: removed/added/changed are each detected and SORTED", () => {
+    const ops = [op({ key: "GET /b" }), op({ key: "GET /a" })];
+    const cols = [col({ ref: { operation: "GET /gone-2" }, name: "g2" }), col({ ref: { operation: "GET /gone-1" }, name: "g1" })];
+    const r = computeDrift(ops, cols);
+    expect(r.added).toEqual(["GET /a", "GET /b"]); // sorted (input was b,a)
+    expect(r.removed).toEqual(["GET /gone-1", "GET /gone-2"]); // sorted
+    expect(r.specOperations).toBe(2);
+    expect(r.collectionOperations).toBe(2);
+    expect(r.ok).toBe(false);
+  });
+
+  it("computeDrift: a referenced op is not 'added'; ok is true only when all three are empty", () => {
+    const r = computeDrift([op({ key: "GET /a" })], [col({ ref: { operation: "GET /a" } })]);
+    expect(r.added).toEqual([]);
+    expect(r.removed).toEqual([]);
+    expect(r.changed).toEqual([]);
+    expect(r.ok).toBe(true);
+  });
+
+  it("computeDrift: changed = required query missing OR required body missing; satisfied/optional don't drift", () => {
+    const specOp = op({ key: "GET /a", parameters: [{ name: "q", in: "query", required: true }], requestBodyRequired: true });
+    const missing = computeDrift([specOp], [col({ ref: { operation: "GET /a" }, queryParams: [], hasBody: false })]);
+    expect(missing.changed.sort()).toEqual(["GET /a: missing required query param 'q'", "GET /a: missing required request body"].sort());
+    const ok = computeDrift([specOp], [col({ ref: { operation: "GET /a" }, queryParams: ["q"], hasBody: true })]);
+    expect(ok.changed).toEqual([]);
+    // optional query missing → not changed
+    const optional = computeDrift([op({ key: "GET /a", parameters: [{ name: "q", in: "query", required: false }] })], [col({ ref: { operation: "GET /a" } })]);
+    expect(optional.changed).toEqual([]);
+  });
+});

packages/core/test/mock-engine-mutation.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, it } from "vitest";
+import { buildRoutes, createMockResponder, generateExample } from "../src/mock/engine";
+
+// Mutation-killing tests for mock/engine.ts: assert EXACT headers, status, bodies, route specificity,
+// validate-mode 400s, and regex matching semantics so string/conditional/object mutants can't survive.
+const spec = (paths: Record<string, unknown>) => `openapi: 3.0.3\ninfo: { title: T, version: "1" }\npaths: ${JSON.stringify(paths)}\n`;
+
+describe("mock engine — exact behavior (mutation kills)", () => {
+  it("a body response carries exactly content-type: application/json", () => {
+    const r = createMockResponder(spec({ "/x": { get: { responses: { "200": { content: { "application/json": { example: { a: 1 } } } } } } } }));
+    const res = r.respond("GET", "/x")!;
+    expect(res.status).toBe(200);
+    expect(res.headers["content-type"]).toBe("application/json");
+    expect(res.body).toBe('{"a":1}');
+  });
+
+  it("no JSON content → empty body, no content-type header", () => {
+    const res = createMockResponder(spec({ "/x": { get: { responses: { "204": { description: "no content" } } } } })).respond("GET", "/x")!;
+    expect(res.body).toBe("");
+    expect(res.headers["content-type"]).toBeUndefined();
+  });
+
+  it("route specificity: a static segment beats a parameter regardless of declaration order", () => {
+    const r = createMockResponder(spec({
+      "/users/{id}": { get: { responses: { "200": { content: { "application/json": { example: "param" } } } } } },
+      "/users/me": { get: { responses: { "200": { content: { "application/json": { example: "static" } } } } } },
+    }));
+    expect(JSON.parse(r.respond("GET", "/users/me")!.body)).toBe("static");
+    expect(JSON.parse(r.respond("GET", "/users/123")!.body)).toBe("param");
+  });
+
+  it("pathToRegex: a param matches one segment but not across '/', and literals match exactly", () => {
+    const route = buildRoutes({ paths: { "/pets/{id}": { get: { responses: {} } } } })[0]!;
+    expect(route.regex.test("/pets/123")).toBe(true);
+    expect(route.regex.test("/pets/123/")).toBe(true); // trailing slash allowed
+    expect(route.regex.test("/pets/123/x")).toBe(false); // not across a slash
+    expect(route.regex.test("/pets/")).toBe(false); // param requires a value
+    expect(route.regex.test("/petsX/123")).toBe(false); // literal is exact
+  });
+
+  it("validate mode: missing required query → 400 with the exact missing list; satisfied → 200", () => {
+    const s = spec({ "/s": { get: { parameters: [{ name: "q", in: "query", required: true }], responses: { "200": { content: { "application/json": { example: {} } } } } } } });
+    const r = createMockResponder(s, { validate: true });
+    const bad = r.respond("GET", "/s", { query: {} })!;
+    expect(bad.status).toBe(400);
+    expect(bad.headers["content-type"]).toBe("application/json");
+    expect(JSON.parse(bad.body)).toEqual({ error: "Request does not satisfy the spec", missing: ["query:q"] });
+    expect(r.respond("GET", "/s", { query: { q: "1" } })!.status).toBe(200);
+  });
+
+  it("validate mode: a required request body that is absent → 400 missing:['body']", () => {
+    const s = spec({ "/s": { post: { requestBody: { required: true, content: { "application/json": {} } }, responses: { "200": { content: { "application/json": { example: {} } } } } } } });
+    const r = createMockResponder(s, { validate: true });
+    expect(JSON.parse(r.respond("POST", "/s", { hasBody: false })!.body).missing).toEqual(["body"]);
+    expect(r.respond("POST", "/s", { hasBody: true })!.status).toBe(200);
+  });
+
+  it("response selection: lowest 2xx wins; else default; else first code; status clamped to 200-599", () => {
+    expect(createMockResponder(spec({ "/x": { get: { responses: { "201": { content: { "application/json": { example: "a" } } }, "200": { content: { "application/json": { example: "b" } } } } } } })).respond("GET", "/x")!.status).toBe(200);
+    expect(createMockResponder(spec({ "/x": { get: { responses: { default: { content: { "application/json": { example: "d" } } } } } } })).respond("GET", "/x")!.status).toBe(200);
+    expect(createMockResponder(spec({ "/x": { get: { responses: { "404": { content: { "application/json": { example: "e" } } } } } } })).respond("GET", "/x")!.status).toBe(404);
+    expect(createMockResponder(spec({ "/x": { get: { responses: { "20000": { content: { "application/json": { example: "f" } } } } } } })).respond("GET", "/x")!.status).toBe(200);
+    expect(createMockResponder(spec({ "/x": { get: { responses: { "100": { content: { "application/json": { example: "g" } } } } } } })).respond("GET", "/x")!.status).toBe(200);
+  });
+
+  it("examples-map first value is used; an empty enum does not short-circuit; allOf skips non-object parts", () => {
+    const ex = createMockResponder(spec({ "/x": { get: { responses: { "200": { content: { "application/json": { examples: { a: { value: { hi: 9 } } } } } } } } } })).respond("GET", "/x")!;
+    expect(JSON.parse(ex.body)).toEqual({ hi: 9 });
+    expect(generateExample({ enum: [], type: "string" }, {})).toBe("string"); // empty enum ignored
+    expect(generateExample({ allOf: [{ type: "string" }, { properties: { a: { type: "integer" } } }] }, {})).toEqual({ a: 0 }); // string part skipped
+  });
+
+  it("generateExample stops at depth 6 (returns null at the bottom of a deep chain)", () => {
+    let schema: Record<string, unknown> = { type: "string" };
+    for (let i = 0; i < 8; i++) schema = { type: "object", properties: { n: schema } };
+    // 8 levels deep > cap 6 → the deepest value is cut to null, not "string"
+    let v = generateExample(schema, {}) as Record<string, unknown>;
+    let depth = 0; while (v && typeof v === "object" && "n" in v) { v = v.n as Record<string, unknown>; depth++; }
+    expect(depth).toBeLessThanOrEqual(7);
+  });
+});

qa/COVERAGE.json

@@ -74,14 +74,18 @@
     ]
   },
   "mutation": {
-    "tool": "Stryker 9.6.1 + vitest-runner",
-    "scope": "packages/core/src/spec/validate-response.ts (most security-critical; full-repo = scheduled CI job per stryker.config _mutateNote)",
-    "score": 85.4,
-    "killed": 300,
-    "survived": 52,
-    "timeout": 10,
-    "noCoverage": 1,
-    "survivorsJustified": "~18 equivalent (memoization L75-83 cache identity, depth-arithmetic that does not change observable output, jsonType default L27); remainder = low-value boundary + unasserted-message conditionals near the depth>100 cutoff"
+    "tool": "Stryker 9.6.1 + vitest-runner (vitest.mutation.config.ts excludes the slow fuzz smoke)",
+    "thresholdBreak": 80,
+    "modules": {
+      "spec/validate-response.ts": 85.4,
+      "runner/resolve.ts": 84.4,
+      "mock/engine.ts": 82.2,
+      "spec/scaffold.ts": 81.5,
+      "runner/capture.ts": 80.4,
+      "spec/drift.ts": 92.3
+    },
+    "allModulesMeetThreshold": true,
+    "note": "engine.ts 77.4->82.2 and drift.ts 75.6->92.3 after adding mock-engine-mutation + drift-mutation killing tests; weekly mutation.yml gates the full scope at break=80"
   },
   "fuzz": {
     "_comment": "Seeded property fuzz from campaigns 1-10; corpus + seeds persisted under qa/. Counts are cumulative across prior campaigns.",
@@ -126,13 +130,28 @@
   },
   "nonFunctional": {
     "loadSoak": "mock server 30k req @ ~791rps, heap stable (no leak, forced-GC verified); web 10k req graceful timeouts under 64-conc; runner defeats gzip-bomb/slow-loris (campaign 5).",
-    "slo": "no formal SLO defined for a local-first CLI; throughput-to-a-target-RPS NOT measured (gap)."
+    "slo": "DEFINED + gated: err-rate 0 (hard), post-GC heap leak (hard), throughput floor, p95 latency cap. See qa/load + .github/workflows/load.yml."
   },
   "e2e": {
     "tool": "Playwright 1.61 + axe-core/playwright",
     "tests": 13,
     "scope": "web UI: XSS, CSRF (cross-origin+cross-port), X-Frame-Options, editor save/keyboard(BUG-O), traversal, double-click, axe a11y(BUG-P)+contrast (main/editor/light)",
     "ci": ".github/workflows/e2e.yml (playwright install --with-deps)",
     "status": "all green"
+  },
+  "deepFuzz": {
+    "tool": "qa/fuzz/deep-fuzz.mjs (feedback-driven, corpus-growing)",
+    "targets": 7,
+    "budget": "1,000,000 execs OR --minutes per target",
+    "ci": ".github/workflows/fuzz-deep.yml (weekly + dispatch)",
+    "lastSmoke": "140000 execs, 0 findings",
+    "corpus": "qa/corpus/*.corpus.json"
+  },
+  "load": {
+    "tool": "qa/load/load-test.mjs (custom concurrent driver)",
+    "ci": ".github/workflows/load.yml (weekly + dispatch)",
+    "hardSLOs": "error-rate 0 + post-GC heap growth bounded (real leak signal; RSS ignored as high-water-mark)",
+    "mock": "~2400 rps, p95 33-43ms, heap +0.6MB (no leak)",
+    "web": "~740 rps static, p95 68ms, heap flat (no leak)"
   }
 }

qa/QA_LOG.md

@@ -109,3 +109,34 @@ state), `qa/TRIED.jsonl` (28 executed attacks — never repeated), `qa/SEEDS.jso
   vitest timeout. Suite now green twice with no flake.
 
 Remaining Issue-#14 items (1 expand mutation, 2 continuous 1M-exec fuzz, 3 throughput-SLO load) stay open.
+
+---
+
+## Cycle v2-3 — mutation expansion, continuous fuzz, load/SLO (Issue #14, items 1–3)
+
+**Item 1 — mutation beyond the validator.** Expanded the Stryker scope from 1 to **6 modules**, all now
+≥ the 80 break threshold. Two were below and were lifted by targeted killing tests:
+- `mock/engine.ts` 77.4% → **82.2%** (`mock-engine-mutation.test.ts`: exact content-types, validate-mode
+  400 body, route specificity, regex semantics, response selection, depth cap)
+- `spec/drift.ts` 75.6% → **92.3%** (`drift-mutation.test.ts`: normalizeKey whitespace/uppercasing,
+  refMatchesOp branches, sorted added/removed/changed, ok logic, required-param drift)
+- already ≥80: validate-response 85.4, resolve 84.4, scaffold 81.5, capture 80.4.
+- Root-caused why engine mutation never finished inline: `fuzz.test.ts` (~11s smoke) was in the covering
+  set and re-ran per mutant. Added `vitest.mutation.config.ts` (excludes the fuzz smoke) → engine run
+  4m34s instead of timing out. Benefits the weekly `mutation.yml` too.
+
+**Item 2 — continuous fuzz to the 1M-exec budget.** `qa/fuzz/deep-fuzz.mjs`: feedback-driven (keeps
+inputs that hit a new output signature — AFL-style corpus growth), 7 targets, up to 1,000,000 execs OR
+`--minutes` per target, crash/hang findings → `qa/fuzz/findings.jsonl`, corpus persisted to `qa/corpus/`.
+Scheduled in `fuzz-deep.yml`. Smoke: **140,000 execs, 0 findings**. The fuzzer itself had two bugs I
+found and fixed: a global (not per-target) findings counter, and treating jsonpath's correct "invalid
+path" throw on a malformed author expression as a crash.
+
+**Item 3 — throughput / latency / leak SLO.** `qa/load/load-test.mjs` (custom concurrent driver) +
+`load.yml`. Hard SLOs: **error-rate 0** and **bounded post-GC heapUsed growth** — the real leak signal;
+I first measured RSS and saw it climb 58MB, then realised RSS is the high-water-mark (V8 never returns
+it) and switched to post-GC heapUsed, which is **flat (+0.6MB over 47k requests) → no leak**. Latency
+gated on **p95** (stable) not p99 (a single GC spike blew p99 to 1469ms). Results: mock ~2400 rps /
+p95 33–43ms, web ~740 rps static / p95 68ms, both leak-free.
+
+### Outcome — Issue #14 fully closed (items 1–5). 326 tests green, typecheck 7/7, all SLOs met.

qa/TRIED.jsonl

@@ -32,3 +32,9 @@
 {"id":"E2E-axe","category":"ui/a11y","target":"web e2e (axe-core)","input":"main/editor/light-theme scans","result":"PASS 0 wcag2a/aa violations (BUG-P + contrast)","cycle":"v2-2"}
 {"id":"FIX-flaky-fuzz","category":"regression/flake","target":"core/test/fuzz.test.ts INV-4","input":"slowest<100ms timing under parallel load","result":"FIXED: structural ReDoS check (no adjacent [^/]+[^/]+) instead of wall-clock","cycle":"v2-2"}
 {"id":"FIX-contrast","category":"ui/a11y","target":"web/src/styles.css","input":"--dimmer ~2.6:1, light run-button 4:1","result":"FIXED: raised --dim/--dimmer both themes + light run-button white text; axe 0 contrast","cycle":"v2-2"}
+{"id":"MUT-engine","category":"mutation","target":"mock/engine.ts","input":"292 mutants","result":"77.4->82.2% via mock-engine-mutation.test.ts","cycle":"v2-3"}
+{"id":"MUT-drift","category":"mutation","target":"spec/drift.ts","input":"78 mutants","result":"75.6->92.3% via drift-mutation.test.ts","cycle":"v2-3"}
+{"id":"FUZZ-deep","category":"fuzz/continuous","target":"7 engine targets","input":"140k execs smoke (1M budget job)","result":"0 findings; corpus persisted","cycle":"v2-3"}
+{"id":"FIX-fuzz-harness","category":"regression/harness","target":"deep-fuzz.mjs","input":"global findings counter + jsonpath false-positive","result":"FIXED both","cycle":"v2-3"}
+{"id":"LOAD-slo","category":"nonfunctional/load","target":"mock+web servers","input":"64/32-conc soak","result":"0 err, post-GC heap flat (no leak), p95 within SLO","cycle":"v2-3"}
+{"id":"FIX-rss-metric","category":"methodology","target":"load-test.mjs","input":"RSS used as leak metric (climbs by high-water-mark)","result":"FIXED: gate on post-GC heapUsed","cycle":"v2-3"}

qa/corpus/importPostman.corpus.json

@@ -0,0 +1 @@
+[{"doc":{"item":[{"name":"../../etc","request":{"method":"weird","url":"http://x/?a=1#f"}}]}}]