QA hardening: 16 bug fixes + measured coverage/mutation/fuzz gates

.github/workflows/ci.yml

@@ -20,5 +20,16 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
       - run: pnpm typecheck
-      - run: pnpm test
+
+      # Coverage gate — fails if line/branch/function coverage regresses below the thresholds in
+      # vitest.config.ts (lines ≥90, branches ≥85). The committed seeded fuzz (fuzz.test.ts) runs here too.
+      - run: pnpm test:coverage
+
       - run: pnpm build
+
+      # Schema gate — the published JSON Schema must stay in sync with the Zod source (catches the
+      # class of bug where the runtime and the contract drift apart).
+      - name: JSON Schema is in sync with the Zod source
+        run: |
+          pnpm gen:schema
+          git diff --exit-code packages/core/schema

.github/workflows/mutation.yml

@@ -0,0 +1,30 @@
+name: Mutation
+
+# Mutation testing is slow, so it runs weekly (and on demand) rather than on every PR.
+# It proves the TESTS are strong — a high pass rate with a low mutation score means the suite is
+# decorative. Scoped to the most security-critical module (the contract validator); expand `mutate`
+# in stryker.config.json to cover more modules as the budget allows.
+on:
+  schedule:
+    - cron: "0 4 * * 1" # Mondays 04:00 UTC
+  workflow_dispatch:
+
+jobs:
+  mutation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      # Fails the job if the mutation score drops below `thresholds.break` (80) in stryker.config.json.
+      - run: pnpm exec stryker run
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: mutation-report
+          path: qa/mutation-report.json

.gitignore

@@ -13,3 +13,9 @@ coverage/
 # VitePress
 docs/.vitepress/dist/
 docs/.vitepress/cache/
+
+# QA tooling artifacts (regenerated)
+coverage/
+.stryker-tmp/
+qa/.stryker-tmp/
+qa/mutation-report.json

package.json

@@ -20,6 +20,8 @@
     "docs:preview": "vitepress preview docs"
   },
   "devDependencies": {
+    "@stryker-mutator/core": "^9.6.1",
+    "@stryker-mutator/vitest-runner": "^9.6.1",
     "@types/node": "^22.10.2",
     "@vitest/coverage-v8": "^2.1.8",
     "tsup": "^8.3.5",

packages/cli/src/commands/run.ts

@@ -61,8 +61,13 @@ export async function runCommand(argv: string[], deps: Partial<CommandDeps> = {}
   if (result.missingSecrets.length > 0) {
     d.stderr(`Warning: unresolved secrets (set as env vars): ${result.missingSecrets.join(", ")}\n`);
   }
-  if (result.results.length === 0) {
-    d.stderr(`Warning: no .tspec.yaml requests found under "${target}".\n`);
+  // Finding ZERO requests is a failure, not a pass: `run` is a CI gate, and a green build when no
+  // request executed silently masks a misconfigured path, uncommitted files, or a bad glob — the
+  // worst kind of false-positive for a gate. (`[].every()` is `true`, so `result.ok` alone says
+  // "pass" here.) Industry test runners (jest, pytest, go test) fail on "no tests found" too.
+  const noRequests = result.results.length === 0;
+  if (noRequests) {
+    d.stderr(`Error: no .tspec.yaml requests found under "${target}".\n`);
   }
 
   const reporter = values.reporter ?? (values.json ? "json" : "human");
@@ -73,5 +78,5 @@ export async function runCommand(argv: string[], deps: Partial<CommandDeps> = {}
         ? formatJson(result)
         : formatHuman(result, d.cwd);
   emit(d, text, values.output);
-  return result.ok ? 0 : 1;
+  return result.ok && !noRequests ? 0 : 1;
 }

packages/cli/test/cli-branches.test.ts

@@ -0,0 +1,107 @@
+import { createServer } from "node:http";
+import { mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { contractCommand } from "../src/commands/contract";
+import { coverageCommand } from "../src/commands/coverage";
+import { driftCommand } from "../src/commands/drift";
+import { genCommand } from "../src/commands/gen";
+import { importCommand } from "../src/commands/import";
+import { mockCommand } from "../src/commands/mock";
+
+const repoRoot = resolve(import.meta.dirname, "..", "..", "..");
+const cap = () => { let o = "", e = ""; return { stdout: (s: string) => (o += s), stderr: (s: string) => (e += s), get out() { return o; }, get err() { return e; } }; };
+const okFetch = (body: unknown, status = 200): typeof fetch => (async () => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } })) as typeof fetch;
+const petstore = resolve(repoRoot, "examples", "petstore");
+
+describe("CLI branch coverage", () => {
+  // ---- coverage ----
+  it("coverage: bad flag → 2, no spec → 2", async () => {
+    expect(await coverageCommand(["--nope"], { cwd: repoRoot, ...cap() })).toBe(2);
+    expect(await coverageCommand(["examples/petstore"], { cwd: repoRoot, ...cap() })).toBe(2);
+  });
+  it("coverage: bad spec path → 1", async () => {
+    expect(await coverageCommand(["--spec", "nope.yaml", "examples/petstore"], { cwd: repoRoot, ...cap() })).toBe(1);
+  });
+  it("coverage: --min above actual fails (1), --json emits report", async () => {
+    const c = cap();
+    expect(await coverageCommand(["--spec", "openapi.yaml", "--min", "100", "--json"], { cwd: petstore, stdout: c.stdout, stderr: c.stderr })).toBe(1);
+    expect(JSON.parse(c.out)).toHaveProperty("percent");
+  });
+  it("coverage: --min 0 passes (0)", async () => {
+    expect(await coverageCommand(["--spec", "openapi.yaml", "--min", "0"], { cwd: petstore, ...cap() })).toBe(0);
+  });
+
+  // ---- gen ----
+  it("gen: bad flag → 2, spec not found → 1", async () => {
+    expect(await genCommand(["--nope"], { cwd: repoRoot, ...cap() })).toBe(2);
+    expect(await genCommand(["--spec", "nope.yaml", "--out", "/tmp/x"], { cwd: repoRoot, ...cap() })).toBe(1);
+  });
+  it("gen: custom --base-url-var + skips unsupported methods", async () => {
+    const out = mkdtempSync(resolve(tmpdir(), "gen-"));
+    try {
+      const c = cap();
+      // a spec with a TRACE op (unsupported) to exercise the skipped branch
+      const dir = mkdtempSync(resolve(tmpdir(), "spec-"));
+      const { writeFileSync } = await import("node:fs");
+      writeFileSync(resolve(dir, "s.yaml"), 'openapi: 3.0.3\ninfo: { title: T, version: "1" }\npaths:\n  /a: { get: { responses: { "200": {} } }, trace: { responses: { "200": {} } } }\n');
+      const code = await genCommand(["--spec", resolve(dir, "s.yaml"), "--out", out, "--base-url-var", "API"], { cwd: repoRoot, stdout: c.stdout, stderr: c.stderr });
+      expect(code).toBe(0);
+      expect(c.out).toMatch(/Generated 1 request/);
+      expect(c.err).toMatch(/skipped \(unsupported method\)/);
+      rmSync(dir, { recursive: true, force: true });
+    } finally { rmSync(out, { recursive: true, force: true }); }
+  });
+
+  // ---- drift (incl. --live + --json) ----
+  it("drift: bad flag → 2, no spec → 2", async () => {
+    expect(await driftCommand(["--nope"], { cwd: repoRoot, ...cap() })).toBe(2);
+    expect(await driftCommand(["examples/petstore"], { cwd: repoRoot, ...cap() })).toBe(2);
+  });
+  it("drift: --json emits report; bad spec → 1", async () => {
+    const c = cap();
+    await driftCommand(["--spec", "openapi.yaml", "--json"], { cwd: petstore, stdout: c.stdout, stderr: c.stderr });
+    expect(JSON.parse(c.out)).toHaveProperty("added");
+    expect(await driftCommand(["--spec", "nope.yaml"], { cwd: petstore, ...cap() })).toBe(1);
+  });
+  it("drift: --live probes a running API", async () => {
+    const server = createServer((q, res) => { res.writeHead(q.url?.startsWith("/pets") ? 200 : 404); res.end(); });
+    await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
+    const port = (server.address() as { port: number }).port;
+    try {
+      const c = cap();
+      await driftCommand(["--spec", "openapi.yaml", "--live", `http://127.0.0.1:${port}`, "--json", "--timeout", "2000"], { cwd: petstore, stdout: c.stdout, stderr: c.stderr });
+      expect(JSON.parse(c.out)).toHaveProperty("liveMissing");
+    } finally { await new Promise((r) => server.close(() => r(undefined))); }
+  });
+
+  // ---- mock ----
+  it("mock: bad flag → 2, no spec → 2, spec not found → 1", async () => {
+    expect(await mockCommand(["--nope"], { cwd: repoRoot, ...cap(), block: false })).toBe(2);
+    expect(await mockCommand([], { cwd: repoRoot, ...cap(), block: false })).toBe(2);
+    expect(await mockCommand(["--spec", "nope.yaml"], { cwd: repoRoot, ...cap(), block: false })).toBe(1);
+  });
+  it("mock: starts with --validate + --delay", async () => {
+    const c = cap();
+    let handle: { url: string; routes: number; close: () => Promise<void> } | undefined;
+    const code = await mockCommand(["--spec", "openapi.yaml", "--port", "0", "--validate", "--delay", "1"], { cwd: petstore, stdout: c.stdout, stderr: c.stderr, block: false, onReady: (h) => { handle = h; } });
+    try { expect(code).toBe(0); expect(handle?.routes).toBeGreaterThan(0); expect(c.out).toMatch(/Mock server on/); } finally { await handle?.close(); }
+  });
+
+  // ---- import ----
+  it("import: bad source → 2, missing input → 2, not found → 1", async () => {
+    expect(await importCommand(["nope", "x"], { cwd: repoRoot, ...cap() })).toBe(2);
+    expect(await importCommand(["postman"], { cwd: repoRoot, ...cap() })).toBe(2);
+    expect(await importCommand(["postman", "nope.json"], { cwd: repoRoot, ...cap() })).toBe(1);
+  });
+
+  // ---- contract ----
+  it("contract: no spec → 2, --json with injected fetch → 0/1, bad spec → 1", async () => {
+    expect(await contractCommand([], { cwd: petstore, ...cap() })).toBe(2);
+    const c = cap();
+    await contractCommand(["--spec", "openapi.yaml", "--env", "local", "--json"], { cwd: petstore, stdout: c.stdout, stderr: c.stderr, fetch: okFetch({ id: 1, name: "Rex" }), processEnv: { token: "secret" }, now: () => 0 });
+    expect(JSON.parse(c.out)).toHaveProperty("conformed");
+    expect(await contractCommand(["--spec", "nope.yaml"], { cwd: petstore, ...cap() })).toBe(1);
+  });
+});

packages/cli/test/output-branches.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest";
+import { formatContract, formatCoverage, formatDrift, formatHuman, formatJson, formatJunit } from "../src/output";
+
+const run = (results: unknown[], passed = 0, failed = 0): never =>
+  ({ results, passed, failed, ok: failed === 0, missingSecrets: [] }) as never;
+
+describe("output formatter branch coverage", () => {
+  it("formatHuman covers pass, fail-with-error, fail-with-assertions, no-response", () => {
+    const r = run([
+      { name: "A", filePath: "/x/a.tspec.yaml", request: { method: "GET", url: "u" }, ok: true, response: { status: 200, statusText: "OK", durationMs: 5, bodyText: "{}", headers: {} }, assertions: [] },
+      { name: "B", request: { method: "GET", url: "u" }, ok: false, error: "boom", assertions: [] },
+      { name: "C", filePath: "/x/c.tspec.yaml", request: { method: "GET", url: "u" }, ok: false, response: { status: 500, statusText: "ERR", durationMs: 9, bodyText: "", headers: {} }, assertions: [{ type: "status", ok: false, message: "status 500 fails == 200" }] },
+    ], 1, 2);
+    const out = formatHuman(r, "/x");
+    expect(out).toMatch(/✓ PASS  A/);
+    expect(out).toMatch(/error: boom/);
+    expect(out).toMatch(/✗ status 500 fails/);
+    expect(out).toMatch(/1 passed, 2 failed, 3 total/);
+  });
+
+  it("formatJson is parseable", () => {
+    expect(JSON.parse(formatJson(run([], 0, 0)))).toHaveProperty("ok");
+  });
+
+  it("formatJunit covers ok, failure(error+assertions), and missing response time", () => {
+    const xml = formatJunit(run([
+      { name: "ok<&>", filePath: "/x/a.tspec.yaml", request: { method: "GET", url: "u" }, ok: true, response: { status: 200, statusText: "OK", durationMs: 1500, bodyText: "", headers: {} }, assertions: [] },
+      { name: "bad", request: { method: "GET", url: "u" }, ok: false, error: "err<x>", assertions: [{ type: "status", ok: false, message: "msg & <b>" }] },
+    ], 1, 1), "/x");
+    expect(xml).toMatch(/<testsuites tests="2" failures="1">/);
+    expect(xml).toMatch(/name="ok&lt;&amp;&gt;"/); // escaped name
+    expect(xml).toMatch(/time="1.500"/);
+    expect(xml).toMatch(/<failure message="err&lt;x&gt;; msg &amp; &lt;b&gt;"\/>/); // escaped, joined
+  });
+
+  it("formatDrift covers added/removed/changed/liveMissing AND the clean case", () => {
+    const drifted = formatDrift({ specOperations: 5, collectionOperations: 4, added: ["GET /a"], removed: ["GET /b"], changed: ["GET /c: x"], liveMissing: ["GET /d"], ok: false });
+    expect(drifted).toMatch(/Untracked in collection \(1\)/);
+    expect(drifted).toMatch(/Stale — not in the spec \(1\)/);
+    expect(drifted).toMatch(/Changed \(1\)/);
+    expect(drifted).toMatch(/Missing from live API \(1\)/);
+    expect(drifted).toMatch(/Drift detected:.*1 missing live/);
+    const clean = formatDrift({ specOperations: 1, collectionOperations: 1, added: [], removed: [], changed: [], ok: true });
+    expect(clean).toMatch(/No drift/);
+  });
+
+  it("formatContract covers conformed/violations/skipped/untested and both verdicts", () => {
+    const bad = formatContract({ specOperations: 4, conformed: ["GET /a"], violations: [{ op: "GET /b", status: 500, message: "x" }], skipped: [{ op: "GET /c", message: "no schema" }], untested: ["GET /d"], ok: false });
+    expect(bad).toMatch(/Violations \(1\)/);
+    expect(bad).toMatch(/Skipped/);
+    expect(bad).toMatch(/Untested/);
+    expect(bad).toMatch(/Contract violations: 1/);
+    const good = formatContract({ specOperations: 1, conformed: ["GET /a"], violations: [], skipped: [], untested: [], ok: true });
+    expect(good).toMatch(/All 1 tested operation\(s\) conform/);
+  });
+
+  it("formatCoverage covers uncovered list and the all-covered case", () => {
+    expect(formatCoverage({ total: 2, covered: ["a"], uncovered: ["GET /b"], percent: 50, ok: false })).toMatch(/Uncovered \(1\)/);
+    expect(formatCoverage({ total: 1, covered: ["a"], uncovered: [], percent: 100, ok: true })).toMatch(/Coverage: 100%/);
+  });
+});

packages/cli/test/run.test.ts

@@ -130,12 +130,14 @@ describe("truspec run", () => {
     expect(cap.out).toMatch(/1 passed/);
   });
 
-  it("warns and exits 0 when no requests are found", async () => {
+  it("exits 1 (fails the CI gate) when no requests are found", async () => {
+    // A green build when zero requests executed silently masks a misconfigured path / uncommitted
+    // files / bad glob — the worst false-positive for a gate. `run` must fail loudly, not pass.
     const dir = mkdtempSync(join(tmpdir(), "truspec-empty-"));
     try {
       const cap = capture();
       const code = await runCommand([dir], { cwd: repoRoot, stdout: cap.stdout, stderr: cap.stderr });
-      expect(code).toBe(0);
+      expect(code).toBe(1);
       expect(cap.err).toMatch(/no .tspec.yaml requests found/i);
     } finally {
       rmSync(dir, { recursive: true, force: true });

packages/cli/test/serve.test.ts

@@ -0,0 +1,54 @@
+import { resolve } from "node:path";
+import { describe, expect, it } from "vitest";
+import { serveCommand } from "../src/commands/serve";
+
+const repoRoot = resolve(import.meta.dirname, "..", "..", "..");
+function capture() {
+  let out = "", err = "";
+  return { stdout: (s: string) => (out += s), stderr: (s: string) => (err += s), get out() { return out; }, get err() { return err; } };
+}
+
+describe("truspec serve", () => {
+  it("starts the web server, prints the URL, and is closable (block:false)", async () => {
+    const cap = capture();
+    let handle: { url: string; dir: string; close: () => Promise<void> } | undefined;
+    const code = await serveCommand(["--dir", "examples/petstore", "--port", "0"], {
+      cwd: repoRoot,
+      stdout: cap.stdout,
+      stderr: cap.stderr,
+      block: false,
+      onReady: (h) => { handle = h as typeof handle; },
+    });
+    try {
+      expect(code).toBe(0);
+      expect(cap.out).toMatch(/TruSpec web UI on http:\/\/127\.0\.0\.1:\d+/);
+      expect(cap.out).toMatch(/examples\/petstore/);
+      expect(handle?.url).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
+    } finally {
+      await handle?.close();
+    }
+  });
+
+  it("exits 2 on an unknown flag", async () => {
+    const cap = capture();
+    const code = await serveCommand(["--bogus"], { cwd: repoRoot, stdout: cap.stdout, stderr: cap.stderr, block: false });
+    expect(code).toBe(2);
+    expect(cap.err.length).toBeGreaterThan(0);
+  });
+
+  it("defaults the served dir to cwd when --dir is omitted", async () => {
+    const cap = capture();
+    let handle: { close: () => Promise<void> } | undefined;
+    const code = await serveCommand(["--port", "0"], {
+      cwd: resolve(repoRoot, "examples", "blog"),
+      stdout: cap.stdout, stderr: cap.stderr, block: false,
+      onReady: (h) => { handle = h as typeof handle; },
+    });
+    try {
+      expect(code).toBe(0);
+      expect(cap.out).toMatch(/examples\/blog/);
+    } finally {
+      await handle?.close();
+    }
+  });
+});