From a05b11b394bc3d766412bddf81b850b4fce6cfa7 Mon Sep 17 00:00:00 2001 From: Rashid Mahmood Date: Tue, 23 Jun 2026 17:14:37 +0400 Subject: [PATCH 1/2] test(e2e): Playwright + axe browser regressions in CI; fix a11y contrast & a flaky fuzz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Issue #14 items 4 (browser e2e in CI) and 5 (color contrast). - e2e/ Playwright suite (13 tests) + .github/workflows/e2e.yml (playwright install --with-deps chromium). Permanently guards the browser-only bugs that node coverage can't reach: - CSRF rejected cross-origin AND cross-port-loopback (BUG-N), X-Frame-Options + CSP (BUG-M), XSS escaped/not-executed - editor save + keyboard (Esc from path input, Ctrl+Enter — BUG-O), traversal refused, double-click re-entrancy - axe-core = 0 WCAG 2 A/AA violations on main/editor/light-theme (BUG-P) - Color contrast fixed at root (styles.css): raised --dim/--dimmer in both themes (dark --dimmer was ~2.6:1) and gave the light-theme run button white text (dark-on-dark-lime was 4:1). axe reports 0 contrast violations. - Fix flaky fuzz.test.ts INV-4: replace the load-sensitive wall-clock assertion with a deterministic structural ReDoS check (no adjacent [^/]+[^/]+); the loop still tests a 5000-char hostile path so a real ReDoS trips the test timeout. Full suite green twice (no flake), typecheck 7/7, e2e 13/13. --- .github/workflows/e2e.yml | 38 ++++++++++++++++++++ .gitignore | 4 +++ e2e/a11y.spec.ts | 35 ++++++++++++++++++ e2e/editor.spec.ts | 63 +++++++++++++++++++++++++++++++++ e2e/fixtures.ts | 43 ++++++++++++++++++++++ e2e/playwright.config.ts | 22 ++++++++++++ e2e/security.spec.ts | 53 +++++++++++++++++++++++++++ package.json | 5 ++- packages/core/test/fuzz.test.ts | 12 ++++--- packages/web/src/styles.css | 16 ++++++--- pnpm-lock.yaml | 57 +++++++++++++++++++++++++++++ qa/COVERAGE.json | 7 ++++ qa/QA_LOG.md | 22 ++++++++++++ qa/TRIED.jsonl | 6 ++++ 14 files changed, 374 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e.yml create mode 100644 e2e/a11y.spec.ts create mode 100644 e2e/editor.spec.ts create mode 100644 e2e/fixtures.ts create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/security.spec.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..183068a --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,38 @@ +name: E2E + +# Browser end-to-end regressions for the web UI — guards the bugs that only a real browser exposes +# (CSRF cross-origin/cross-port, X-Frame-Options, editor keyboard a11y, XSS escaping, axe a11y). +on: + push: + branches: [main] + pull_request: + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v6 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + + # The e2e suite serves the built web client + server, so build first. + - run: pnpm --filter @truspec/web build + + - name: Install Playwright chromium + OS deps + run: pnpm exec playwright install --with-deps chromium + + - run: pnpm test:e2e + + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: playwright-report + path: | + playwright-report/ + test-results/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index f483e59..28a2afb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ coverage/ .stryker-tmp/ qa/.stryker-tmp/ qa/mutation-report.json + +# Playwright e2e artifacts +test-results/ +playwright-report/ diff --git a/e2e/a11y.spec.ts b/e2e/a11y.spec.ts new file mode 100644 index 0000000..fa194dc --- /dev/null +++ b/e2e/a11y.spec.ts @@ -0,0 +1,35 @@ +import AxeBuilder from "@axe-core/playwright"; +import { test, expect } from "./fixtures"; + +// Guards the accessibility fixes (BUG-P: labels/heading/landmark; color-contrast raised to WCAG AA). +// Asserts ZERO axe violations on the main view, the editor, and in light theme. +const scan = (page: import("@playwright/test").Page) => new AxeBuilder({ page }).withTags(["wcag2a", "wcag2aa"]).analyze(); + +test.describe("accessibility (axe-core)", () => { + test("main view has no WCAG 2 A/AA violations", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.waitForSelector(".rname"); + const { violations } = await scan(page); + expect(violations, JSON.stringify(violations.map((v) => ({ id: v.id, nodes: v.nodes.length })))).toEqual([]); + }); + + test("editor view has no WCAG 2 A/AA violations (labeled controls)", async ({ app, page }) => { + await page.goto(`${app.url}/?new=1`, { waitUntil: "networkidle" }); + await page.waitForSelector(".editor .editor-text"); + const { violations } = await scan(page); + expect(violations, JSON.stringify(violations.map((v) => ({ id: v.id, nodes: v.nodes.length })))).toEqual([]); + }); + + test("light theme has no WCAG 2 A/AA violations (contrast)", async ({ app, page }) => { + await page.goto(`${app.url}/?theme=light`, { waitUntil: "networkidle" }); + await page.waitForSelector(".rname"); + const { violations } = await scan(page); + expect(violations, JSON.stringify(violations.map((v) => ({ id: v.id, nodes: v.nodes.length })))).toEqual([]); + }); + + test("the spec dropdown and editor fields have accessible names", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await expect(page.locator(".spec-pick select")).toHaveAttribute("aria-label", "OpenAPI spec"); + await expect(page.locator("h1")).toHaveCount(1); + }); +}); diff --git a/e2e/editor.spec.ts b/e2e/editor.spec.ts new file mode 100644 index 0000000..2376039 --- /dev/null +++ b/e2e/editor.spec.ts @@ -0,0 +1,63 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test, expect } from "./fixtures"; + +// Guards the editor interaction bugs (BUG-O keyboard scoping) and the save flow. +test.describe("editor interactions (real browser)", () => { + test("valid save writes the file and updates the sidebar", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.waitForSelector(".newreq"); + await page.click(".newreq"); + await page.fill(".editor .path-input", "folderx/created.tspec.yaml"); + await page.fill(".editor .editor-text", 'tspec: "0.1"\nname: Created Req\nmethod: POST\nurl: "http://x/y"\nassertions: []\n'); + await page.click(".editor .btn.run"); + await page.waitForTimeout(700); + expect(existsSync(join(app.dir, "folderx", "created.tspec.yaml"))).toBe(true); + await expect(page.locator(".rname", { hasText: "Created Req" })).toHaveCount(1); + }); + + test("BUG-O: Esc cancels the editor from the path input (not only the textarea)", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.click(".newreq"); + await page.click(".editor .path-input"); + await page.keyboard.press("Escape"); + await page.waitForTimeout(300); + await expect(page.locator(".editor")).toHaveCount(0); + }); + + test("BUG-O: Ctrl+Enter saves from anywhere in the editor", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.click(".newreq"); + await page.fill(".editor .path-input", "kbd.tspec.yaml"); + await page.fill(".editor .editor-text", 'tspec: "0.1"\nname: Kbd\nurl: "http://x"\nassertions: []\n'); + await page.click(".editor .editor-text"); + await page.keyboard.press("Control+Enter"); + await page.waitForTimeout(700); + expect(existsSync(join(app.dir, "kbd.tspec.yaml"))).toBe(true); + }); + + test("a traversal save path is refused and writes nothing outside the workspace", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.click(".newreq"); + await page.fill(".editor .path-input", "../../../../tmp/tspec-e2e-escape.tspec.yaml"); + await page.fill(".editor .editor-text", 'tspec: "0.1"\nname: Evil\nurl: "http://x"\nassertions: []\n'); + await page.click(".editor .btn.run"); + await page.waitForTimeout(500); + await expect(page.locator(".editor-err")).toHaveCount(1); + expect(existsSync("/tmp/tspec-e2e-escape.tspec.yaml")).toBe(false); + }); + + test("double-click save produces one uncorrupted file (re-entrancy)", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.click(".newreq"); + await page.fill(".editor .path-input", "dbl.tspec.yaml"); + await page.fill(".editor .editor-text", 'tspec: "0.1"\nname: Dbl\nurl: "http://x"\nassertions: []\n'); + const save = page.locator(".editor .btn.run"); + await save.click(); + await save.click({ timeout: 400 }).catch(() => {}); + await page.waitForTimeout(700); + const content = existsSync(join(app.dir, "dbl.tspec.yaml")) ? readFileSync(join(app.dir, "dbl.tspec.yaml"), "utf8") : ""; + expect(content).toMatch(/name: Dbl/); + expect(content.trim().endsWith("assertions: []")).toBe(true); + }); +}); diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts new file mode 100644 index 0000000..d7749e9 --- /dev/null +++ b/e2e/fixtures.ts @@ -0,0 +1,43 @@ +import { createServer } from "node:http"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test as base } from "@playwright/test"; + +const ROOT = resolve(fileURLToPath(new URL(".", import.meta.url)), ".."); + +export interface App { + url: string; + /** Absolute path to the temp workspace the server is serving (for server-side assertions). */ + dir: string; +} + +// A per-test web server over a fresh temp workspace (with a benign request, an XSS-payload-named +// request, an env, and a spec) plus a mock upstream so `run` succeeds. +export const test = base.extend<{ app: App }>({ + app: async ({}, use) => { + const { startWebServer } = (await import(`${ROOT}/packages/web/dist/server/index.js`)) as { + startWebServer: (o: { dir: string; port: number; clientDir: string }) => Promise<{ url: string; close: () => Promise }>; + }; + const mock = createServer((_q, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end('{"id":1,"name":"Rex"}'); }); + await new Promise((r) => mock.listen(0, "127.0.0.1", () => r())); + const mockPort = (mock.address() as { port: number }).port; + + const dir = mkdtempSync(join(tmpdir(), "tspec-e2e-")); + mkdirSync(join(dir, "environments"), { recursive: true }); + writeFileSync(join(dir, "environments", "local.env.yaml"), `tspec: "0.1"\nname: local\nvariables: { baseUrl: "http://127.0.0.1:${mockPort}" }\n`); + writeFileSync(join(dir, "get.tspec.yaml"), 'tspec: "0.1"\nname: Get pet\nmethod: GET\nurl: "{{baseUrl}}/pets/1"\nspec: { operation: "GET /pets/{id}" }\nassertions: [ { type: status, equals: 200 } ]\n'); + // XSS probe: a request name that WOULD execute if the UI didn't escape it. + writeFileSync(join(dir, "evil.tspec.yaml"), 'tspec: "0.1"\nname: ""\nmethod: GET\nurl: "{{baseUrl}}/x"\nassertions: []\n'); + writeFileSync(join(dir, "openapi.yaml"), 'openapi: 3.0.3\ninfo: { title: T, version: "1" }\npaths:\n /pets/{id}: { get: { operationId: getPet, responses: { "200": {} } } }\n /other: { get: { responses: { "200": {} } } }\n'); + + const web = await startWebServer({ dir, port: 0, clientDir: `${ROOT}/packages/web/dist/client` }); + await use({ url: web.url, dir }); + await web.close(); + await new Promise((r) => mock.close(() => r(undefined))); + rmSync(dir, { recursive: true, force: true }); + }, +}); + +export { expect } from "@playwright/test"; diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 0000000..9116c80 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from "@playwright/test"; + +// E2E regression suite for the web UI — guards the browser-only bugs that unit/coverage testing can't +// reach (BUG-M/N CSRF, BUG-O keyboard a11y, BUG-P screen-reader a11y, XSS escaping). +// CI installs the browser via `playwright install --with-deps chromium`. Locally you can point at an +// existing chromium with PW_EXECUTABLE_PATH (+ LD_LIBRARY_PATH if needed). +export default defineConfig({ + testDir: ".", + fullyParallel: false, + workers: 1, + forbidOnly: !!process.env.CI, + retries: 0, + timeout: 30_000, + reporter: [["list"]], + use: { + launchOptions: { + executablePath: process.env.PW_EXECUTABLE_PATH || undefined, + args: ["--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"], + }, + }, + projects: [{ name: "chromium", use: { browserName: "chromium" } }], +}); diff --git a/e2e/security.spec.ts b/e2e/security.spec.ts new file mode 100644 index 0000000..a47acd5 --- /dev/null +++ b/e2e/security.spec.ts @@ -0,0 +1,53 @@ +import { createServer } from "node:http"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { test, expect } from "./fixtures"; + +// Guards the security bugs that only a real browser exposes. +test.describe("web UI security (real browser)", () => { + test("XSS: a malicious request name is escaped, never executed (BUG-P precursor / campaign 7)", async ({ app, page }) => { + await page.goto(`${app.url}/`, { waitUntil: "networkidle" }); + await page.waitForSelector(".rname"); + // the onerror payload must NOT have fired, and no live element exists + expect(await page.evaluate(() => (window as unknown as { __xss?: boolean }).__xss)).toBeFalsy(); + expect(await page.locator("img[onerror]").count()).toBe(0); + // the payload text is present, but escaped (visible as text) + expect(await page.locator(".rname", { hasText: "onerror" }).count()).toBeGreaterThan(0); + }); + + test("clickjacking: every response carries X-Frame-Options: DENY (BUG-M)", async ({ app, request }) => { + const res = await request.get(`${app.url}/`); + expect(res.headers()["x-frame-options"]).toBe("DENY"); + expect(res.headers()["content-security-policy"]).toMatch(/frame-ancestors 'none'/); + }); + + test("CSRF: a page on ANOTHER loopback port cannot make /api/request execute (BUG-N)", async ({ app, page }) => { + // Attacker origin on a different port — the exact cross-port-loopback vector BUG-N exposed. + const attacker = createServer((_q, res) => { + res.writeHead(200, { "content-type": "text/html" }); + res.end(``); + }); + await new Promise((r) => attacker.listen(0, "127.0.0.1", () => r())); + const attackerURL = `http://127.0.0.1:${(attacker.address() as { port: number }).port}`; + try { + await page.goto(`${attackerURL}/`, { waitUntil: "load" }); + await page.waitForTimeout(1500); + // The browser fetch is CORS-blocked either way; the real assertion is SERVER-SIDE: no file written. + expect(existsSync(join(app.dir, "csrf-proof.tspec.yaml"))).toBe(false); + } finally { + await new Promise((r) => attacker.close(() => r(undefined))); + } + }); + + test("CSRF: the API rejects a cross-origin Origin and accepts same-origin (BUG-M/N)", async ({ app, request }) => { + const cross = await request.post(`${app.url}/api/run`, { headers: { origin: "http://evil.com", "content-type": "text/plain" }, data: "{}" }); + expect(cross.status()).toBe(403); + const same = await request.post(`${app.url}/api/run`, { headers: { origin: app.url, "content-type": "application/json" }, data: "{}" }); + expect(same.status()).toBe(200); + }); +}); diff --git a/package.json b/package.json index a73bd78..17e3332 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,12 @@ "gen:schema": "pnpm --filter @truspec/core build && pnpm --filter @truspec/core gen:schema", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", - "docs:preview": "vitepress preview docs" + "docs:preview": "vitepress preview docs", + "test:e2e": "playwright test -c e2e/playwright.config.ts" }, "devDependencies": { + "@axe-core/playwright": "^4.11.3", + "@playwright/test": "^1.61.0", "@stryker-mutator/core": "^9.6.1", "@stryker-mutator/vitest-runner": "^9.6.1", "@types/node": "^22.10.2", diff --git a/packages/core/test/fuzz.test.ts b/packages/core/test/fuzz.test.ts index 39bbe98..0aee56e 100644 --- a/packages/core/test/fuzz.test.ts +++ b/packages/core/test/fuzz.test.ts @@ -52,8 +52,12 @@ describe("seeded property fuzz (invariants)", () => { expect(crashes).toBe(0); }); - it("INV-4 / no-ReDoS: mock pathToRegex matches a hostile path in bounded time; status always 200-599", () => { - let slowest = 0, badStatus = 0; + it("INV-4 / no-ReDoS: mock pathToRegex has no catastrophic-backtracking shape; status always 200-599", () => { + // Structural (deterministic, load-immune) ReDoS check: the generated regex must never contain two + // adjacent unbounded quantifiers over the same class (`[^/]+[^/]+`) — that's the O(n^k) shape the + // BUG-B fix collapsed away. We still `.test()` a 5000-char hostile path each iteration, so a real + // ReDoS would hang the loop and trip the vitest timeout; we just don't assert a flaky wall-clock. + let badStatus = 0, dangerousRegex = 0; const hostile = "/" + "a".repeat(5000) + "/x/y/z"; for (const seed of SEEDS) { const r = mulberry32(seed); const pick = (a: T[]): T => a[Math.floor(r() * a.length)]!; @@ -62,12 +66,12 @@ describe("seeded property fuzz (invariants)", () => { const code = pick(["100", "200", "404", "600", "20000", "0", "default"]); const spec = `openapi: 3.0.3\ninfo: { title: T, version: "1" }\npaths: ${JSON.stringify({ [path]: { get: { responses: { [code]: { content: { "application/json": { schema: {} } } } } } } })}\n`; const re = buildRoutes(JSON.parse(`{"paths":${JSON.stringify({ [path]: { get: { responses: {} } } })}}`))[0]?.regex; - if (re) { const t0 = Date.now(); re.test(hostile); slowest = Math.max(slowest, Date.now() - t0); } + if (re) { re.test(hostile); if (/\[\^\/\]\+\[\^\/\]\+/.test(re.source)) dangerousRegex++; } const res = createMockResponder(spec).respond("GET", path.replace(/\{[^}]+\}/g, "1")); if (res && (res.status < 200 || res.status > 599)) badStatus++; } } - expect(slowest).toBeLessThan(100); + expect(dangerousRegex).toBe(0); expect(badStatus).toBe(0); }); diff --git a/packages/web/src/styles.css b/packages/web/src/styles.css index a456baa..c8612e7 100644 --- a/packages/web/src/styles.css +++ b/packages/web/src/styles.css @@ -21,8 +21,10 @@ --line: #23262d; --line-2: #2d313a; --text: #e8e9eb; - --dim: #888e98; - --dimmer: #565c66; + /* dim/dimmer raised to meet WCAG AA 4.5:1 on the darkest panel bg (was #888e98/#565c66, the latter + ~2.6:1). Hierarchy preserved: dim is still clearly fainter than --text. */ + --dim: #9499a3; + --dimmer: #868c96; --lime: #c8ff32; --green: #34d977; @@ -45,8 +47,9 @@ --line: #ddd9cc; --line-2: #cbc6b6; --text: #1a1c20; - --dim: #6b6f63; - --dimmer: #9a9d92; + /* darkened to meet WCAG AA on the lightest panel bg (was #6b6f63/#9a9d92, the latter ~2.8:1). */ + --dim: #5d6157; + --dimmer: #6b6f63; --lime: #5f7a00; --green: #1d9e57; --red: #d63b3b; @@ -201,6 +204,11 @@ select:focus { border-color: var(--lime); font-weight: 600; } +/* In the light theme --lime is a dark green, so dark button text falls below WCAG AA (4:1). + Use light text on it instead (5:1). */ +[data-theme="light"] .btn.run { + color: #ffffff; +} .btn.run:hover { box-shadow: 0 0 0 3px color-mix(in oklab, var(--lime) 30%, transparent); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d596e82..5196948 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,12 @@ importers: .: devDependencies: + '@axe-core/playwright': + specifier: ^4.11.3 + version: 4.11.3(playwright-core@1.61.0) + '@playwright/test': + specifier: ^1.61.0 + version: 1.61.0 '@stryker-mutator/core': specifier: ^9.6.1 version: 9.6.1(@types/node@22.19.21) @@ -192,6 +198,11 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} + '@axe-core/playwright@4.11.3': + resolution: {integrity: sha512-h/kfksv4F0cVIDlKpT4700OehdRgpvuVskuQ2nb7/JmtWUXpe9ftHAPtwyXGvVSsa6SJ64A9ER7Zrzc/sIvC4w==} + peerDependencies: + playwright-core: '>= 1.0.0' + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -1023,6 +1034,11 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@playwright/test@1.61.0': + resolution: {integrity: sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==} + engines: {node: '>=18'} + hasBin: true + '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} @@ -1484,6 +1500,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + axe-core@4.11.4: + resolution: {integrity: sha512-KunSNx+TVpkAw/6ULfhnx+HWRecjqZGTOyquAoWHYLRSdK1tB5Ihce1ZW+UY3fj33bYAFWPu7W/GRSmmrCGuxA==} + engines: {node: '>=4'} + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -1821,6 +1841,11 @@ packages: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -2202,6 +2227,16 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + playwright-core@1.61.0: + resolution: {integrity: sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.0: + resolution: {integrity: sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==} + engines: {node: '>=18'} + hasBin: true + postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -2871,6 +2906,11 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 + '@axe-core/playwright@4.11.3(playwright-core@1.61.0)': + dependencies: + axe-core: 4.11.4 + playwright-core: 1.61.0 + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -3539,6 +3579,10 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@playwright/test@1.61.0': + dependencies: + playwright: 1.61.0 + '@rolldown/pluginutils@1.0.0-beta.27': {} '@rollup/rollup-android-arm-eabi@4.62.0': @@ -4040,6 +4084,8 @@ snapshots: assertion-error@2.0.1: {} + axe-core@4.11.4: {} + balanced-match@1.0.2: {} balanced-match@4.0.4: {} @@ -4434,6 +4480,9 @@ snapshots: fresh@2.0.0: {} + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -4778,6 +4827,14 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 + playwright-core@1.61.0: {} + + playwright@1.61.0: + dependencies: + playwright-core: 1.61.0 + optionalDependencies: + fsevents: 2.3.2 + postcss-load-config@6.0.1(postcss@8.5.15)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 diff --git a/qa/COVERAGE.json b/qa/COVERAGE.json index 97b0936..66bcf1d 100644 --- a/qa/COVERAGE.json +++ b/qa/COVERAGE.json @@ -127,5 +127,12 @@ "nonFunctional": { "loadSoak": "mock server 30k req @ ~791rps, heap stable (no leak, forced-GC verified); web 10k req graceful timeouts under 64-conc; runner defeats gzip-bomb/slow-loris (campaign 5).", "slo": "no formal SLO defined for a local-first CLI; throughput-to-a-target-RPS NOT measured (gap)." + }, + "e2e": { + "tool": "Playwright 1.61 + axe-core/playwright", + "tests": 13, + "scope": "web UI: XSS, CSRF (cross-origin+cross-port), X-Frame-Options, editor save/keyboard(BUG-O), traversal, double-click, axe a11y(BUG-P)+contrast (main/editor/light)", + "ci": ".github/workflows/e2e.yml (playwright install --with-deps)", + "status": "all green" } } \ No newline at end of file diff --git a/qa/QA_LOG.md b/qa/QA_LOG.md index 2095f84..53097f7 100644 --- a/qa/QA_LOG.md +++ b/qa/QA_LOG.md @@ -87,3 +87,25 @@ state), `qa/TRIED.jsonl` (28 executed attacks — never repeated), `qa/SEEDS.jso - `pnpm gen:schema` no-diff check (catches the BUG-G class). - Playwright + axe-core e2e (covers BUG-M/N/O/P regressions live). - The committed `fuzz.test.ts` on every run + a scheduled deep-fuzz from `qa/corpus/`. + +--- + +## Cycle v2-2 — browser e2e in CI + a11y contrast (Issue #14, items 4 & 5) + +- **Playwright + axe-core e2e suite** (`e2e/`, 13 tests, CI job `e2e.yml` via `playwright install + --with-deps chromium`) — permanently guards the browser-only regressions that node coverage can't reach: + - security: XSS escaped/not-executed; CSRF rejected cross-origin AND cross-port-loopback (BUG-N, the + exact vector that needed a real browser); `X-Frame-Options: DENY` + CSP on every response (BUG-M) + - editor: save writes + sidebar update; **Esc from the path input** and **Ctrl+Enter** (BUG-O); + traversal refused with nothing written outside; double-click → one uncorrupted file + - a11y: **axe-core = 0 WCAG 2 A/AA violations** on main view, editor, and light theme (BUG-P), plus + explicit checks that the spec select is labeled and an h1 exists +- **Color contrast (item 5, axe `serious`)** fixed at root: raised `--dim`/`--dimmer` in both themes + (dark `--dimmer` was ~2.6:1) and gave the light-theme run button white text (dark text on the + dark-green lime was 4:1). axe now reports **0 contrast violations** in both themes. +- **Flaky test fixed:** `fuzz.test.ts` INV-4 asserted a wall-clock `slowest < 100ms`, which spiked under + parallel-suite load. Replaced with a deterministic structural check (the regex must never contain + adjacent `[^/]+[^/]+`); the loop still `.test()`s a 5000-char hostile path, so a real ReDoS trips the + vitest timeout. Suite now green twice with no flake. + +Remaining Issue-#14 items (1 expand mutation, 2 continuous 1M-exec fuzz, 3 throughput-SLO load) stay open. diff --git a/qa/TRIED.jsonl b/qa/TRIED.jsonl index d4e3664..b1c139a 100644 --- a/qa/TRIED.jsonl +++ b/qa/TRIED.jsonl @@ -26,3 +26,9 @@ {"id":"HELD-xss","category":"security/xss","target":"web client + vscode webview","input":"img/script/svg payloads (renderToStaticMarkup + live browser + axe)","result":"HELD react-escaped; vscode esc()+enableScripts:false","cycle":"c7-c10"} {"id":"HELD-traversal","category":"security/path","target":"confinePath web+mcp+live UI save","input":"../../../etc raw/encoded/null-byte","result":"HELD realpath-confined no escape","cycle":"c1-c9"} {"id":"HELD-script-vm","category":"sandbox","target":"runner/script.ts","input":"~1200 random pre/post scripts incl infinite loops","result":"HELD timeout 1s, no crash/unhandled","cycle":"c6"} +{"id":"E2E-csrf","category":"security/csrf","target":"web e2e (real chromium)","input":"cross-port loopback attacker page POST /api/request","result":"PASS no file written (Origin==Host)","cycle":"v2-2"} +{"id":"E2E-xss","category":"security/xss","target":"web e2e","input":"malicious request name ","result":"PASS not executed, escaped","cycle":"v2-2"} +{"id":"E2E-kbd","category":"ui/a11y","target":"web e2e","input":"Esc from path input / Ctrl+Enter","result":"PASS (BUG-O regression)","cycle":"v2-2"} +{"id":"E2E-axe","category":"ui/a11y","target":"web e2e (axe-core)","input":"main/editor/light-theme scans","result":"PASS 0 wcag2a/aa violations (BUG-P + contrast)","cycle":"v2-2"} +{"id":"FIX-flaky-fuzz","category":"regression/flake","target":"core/test/fuzz.test.ts INV-4","input":"slowest<100ms timing under parallel load","result":"FIXED: structural ReDoS check (no adjacent [^/]+[^/]+) instead of wall-clock","cycle":"v2-2"} +{"id":"FIX-contrast","category":"ui/a11y","target":"web/src/styles.css","input":"--dimmer ~2.6:1, light run-button 4:1","result":"FIXED: raised --dim/--dimmer both themes + light run-button white text; axe 0 contrast","cycle":"v2-2"} From 33f488eff7daf13e3fe5a6f2ade96ed45fc84af8 Mon Sep 17 00:00:00 2001 From: Rashid Mahmood Date: Tue, 23 Jun 2026 17:19:44 +0400 Subject: [PATCH 2/2] ci(e2e): build the full package graph so @truspec/core dist exists for the web server The e2e job built only @truspec/web, but its server imports the built @truspec/core; on a clean CI checkout core/dist did not exist. Verified by wiping all dist/ locally, running pnpm build, and passing 13/13 e2e. --- .github/workflows/e2e.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 183068a..b0ae500 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,8 +20,9 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - # The e2e suite serves the built web client + server, so build first. - - run: pnpm --filter @truspec/web build + # The e2e suite serves the built web client + server, which imports the built @truspec/core, + # so build the whole graph (turbo orders core before web). + - run: pnpm build - name: Install Playwright chromium + OS deps run: pnpm exec playwright install --with-deps chromium