[POC] Add color-contrast a11y regression suite + token tuning experiment

package.json

@@ -57,7 +57,7 @@
     "test:e2e:dev": "pnpm -F ./test/e2e dev",
     "test:e2e-website": "playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
     "test:e2e-website:dev": "cross-env PLAYWRIGHT_TEST_BASE_URL=http://localhost:3000 playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
-    "test:regressions": "cross-env NODE_ENV=production pnpm test:regressions:build && concurrently --success first --kill-others \"pnpm test:regressions:run\" \"pnpm test:regressions:server\" && prettier --write \"docs/data/material/components/**/*.a11y.json\"",
+    "test:regressions": "cross-env NODE_ENV=production pnpm test:regressions:build && concurrently --success first --kill-others \"pnpm test:regressions:run\" \"pnpm test:regressions:server\" && prettier --write \"docs/data/material/components/**/*.a11y.json\" \"test/regressions/a11y/results/*.a11y.json\"",
     "test:regressions:build": "vite build test/regressions",
     "test:regressions:dev": "vite test/regressions --port 5001",
     "test:regressions:run": "vitest run -r ./test/regressions/",

test/regressions/a11y/a11yReporter.ts

@@ -11,9 +11,10 @@ import type {
   Vitest,
 } from 'vitest/node';
 import type { SerializedError } from 'vitest';
-import type { A11yMeta } from './axe';
+import type { A11yMeta, RouteKind } from './axe';
 
-const COMPONENTS_DIR = path.resolve(__dirname, '../../../docs/data/material/components');
+const DOCS_COMPONENTS_DIR = path.resolve(__dirname, '../../../docs/data/material/components');
+const REGRESSION_RESULTS_DIR = path.resolve(__dirname, './results');
 const VRT_MODULE_PATH = path.resolve(__dirname, '../index.test.js');
 
 interface DemoEntry {
@@ -30,15 +31,50 @@ function* walkTests(node: TestModule | TestSuite): Generator<TestCase, undefined
   }
 }
 
-function hasStatus(meta: A11yMeta, status: 'fail' | 'incomplete'): boolean {
-  for (const rule of Object.values(meta.rules)) {
-    if (rule.status === status) {
-      return true;
-    }
-  }
-  return false;
+function hasStatus(metas: ReadonlyArray<A11yMeta>, status: 'fail' | 'incomplete'): boolean {
+  return metas.some((meta) => Object.values(meta.rules).some((rule) => rule.status === status));
+}
+
+// Regression a11y fixtures live in `fixtures/ColorContrast/{Component}{Light|Dark}.js`,
+// so the demo name encodes both the component (output file) and the mode
+// (entry key within that file). Fixtures that don't follow the suffix
+// convention fall back to a single self-named file/entry.
+const REGRESSION_DEMO_RE = /^(.*?)(Light|Dark)$/;
+
+interface OutputTarget {
+  /** Root dir holding the result files. */
+  dir: string;
+  /** Group key for a recorded meta — also the output file's identity. */
+  bucketOf: (meta: A11yMeta) => string;
+  /** Key for this meta's entry inside its file. */
+  entryKeyOf: (meta: A11yMeta) => string;
+  /** Absolute path of the output file for a bucket. */
+  fileFor: (bucket: string) => string;
+  /** Glob (relative to `dir`) for discovering existing result files. */
+  pruneGlob: string;
+  /** Map a discovered file path back to its bucket id. */
+  pruneBucketOf: (absPath: string) => string;
 }
 
+const TARGETS: Record<RouteKind, OutputTarget> = {
+  docs: {
+    dir: DOCS_COMPONENTS_DIR,
+    bucketOf: (meta) => meta.slug,
+    entryKeyOf: (meta) => meta.demo,
+    fileFor: (slug) => path.join(DOCS_COMPONENTS_DIR, slug, `${slug}.a11y.json`),
+    pruneGlob: '*/*.a11y.json',
+    pruneBucketOf: (file) => path.basename(path.dirname(file)),
+  },
+  regression: {
+    dir: REGRESSION_RESULTS_DIR,
+    bucketOf: (meta) => meta.demo.match(REGRESSION_DEMO_RE)?.[1] ?? meta.demo,
+    entryKeyOf: (meta) => meta.demo.match(REGRESSION_DEMO_RE)?.[2] ?? meta.demo,
+    fileFor: (component) => path.join(REGRESSION_RESULTS_DIR, `${component}.a11y.json`),
+    pruneGlob: '*.a11y.json',
+    pruneBucketOf: (file) => path.basename(file, '.a11y.json'),
+  },
+};
+
 export default class A11yReporter implements Reporter {
   private filtered = false;
 
@@ -51,77 +87,92 @@ export default class A11yReporter implements Reporter {
     _unhandledErrors: ReadonlyArray<SerializedError>,
     reason: TestRunEndReason,
   ) {
-    const entries: A11yMeta[] = [];
+    // bucket id -> recorded metas, kept separate per kind
+    const byKind: Record<RouteKind, Map<string, A11yMeta[]>> = {
+      docs: new Map(),
+      regression: new Map(),
+    };
     for (const mod of testModules) {
       for (const test of walkTests(mod)) {
         const meta = (test.meta() as { a11y?: A11yMeta }).a11y;
-        if (meta) {
-          entries.push(meta);
+        if (!meta) {
+          continue;
         }
+        const bucket = TARGETS[meta.kind].bucketOf(meta);
+        const list = byKind[meta.kind].get(bucket) ?? [];
+        list.push(meta);
+        byKind[meta.kind].set(bucket, list);
       }
     }
 
-    const bySlug = new Map<string, A11yMeta[]>();
-    for (const meta of entries) {
-      const list = bySlug.get(meta.slug) ?? [];
-      list.push(meta);
-      bySlug.set(meta.slug, list);
-    }
-
-    // One file per slug, co-located with the component's other files. The docs
-    // toolbar reads these via a webpack require.context (eager) at build time.
-    for (const [slug, metas] of bySlug) {
-      const slugDir = path.join(COMPONENTS_DIR, slug);
-      fs.mkdirSync(slugDir, { recursive: true });
-      const sorted = [...metas].sort((a, b) => a.demo.localeCompare(b.demo));
-      const file: Record<string, DemoEntry> = {};
-      for (const meta of sorted) {
-        file[meta.demo] = { rules: meta.rules };
+    for (const kind of Object.keys(TARGETS) as RouteKind[]) {
+      const target = TARGETS[kind];
+      for (const [bucket, metas] of byKind[kind]) {
+        const outFile = target.fileFor(bucket);
+        fs.mkdirSync(path.dirname(outFile), { recursive: true });
+        const sorted = [...metas].sort((a, b) =>
+          target.entryKeyOf(a).localeCompare(target.entryKeyOf(b)),
+        );
+        const file: Record<string, DemoEntry> = {};
+        for (const meta of sorted) {
+          file[target.entryKeyOf(meta)] = { rules: meta.rules };
+        }
+        fs.writeFileSync(outFile, `${JSON.stringify(file, null, 2)}\n`);
       }
-      fs.writeFileSync(
-        path.join(slugDir, `${slug}.a11y.json`),
-        `${JSON.stringify(file, null, 2)}\n`,
-      );
     }
 
     // Only prune when this run is authoritative for the full enrolment set:
     // VRT module must have actually executed, no `-t` filter narrowed it, and
-    // the run completed cleanly. A partial/failed run can omit slugs whose
+    // the run completed cleanly. A partial/failed run can omit buckets whose
     // tests crashed before recording, so pruning then would delete tracked
     // JSON for still-enrolled demos.
     const ranVrtSuite = testModules.some((m) => m.moduleId === VRT_MODULE_PATH);
     if (ranVrtSuite && !this.filtered && reason === 'passed') {
-      for (const file of globbySync('*/*.a11y.json', { cwd: COMPONENTS_DIR, absolute: true })) {
-        const slug = path.basename(path.dirname(file));
-        if (!bySlug.has(slug)) {
-          fs.unlinkSync(file);
+      for (const kind of Object.keys(TARGETS) as RouteKind[]) {
+        const target = TARGETS[kind];
+        if (!fs.existsSync(target.dir)) {
+          continue;
+        }
+        const seen = byKind[kind];
+        for (const file of globbySync(target.pruneGlob, { cwd: target.dir, absolute: true })) {
+          if (!seen.has(target.pruneBucketOf(file))) {
+            fs.unlinkSync(file);
+          }
         }
       }
     }
 
-    if (entries.length === 0) {
-      return;
+    for (const kind of Object.keys(TARGETS) as RouteKind[]) {
+      const buckets = byKind[kind];
+      if (buckets.size === 0) {
+        continue;
+      }
+      const names = [...buckets.keys()].sort();
+      const partial = names.filter((n) => hasStatus(buckets.get(n)!, 'fail'));
+      const needsReview = names.filter(
+        (n) => !partial.includes(n) && hasStatus(buckets.get(n)!, 'incomplete'),
+      );
+      const pass = names.filter((n) => !partial.includes(n) && !needsReview.includes(n));
+      const totalDemos = [...buckets.values()].reduce((n, ms) => n + ms.length, 0);
+      const target = TARGETS[kind];
+      const dirLabel =
+        kind === 'docs'
+          ? `${path.relative(process.cwd(), target.dir)}/{slug}/{slug}.a11y.json`
+          : `${path.relative(process.cwd(), target.dir)}/{component}.a11y.json`;
+      // eslint-disable-next-line no-console
+      console.log(
+        [
+          '',
+          chalk.bold(
+            `a11y ${kind} results (${totalDemos} demos, ${names.length} components) -> ${dirLabel}`,
+          ),
+          '',
+          `  ✅ Pass (${pass.length}):         ${pass.join(', ') || '—'}`,
+          `  ⚠️  Partial (${partial.length}):      ${partial.join(', ') || '—'}`,
+          `  🔍 Needs review (${needsReview.length}): ${needsReview.join(', ') || '—'}`,
+          '',
+        ].join('\n'),
+      );
     }
-
-    const slugs = [...bySlug.keys()].sort();
-    const partial = slugs.filter((s) => bySlug.get(s)!.some((m) => hasStatus(m, 'fail')));
-    const needsReview = slugs.filter(
-      (s) => !partial.includes(s) && bySlug.get(s)!.some((m) => hasStatus(m, 'incomplete')),
-    );
-    const pass = slugs.filter((s) => !partial.includes(s) && !needsReview.includes(s));
-    // eslint-disable-next-line no-console
-    console.log(
-      [
-        '',
-        chalk.bold(
-          `a11y results (${entries.length} demos, ${slugs.length} slugs) -> ${path.relative(process.cwd(), COMPONENTS_DIR)}/{slug}/{slug}.a11y.json`,
-        ),
-        '',
-        `  ✅ Pass (${pass.length}):         ${pass.join(', ') || '—'}`,
-        `  ⚠️  Partial (${partial.length}):      ${partial.join(', ') || '—'}`,
-        `  🔍 Needs review (${needsReview.length}): ${needsReview.join(', ') || '—'}`,
-        '',
-      ].join('\n'),
-    );
   }
 }

test/regressions/a11y/axe.ts

@@ -11,12 +11,29 @@ export const GLOBAL_DISABLED_RULES = ['region', 'page-has-heading-one'];
 
 export type RuleStatus = 'pass' | 'fail' | 'incomplete';
 
+/**
+ * One failing cell. Prop-axis keys (`variant`, `color`, etc.) are sourced
+ * from `data-*` attributes the fixture stamps on each cell, so this shape
+ * adapts to any component without code changes — Button has `variant` +
+ * `color`, Alert would have `severity` + `variant`, etc.
+ */
+export interface Instance {
+  contrastRatio?: number;
+  [prop: string]: string | number | undefined;
+}
+
 export interface RuleEntry {
   status: RuleStatus;
   tags: string[];
+  /** Populated for `fail` / `incomplete` statuses, one per offending DOM node. */
+  instances?: Instance[];
 }
 
+/** Which side of the suite produced the route — controls reporter output dir. */
+export type RouteKind = 'docs' | 'regression';
+
 export interface A11yMeta {
+  kind: RouteKind;
   slug: string;
   demo: string;
   rules: Record<string, RuleEntry>;
@@ -54,6 +71,7 @@ function formatResults(results: AxeResults['violations']) {
 }
 
 interface RecordA11yOptions {
+  kind: RouteKind;
   slug: string;
   demo: string;
   /**
@@ -64,6 +82,29 @@ interface RecordA11yOptions {
   skipAssertions?: string[];
 }
 
+/**
+ * The page-side script in `index.test.js` enriches each failing node with a
+ * `dataAttrs` map collected by walking up the DOM from the violation node.
+ * Anything outside that walk doesn't reach this code.
+ */
+type EnrichedNode = AxeResults['violations'][number]['nodes'][number] & {
+  dataAttrs?: Record<string, string>;
+};
+
+function extractInstance(node: EnrichedNode): Instance {
+  const result: Instance = { ...(node.dataAttrs ?? {}) };
+  for (const check of [...node.any, ...node.all, ...node.none]) {
+    if (!check.data || typeof check.data !== 'object') {
+      continue;
+    }
+    const { contrastRatio } = check.data as { contrastRatio?: unknown };
+    if (typeof contrastRatio === 'number') {
+      result.contrastRatio = contrastRatio;
+    }
+  }
+  return result;
+}
+
 /**
  * Node-side recorder for axe results produced inside a Playwright page.
  *
@@ -74,7 +115,7 @@ interface RecordA11yOptions {
 export function recordA11y(
   ctx: TestContext,
   results: AxeResults,
-  { slug, demo, skipAssertions = [] }: RecordA11yOptions,
+  { kind, slug, demo, skipAssertions = [] }: RecordA11yOptions,
 ): void {
   const rules: Record<string, RuleEntry> = {};
   const buckets: ReadonlyArray<[AxeResults['passes'], RuleStatus]> = [
@@ -85,14 +126,19 @@ export function recordA11y(
   for (const [list, status] of buckets) {
     for (const rule of list) {
       const tags = rule.tags.filter((t) => WCAG_TAGS.includes(t)).sort();
-      rules[rule.id] = { status, tags };
+      const entry: RuleEntry = { status, tags };
+      if (status !== 'pass' && rule.nodes.length > 0) {
+        entry.instances = rule.nodes.map(extractInstance);
+      }
+      rules[rule.id] = entry;
     }
   }
   const sortedRules = Object.fromEntries(
     Object.entries(rules).sort(([a], [b]) => a.localeCompare(b)),
   );
 
   const meta: A11yMeta = {
+    kind,
     slug,
     demo,
     rules: sortedRules,

test/regressions/a11y/results/Alert.a11y.json

@@ -0,0 +1,30 @@
+{
+  "Dark": {
+    "rules": {
+      "color-contrast": {
+        "status": "pass",
+        "tags": ["wcag2aa"]
+      }
+    }
+  },
+  "Light": {
+    "rules": {
+      "color-contrast": {
+        "status": "fail",
+        "tags": ["wcag2aa"],
+        "instances": [
+          {
+            "variant": "filled",
+            "severity": "info",
+            "contrastRatio": 3.85
+          },
+          {
+            "variant": "filled",
+            "severity": "warning",
+            "contrastRatio": 3.11
+          }
+        ]
+      }
+    }
+  }
+}

test/regressions/a11y/results/AppBar.a11y.json

@@ -0,0 +1,34 @@
+{
+  "Dark": {
+    "rules": {
+      "color-contrast": {
+        "status": "fail",
+        "tags": ["wcag2aa"],
+        "instances": [
+          {
+            "color": "error",
+            "contrastRatio": 3.68
+          }
+        ]
+      }
+    }
+  },
+  "Light": {
+    "rules": {
+      "color-contrast": {
+        "status": "fail",
+        "tags": ["wcag2aa"],
+        "instances": [
+          {
+            "color": "info",
+            "contrastRatio": 3.85
+          },
+          {
+            "color": "warning",
+            "contrastRatio": 3.11
+          }
+        ]
+      }
+    }
+  }
+}