feat(test-runner): add Reporter.preprocessSuite() hook for test filtering

docs/src/test-reporter-api/class-reporter.md

@@ -297,3 +297,26 @@ Result of the test run.
 - returns: <[boolean]>
 
 Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`.
+
+## optional async method: Reporter.preprocessSuite
+* since: v1.61
+- `result` ?<[Object]>
+  - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestCase.exclude`] on out-of-shard tests).
+
+Called after the configuration has been resolved and before [`method: Reporter.onBegin`]. Allows a reporter to mark individual tests as skipped, excluded, fixed or failing.
+
+### param: Reporter.preprocessSuite.config
+* since: v1.61
+- `config` <[FullConfig]>
+
+Resolved configuration.
+
+### param: Reporter.preprocessSuite.suite
+* since: v1.61
+- `suite` <[Suite]>
+
+The root suite that contains the projects, files and test cases that will run.
+
+The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here.
+
+The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`.

docs/src/test-reporter-api/class-suite.md

@@ -85,3 +85,41 @@ Returns a list of titles from the root down to this suite.
 
 Returns the type of the suite. The Suites form the following hierarchy:
 `root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`.
+
+## method: Suite.skip
+* since: v1.61
+
+Mark every [TestCase] of this suite as skipped, see [`method: TestCase.skip`].
+
+### param: Suite.skip.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: Suite.fixme
+* since: v1.61
+
+Mark every [TestCase] of this suite as fixme, see [`method: TestCase.fixme`].
+
+### param: Suite.fixme.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: Suite.fail
+* since: v1.61
+
+Mark every [TestCase] of this suite as expected-to-fail, see [`method: TestCase.fail`].
+
+### param: Suite.fail.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: Suite.exclude
+* since: v1.61
+
+Must be called from inside [`method: Reporter.preprocessSuite`], exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed.

docs/src/test-reporter-api/class-testcase.md

@@ -107,3 +107,41 @@ Returns a list of titles from the root down to this test.
 - returns: <[TestCaseType]<"test">>
 
 Returns "test". Useful for detecting test cases in [`method: Suite.entries`].
+
+## method: TestCase.skip
+* since: v1.61
+
+Must be called from inside [`method: Reporter.preprocessSuite`], skip this test. The test body is not executed and the test is reported as skipped.
+
+### param: TestCase.skip.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: TestCase.fixme
+* since: v1.61
+
+Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to fix it.
+
+### param: TestCase.fixme.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: TestCase.fail
+* since: v1.61
+
+Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for documenting broken functionality until it is fixed.
+
+### param: TestCase.fail.reason
+* since: v1.61
+- `reason` ?<[string]>
+
+Optional explanation surfaced as the annotation description.
+
+## method: TestCase.exclude
+* since: v1.61
+
+Must be called from inside [`method: Reporter.preprocessSuite`], exclude this test from the run. Excluded tests do not appear in the report and their body is not executed.

packages/playwright/src/common/ipc.ts

@@ -142,6 +142,7 @@ export type StepEndPayload = {
 export type TestEntry = {
   testId: string;
   retry: number;
+  planAnnotations: { type: string, description?: string, location?: { file: string, line: number, column: number } }[];
 };
 
 export type RunPayload = {

packages/playwright/src/common/test.ts

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
+import { captureRawStack } from '@isomorphic/stackTrace';
 import { rootTestType } from './testType';
 import { computeTestCaseOutcome } from '../isomorphic/teleReceiver';
+import { filteredStackTrace } from '../util';
 
 import type { FixturesWithLocation, FullProjectInternal } from './config';
 import type { FixturePool } from './fixtures';
@@ -58,6 +60,9 @@ export class Suite extends Base {
   _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
   _fullProject: FullProjectInternal | undefined;
   _fileId: string | undefined;
+  // Set on the root suite for the duration of Reporter.preprocessSuite(), gating
+  // the disposition methods (skip/fixme/fail/exclude) to that phase only.
+  _preprocessing = false;
   readonly _type: 'root' | 'project' | 'file' | 'describe';
 
   constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') {
@@ -96,6 +101,14 @@ export class Suite extends Base {
     this._entries.unshift(suite);
   }
 
+  _detach(child: Suite | TestCase) {
+    const idx = this._entries.indexOf(child);
+    if (idx !== -1)
+      this._entries.splice(idx, 1);
+    if (this._entries.length === 0)
+      this.parent?._detach(this);
+  }
+
   allTests(): TestCase[] {
     const result: TestCase[] = [];
     const visit = (suite: Suite) => {
@@ -252,6 +265,34 @@ export class Suite extends Base {
   project(): FullProject | undefined {
     return this._fullProject?.project || this.parent?.project();
   }
+
+  skip(reason?: string): void {
+    for (const entry of this.entries())
+      entry.skip(reason);
+  }
+
+  fixme(reason?: string): void {
+    for (const entry of this.entries())
+      entry.fixme(reason);
+  }
+
+  fail(reason?: string): void {
+    for (const entry of this.entries())
+      entry.fail(reason);
+  }
+
+  exclude(): void {
+    if (!this._rootSuite()._preprocessing)
+      throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`);
+    if (this.parent)
+      this.parent._detach(this);
+    else
+      this._entries = [];
+  }
+
+  _rootSuite(): Suite {
+    return this.parent?._rootSuite() ?? this;
+  }
 }
 
 export class TestCase extends Base implements reporterTypes.TestCase {
@@ -275,6 +316,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   _projectId = '';
   // Explicitly declared tags that are not a part of the title.
   _tags: string[] = [];
+  _planAnnotations: TestAnnotation[] = [];
 
   constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) {
     super(title);
@@ -309,6 +351,44 @@ export class TestCase extends Base implements reporterTypes.TestCase {
     ];
   }
 
+  skip(reason?: string): void {
+    if (!this._rootSuite()._preprocessing)
+      throw new Error(`TestCase.skip() can only be called from Reporter.preprocessSuite().`);
+    const annotation: TestAnnotation = { type: 'skip', description: reason, location: captureCallerLocation() };
+    this.annotations.push(annotation);
+    this._planAnnotations.push(annotation);
+    this.expectedStatus = 'skipped';
+  }
+
+  fixme(reason?: string): void {
+    if (!this._rootSuite()._preprocessing)
+      throw new Error(`TestCase.fixme() can only be called from Reporter.preprocessSuite().`);
+    const annotation: TestAnnotation = { type: 'fixme', description: reason, location: captureCallerLocation() };
+    this.annotations.push(annotation);
+    this._planAnnotations.push(annotation);
+    this.expectedStatus = 'skipped';
+  }
+
+  fail(reason?: string): void {
+    if (!this._rootSuite()._preprocessing)
+      throw new Error(`TestCase.fail() can only be called from Reporter.preprocessSuite().`);
+    const annotation: TestAnnotation = { type: 'fail', description: reason, location: captureCallerLocation() };
+    this.annotations.push(annotation);
+    this._planAnnotations.push(annotation);
+    if (this.expectedStatus !== 'skipped')
+      this.expectedStatus = 'failed';
+  }
+
+  exclude(): void {
+    if (!this._rootSuite()._preprocessing)
+      throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`);
+    this.parent._detach(this);
+  }
+
+  _rootSuite(): Suite {
+    return this.parent._rootSuite();
+  }
+
   _serialize(): any {
     return {
       kind: 'test',
@@ -384,3 +464,10 @@ export class TestCase extends Base implements reporterTypes.TestCase {
     return path.join(' ');
   }
 }
+
+function captureCallerLocation(): Location | undefined {
+  const frame = filteredStackTrace(captureRawStack())[0];
+  if (!frame?.file)
+    return undefined;
+  return { file: frame.file, line: frame.line ?? 0, column: frame.column ?? 0 };
+}

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -648,6 +648,19 @@ export class TeleSuite implements reporterTypes.Suite {
     suite.parent = this;
     this._entries.push(suite);
   }
+
+  skip(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleSuite (read-only).');
+  }
+  fixme(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleSuite (read-only).');
+  }
+  fail(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleSuite (read-only).');
+  }
+  exclude(): void {
+    throw new Error('Disposition methods are not supported on a TeleSuite (read-only).');
+  }
 }
 
 export class TeleTestCase implements reporterTypes.TestCase {
@@ -693,6 +706,19 @@ export class TeleTestCase implements reporterTypes.TestCase {
     this.results.push(result);
     return result;
   }
+
+  skip(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).');
+  }
+  fixme(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).');
+  }
+  fail(_reason?: string): void {
+    throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).');
+  }
+  exclude(): void {
+    throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).');
+  }
 }
 
 class TeleTestStep implements reporterTypes.TestStep {

packages/playwright/src/reporters/internalReporter.ts

@@ -50,6 +50,15 @@ export class InternalReporter implements ReporterV2 {
     this._reporter.onConfigure?.(config);
   }
 
+  async preprocessSuite(config: FullConfig, suite: testNs.Suite) {
+    suite._preprocessing = true;
+    try {
+      return await this._reporter.preprocessSuite?.(config, suite);
+    } finally {
+      suite._preprocessing = false;
+    }
+  }
+
   onBegin(suite: testNs.Suite) {
     this._didBegin = true;
     this._reporter.onBegin?.(suite);
@@ -108,7 +117,7 @@ export class InternalReporter implements ReporterV2 {
   }
 
   printsToStdio() {
-    return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true;
+    return this._reporter.printsToStdio?.() ?? true;
   }
 
   private _addSnippetToTestErrors(test: TestCase, result: TestResult) {

packages/playwright/src/reporters/multiplexer.ts

@@ -34,6 +34,22 @@ export class Multiplexer implements ReporterV2 {
       wrap(() => reporter.onConfigure?.(config));
   }
 
+  async preprocessSuite(config: FullConfig, suite: test.Suite) {
+    // Unlike other reporter callbacks, `preprocessSuite` errors are NOT swallowed —
+    // they propagate so the run aborts before onBegin. Reporters use preprocessSuite
+    // to mutate the corpus; silently dropping a planning error would let
+    // an inconsistent (partial-mutation) state reach the workers.
+    const shardingReporters: ReporterV2[] = [];
+    for (const reporter of this._reporters) {
+      const result = await reporter.preprocessSuite?.(config, suite);
+      if (result?.implementsSharding)
+        shardingReporters.push(reporter);
+    }
+    if (shardingReporters.length > 1)
+      throw new Error(`Multiple reporters declare 'implementsSharding': ${shardingReporters.map(r => r.constructor?.name ?? 'reporter').join(', ')}. Only one reporter may handle sharding.`);
+    return { implementsSharding: shardingReporters.length > 0 };
+  }
+
   onBegin(suite: test.Suite) {
     for (const reporter of this._reporters)
       wrap(() => reporter.onBegin?.(suite));

packages/playwright/src/reporters/reporterV2.ts

@@ -28,6 +28,7 @@ export interface ReportEndParams {
 
 export interface ReporterV2 {
   onConfigure?(config: FullConfig): void;
+  preprocessSuite?(config: FullConfig, suite: Suite): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void;
   onBegin?(suite: Suite): void;
   onTestBegin?(test: TestCase, result: TestResult): void;
   onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
@@ -79,6 +80,10 @@ class ReporterV2Wrapper implements ReporterV2 {
     this._config = config;
   }
 
+  async preprocessSuite(config: FullConfig, suite: Suite) {
+    return await this._reporter.preprocessSuite?.(config, suite);
+  }
+
   onBegin(suite: Suite) {
     this._reporter.onBegin?.(this._config, suite);
 

packages/playwright/src/runner/dispatcher.ts

@@ -563,7 +563,7 @@ class JobDispatcher {
     const runPayload: ipc.RunPayload = {
       file: this.job.requireFile,
       entries: this.job.tests.map(test => {
-        return { testId: test.id, retry: test.results.length };
+        return { testId: test.id, retry: test.results.length, planAnnotations: test._planAnnotations };
       }),
     };
     worker.runTestGroup(runPayload);

packages/playwright/src/runner/loadUtils.ts

@@ -165,8 +165,9 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     }
   }
 
+  const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
   // Shard only the top-level projects.
-  if (config.config.shard) {
+  if (config.config.shard && !preprocessResult?.implementsSharding) {
     // Create test groups for top-level projects.
     const testGroups: TestGroup[] = [];
     for (const projectSuite of rootSuite.suites) {

packages/playwright/src/worker/workerMain.ts

@@ -222,6 +222,8 @@ export class WorkerMain extends ProcessRunner {
         suiteUtils.applyRepeatEachIndex(this._project, suite, this._params.repeatEachIndex);
       suiteUtils.filterTestsRemoveEmptySuites(suite, test => entries.has(test.id));
       const tests = suite.allTests();
+      for (const test of tests)
+        test.annotations.push(...entries.get(test.id)!.planAnnotations);
 
       // Collect test IDs that were not found in the worker
       // (e.g. test titles changed between runner and worker).