test: accomodate multi-global tests in WPT{Runner,TestSpec,Report}

Signed-off-by: Filip Skokan <panva.ip@gmail.com>

Author: panva

test/common/wpt.js

@@ -108,7 +108,7 @@ class WPTReport {
    * @returns {ReportResult}
    */
   getResult(spec) {
-    const name = `/${spec.getRelativePath()}${spec.variant}`;
+    const name = `/${spec.getTestPath()}`;
     if (this.results.has(name)) {
       return this.results.get(name);
     }
@@ -122,13 +122,7 @@ class WPTReport {
    */
   write() {
     this.time_end = Date.now();
-    const results = Array.from(this.results.values())
-      .map((result) => {
-        const url = new URL(result.test, 'http://wpt');
-        url.pathname = url.pathname.replace(/\.js$/, '.html');
-        result.test = url.href.slice(url.origin.length);
-        return result;
-      });
+    const results = Array.from(this.results.values());
 
     /**
      * Return required and some optional properties
@@ -323,18 +317,21 @@ class WPTTestSpec {
    *   'test.any.js'
    * @param {StatusRule[]} rules
    * @param {string} variant test file variant
+   * @param {'window'|'dedicatedworker'} [globalScope] generated test global
    */
-  constructor(mod, filename, rules, variant = '') {
+  constructor(mod, filename, rules, variant = '', globalScope) {
     this.module = mod;
     this.filename = filename;
     this.variant = variant;
+    this.globalScope = globalScope;
+    this.rules = [...new Set(rules)];
 
     this.requires = new Set();
     this.failedTests = [];
     this.flakyTests = [];
     this.skipReasons = [];
     this.skippedTests = [];
-    for (const item of rules) {
+    for (const item of this.rules) {
       if (item.requires.length) {
         for (const req of item.requires) {
           this.requires.add(req);
@@ -364,12 +361,63 @@ class WPTTestSpec {
    * @param {string} mod
    * @param {string} filename
    * @param {StatusRule[]} rules
-   * @returns {ReturnType<WPTTestSpec['getMeta']>[]}
+   * @param {(spec: WPTTestSpec) => StatusRule[]} [getAdditionalRules]
+   * @returns {WPTTestSpec[]}
    */
-  static from(mod, filename, rules) {
+  static from(mod, filename, rules, getAdditionalRules) {
     const spec = new WPTTestSpec(mod, filename, rules);
     const meta = spec.getMeta();
-    return meta.variant?.map((variant) => new WPTTestSpec(mod, filename, rules, variant)) || [spec];
+    const variants = meta.variant || [''];
+    const createSpec = (variant, globalScope) => {
+      let result = new WPTTestSpec(mod, filename, rules, variant, globalScope);
+      const additionalRules = getAdditionalRules?.(result) || [];
+      if (additionalRules.length > 0) {
+        result = new WPTTestSpec(
+          mod,
+          filename,
+          [...new Set([...rules, ...additionalRules])],
+          variant,
+          globalScope,
+        );
+      }
+      return result;
+    };
+
+    if (!spec.isAnyTest()) {
+      return variants.map((variant) => createSpec(variant));
+    }
+
+    const requestedGlobals = meta.global ?
+      meta.global.split(',').map((item) => item.trim()) :
+      ['window', 'dedicatedworker'];
+    const supportedGlobals = new Set();
+    for (const globalScope of requestedGlobals) {
+      if (globalScope === 'window' || globalScope === 'dedicatedworker') {
+        supportedGlobals.add(globalScope);
+      } else if (globalScope === 'worker') {
+        supportedGlobals.add('dedicatedworker');
+      }
+    }
+
+    return ['window', 'dedicatedworker']
+      .filter((globalScope) => supportedGlobals.has(globalScope))
+      .flatMap((globalScope) => variants.map(
+        (variant) => createSpec(variant, globalScope)));
+  }
+
+  /**
+   * @returns {boolean}
+   */
+  isAnyTest() {
+    return /\.any\.js$/.test(this.filename);
+  }
+
+  /**
+   * @returns {boolean}
+   */
+  isWebWorkerTest() {
+    return /\.worker\.js$/.test(this.filename) ||
+      this.globalScope === 'dedicatedworker';
   }
 
   /**
@@ -392,6 +440,22 @@ class WPTTestSpec {
     return path.join(this.module, this.filename);
   }
 
+  getStatusKey() {
+    if (this.globalScope === 'dedicatedworker') {
+      return this.filename.replace(/\.any\.js$/, '.any.worker.html');
+    }
+    if (this.globalScope === 'window') {
+      return this.filename.replace(/\.any\.js$/, '.any.html');
+    }
+    return this.filename;
+  }
+
+  getTestPath() {
+    let testPath = path.join(this.module, this.getStatusKey());
+    testPath = testPath.replace(/\.js$/, '.html');
+    return `${testPath.split(path.sep).join('/')}${this.variant}`;
+  }
+
   getAbsolutePath() {
     return fixtures.path('wpt', this.getRelativePath());
   }
@@ -541,7 +605,15 @@ class StatusLoader {
     for (const file of list) {
       const relativePath = path.relative(subDir, file);
       const match = this.rules.match(relativePath);
-      this.specs.push(...WPTTestSpec.from(this.path, relativePath, match));
+      this.specs.push(...WPTTestSpec.from(
+        this.path,
+        relativePath,
+        match,
+        (spec) => [
+          ...this.rules.match(spec.getStatusKey()),
+          ...this.rules.match(`${spec.getStatusKey()}${spec.variant}`),
+        ],
+      ));
     }
   }
 }
@@ -577,6 +649,31 @@ const limit = (concurrency) => {
   return execute;
 };
 
+function getUnexpectedPasses(queue, results) {
+  const specsByRule = new Map();
+  for (const spec of queue) {
+    for (const rule of spec.rules) {
+      if (Array.isArray(rule.fail?.expected)) {
+        const specs = specsByRule.get(rule) || [];
+        specs.push(spec);
+        specsByRule.set(rule, specs);
+      }
+    }
+  }
+
+  const unexpectedPasses = [];
+  for (const [rule, specs] of specsByRule) {
+    for (const expectedToFail of rule.fail.expected) {
+      const failed = specs.some((spec) =>
+        results[spec.getStatusKey()]?.fail?.expected?.includes(expectedToFail));
+      if (!failed) {
+        unexpectedPasses.push(`${rule.key}:${expectedToFail}`);
+      }
+    }
+  }
+  return unexpectedPasses;
+}
+
 class WPTRunner {
   constructor(path, { concurrency = os.availableParallelism() - 1 || 1 } = {}) {
     // RISC-V has very limited virtual address space in the currently common
@@ -608,6 +705,7 @@ class WPTRunner {
     this.inProgress = new Set();
     this.workers = new Map();
     this.unexpectedFailures = [];
+    this.skippedSpecCount = 0;
 
     this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
 
@@ -645,7 +743,7 @@ class WPTRunner {
    * @returns {string}
    */
   fullInitScript(spec) {
-    const url = new URL(`/${spec.getRelativePath().replace(/\.js$/, '.html')}${spec.variant}`, 'http://wpt');
+    const url = new URL(`/${spec.getTestPath()}`, 'http://wpt');
     const title = spec.getMeta().title;
     let { initScript } = this;
 
@@ -704,20 +802,12 @@ class WPTRunner {
       const absolutePath = spec.getAbsolutePath();
       const relativePath = spec.getRelativePath();
       const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
-      // *.worker.js tests are dedicated worker tests by definition.
-      // Multi-global (*.any.js) tests whose global scopes include a
-      // dedicated worker but not a window also run inside an actual Web
-      // Worker, like the .any.worker.html variant generated by the WPT
-      // server does. Refs:
+      // *.worker.js tests are dedicated worker tests by definition. Each
+      // dedicated worker variant generated from a multi-global (*.any.js)
+      // test also runs inside an actual Web Worker. Refs:
       // https://web-platform-tests.org/writing-tests/testharness.html#multi-global-tests
-      const isAnyTest = /\.any\.js$/.test(spec.filename);
-      const globalScopes = isAnyTest ?
-        (meta.global?.split(',').map((s) => s.trim()) ??
-          ['window', 'dedicatedworker']) : [];
-      const isWebWorkerTest = /\.worker\.js$/.test(spec.filename) ||
-        (isAnyTest && !globalScopes.includes('window') &&
-         (globalScopes.includes('worker') ||
-          globalScopes.includes('dedicatedworker')));
+      const isAnyTest = spec.isAnyTest();
+      const isWebWorkerTest = spec.isWebWorkerTest();
 
       // Scripts specified with the `// META: script=` header. For tests
       // that run inside a Web Worker they are imported by the worker
@@ -757,6 +847,8 @@ class WPTRunner {
             webWorker: isWebWorkerTest ? {
               path: absolutePath,
               isAnyTest,
+              initScript: this.initScript,
+              variant: spec.variant,
               scripts: meta.script?.map(
                 (script) => this.resource.toRealFilePath(relativePath, script),
               ) ?? [],
@@ -832,45 +924,16 @@ class WPTRunner {
 
       const failures = [];
       let expectedFailures = 0;
-      let skipped = 0;
       for (const [key, item] of Object.entries(this.results)) {
         if (item.fail?.unexpected) {
           failures.push(key);
         }
         if (item.fail?.expected) {
           expectedFailures++;
         }
-        if (item.skip) {
-          skipped++;
-        }
       }
 
-      const unexpectedPasses = [];
-      for (const specs of queue) {
-        const key = specs.filename;
-
-        // File has no expected failures
-        if (!specs.failedTests.length) {
-          continue;
-        }
-
-        // File was (maybe even conditionally) skipped
-        if (this.results[key]?.skip) {
-          continue;
-        }
-
-        // Full check: every expected to fail test is present
-        const _unexpectedPasses = specs.failedTests.filter((expectedToFail) => {
-          if (specs.flakyTests.includes(expectedToFail)) {
-            return false;
-          }
-          return this.results[key]?.fail?.expected?.includes(expectedToFail) !== true;
-        });
-        if (_unexpectedPasses.length) {
-          unexpectedPasses.push(..._unexpectedPasses.map((name) => `${key}:${name}`));
-          continue;
-        }
-      }
+      const unexpectedPasses = getUnexpectedPasses(queue, this.results);
 
       // Write the report on clean exit. The report is also written
       // incrementally after each spec completes (see completionCallback)
@@ -879,6 +942,7 @@ class WPTRunner {
 
       const p = (n, word, suffix = 's') => `${n} ${word}${n === 1 ? '' : suffix}`;
       const ran = queue.length;
+      const skipped = this.skippedSpecCount;
       const total = ran + skipped;
       const passed = ran - expectedFailures - failures.length;
       const { subtestCounts } = this;
@@ -976,8 +1040,9 @@ class WPTRunner {
   }
 
   addTestResult(spec, item) {
-    let result = this.results[spec.filename];
-    result ||= this.results[spec.filename] = {};
+    const key = spec.getStatusKey();
+    let result = this.results[key];
+    result ||= this.results[key] = {};
     if (item.status === kSkip) {
       if (item.name) {
         // Subtest-level skip: { filename: { skipTests: [ ... ] } }
@@ -1056,7 +1121,8 @@ class WPTRunner {
 
   skip(spec, reasons) {
     const joinedReasons = reasons.join('; ');
-    console.log(`[SKIPPED] ${spec.filename}${spec.variant}: ${joinedReasons}`);
+    console.log(`[SKIPPED] ${spec.getTestPath()}: ${joinedReasons}`);
+    this.skippedSpecCount++;
     this.addTestResult(spec, {
       status: kSkip,
       reason: joinedReasons,
@@ -1065,6 +1131,7 @@ class WPTRunner {
 
   buildQueue() {
     const queue = [];
+    this.skippedSpecCount = 0;
     let argFilename;
     let argVariant;
     if (process.argv[2]) {
@@ -1105,7 +1172,9 @@ class WPTRunner {
 }
 
 module.exports = {
+  getUnexpectedPasses,
   harness: harnessMock,
   ResourceLoader,
+  WPTTestSpec,
   WPTRunner,
 };

test/common/wpt/webworker.js

@@ -4,6 +4,10 @@
 // Refs: https://web-platform-tests.org/writing-tests/testharness.html
 
 const { pathToFileURL } = require('url');
+const {
+  runInThisContext,
+  constants: { USE_MAIN_CONTEXT_DEFAULT_LOADER },
+} = require('vm');
 
 globalThis.onmessage = ({ data }) => {
   // Let the test install its own handler.
@@ -12,12 +16,16 @@ globalThis.onmessage = ({ data }) => {
   const { ResourceLoader } = require(data.wptRunner);
   const resource = new ResourceLoader(data.wptPath);
 
+  globalThis.fetch = function fetch(file) {
+    return resource.readAsFetch(data.testRelativePath, file);
+  };
+
   // Pretend the worker was served from the URL the WPT server would have
   // used
   const fakePath = (data.isAnyTest ?
     data.testRelativePath.replace(/\.any\.js$/, '.any.worker.js') :
     data.testRelativePath).replace(/\\/g, '/');
-  const fakeURL = new URL(`/${fakePath}`, 'http://wpt');
+  const fakeURL = new URL(`/${fakePath}${data.variant}`, 'http://wpt');
   // eslint-disable-next-line no-undef
   const fakeLocation = { __proto__: WorkerLocation.prototype };
   for (const key of ['href', 'origin', 'protocol', 'host', 'hostname',
@@ -98,6 +106,15 @@ globalThis.onmessage = ({ data }) => {
       isWorker() { return true; },
       isShadowRealm() { return false; },
     };
+  }
+
+  if (data.initScript) {
+    runInThisContext(data.initScript, {
+      importModuleDynamically: USE_MAIN_CONTEXT_DEFAULT_LOADER,
+    });
+  }
+
+  if (data.isAnyTest) {
     globalThis.importScripts('/resources/testharness.js');
     for (const script of data.scripts) {
       globalThis.importScripts(pathToFileURL(script).href);