chore(tests): Added variant flag

Author: logaretm

dev-packages/e2e-tests/README.md

@@ -25,6 +25,14 @@ Or run only a single E2E test app:
 yarn test:run <app-name>
 ```
 
+Or you can run a single E2E test app with a specific variant:
+
+```bash
+yarn test:run <app-name> --variant <variant-name>
+```
+
+Variant name matching is case-insensitive and partial. For example, `--variant 13` will match `nextjs-pages-dir (next@13)` if a matching variant is present in the test app's `package.json`.
+
 ## How they work
 
 Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we

dev-packages/e2e-tests/run.ts

@@ -1,13 +1,25 @@
 /* eslint-disable no-console */
 import { spawn } from 'child_process';
 import * as dotenv from 'dotenv';
-import { mkdtemp, rm } from 'fs/promises';
+import { mkdtemp, readFile, rm } from 'fs/promises';
 import { sync as globSync } from 'glob';
 import { tmpdir } from 'os';
 import { join, resolve } from 'path';
 import { copyToTemp } from './lib/copyToTemp';
 import { registrySetup } from './registrySetup';
 
+interface SentryTestVariant {
+  'build-command': string;
+  label: string;
+}
+
+interface PackageJson {
+  sentryTest?: {
+    variants?: SentryTestVariant[];
+    optionalVariants?: SentryTestVariant[];
+  };
+}
+
 const DEFAULT_DSN = 'https://username@domain/123';
 const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks';
 const DEFAULT_SENTRY_PROJECT = 'sentry-javascript-e2e-tests';
@@ -58,14 +70,84 @@ function asyncExec(
   });
 }
 
+function findMatchingVariant(variants: SentryTestVariant[], variantLabel: string): SentryTestVariant | undefined {
+  const variantLabelLower = variantLabel.toLowerCase();
+
+  return variants.find(variant => variant.label.toLowerCase().includes(variantLabelLower));
+}
+
+async function getVariantBuildCommand(
+  packageJsonPath: string,
+  variantLabel: string,
+  testAppPath: string,
+): Promise<{ buildCommand: string; testLabel: string; matchedVariantLabel?: string }> {
+  try {
+    const packageJsonContent = await readFile(packageJsonPath, 'utf-8');
+    const packageJson: PackageJson = JSON.parse(packageJsonContent);
+
+    const allVariants = [
+      ...(packageJson.sentryTest?.variants || []),
+      ...(packageJson.sentryTest?.optionalVariants || []),
+    ];
+
+    const matchingVariant = findMatchingVariant(allVariants, variantLabel);
+
+    if (matchingVariant) {
+      return {
+        buildCommand: `volta run ${matchingVariant['build-command']}`,
+        testLabel: matchingVariant.label,
+        matchedVariantLabel: matchingVariant.label,
+      };
+    }
+
+    console.log(`No matching variant found for "${variantLabel}" in ${testAppPath}, using default build`);
+  } catch (error) {
+    console.log(`Could not read variants from package.json for ${testAppPath}, using default build`);
+  }
+
+  return {
+    buildCommand: 'volta run pnpm test:build',
+    testLabel: testAppPath,
+  };
+}
+
 async function run(): Promise<void> {
   // Load environment variables from .env file locally
   dotenv.config();
 
   // Allow to run a single app only via `yarn test:run <app-name>`
   const appName = process.argv[2] || '';
   // Forward any additional flags to the test command
-  const testFlags = process.argv.slice(3);
+  const allTestFlags = process.argv.slice(3);
+
+  // Check for --variant flag
+  let variantLabel: string | undefined;
+  let skipNextFlag = false;
+
+  const testFlags = allTestFlags.filter((flag, index) => {
+    // Skip this flag if it was marked to skip (variant value after --variant)
+    if (skipNextFlag) {
+      skipNextFlag = false;
+      return false;
+    }
+
+    // Handle --variant=<value> format
+    if (flag.startsWith('--variant=')) {
+      variantLabel = flag.split('=')[1];
+      return false; // Remove this flag from testFlags
+    }
+
+    // Handle --variant <value> format
+    if (flag === '--variant') {
+      if (index + 1 < allTestFlags.length) {
+        variantLabel = allTestFlags[index + 1];
+        skipNextFlag = true; // Mark next flag to be skipped
+      }
+      return false;
+    }
+
+    return true;
+  });
 
   const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN;
 
@@ -107,11 +189,20 @@ async function run(): Promise<void> {
 
       await copyToTemp(originalPath, tmpDirPath);
       const cwd = tmpDirPath;
+      // Resolve variant if needed
+      const { buildCommand, testLabel, matchedVariantLabel } = variantLabel
+        ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath)
+        : { buildCommand: 'volta run pnpm test:build', testLabel: testAppPath };
+
+      // Print which variant we're using if found
+      if (matchedVariantLabel) {
+        console.log(`Using variant: "${matchedVariantLabel}"`);
+      }
 
-      console.log(`Building ${testAppPath} in ${tmpDirPath}...`);
-      await asyncExec('volta run pnpm test:build', { env, cwd });
+      console.log(`Building ${testLabel} in ${tmpDirPath}...`);
+      await asyncExec(buildCommand, { env, cwd });
 
-      console.log(`Testing ${testAppPath}...`);
+      console.log(`Testing ${testLabel}...`);
       // Pass command and arguments as an array to prevent command injection
       const testCommand = ['volta', 'run', 'pnpm', 'test:assert', ...testFlags];
       await asyncExec(testCommand, { env, cwd });