diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 2c793fa05df0..133b53268d52 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -25,6 +25,37 @@ Or run only a single E2E test app: yarn test:run ``` +Or you can run a single E2E test app with a specific variant: + +```bash +yarn test:run --variant +``` + +Variant name matching is case-insensitive and partial. For example, `--variant 13` will match `nextjs-pages-dir (next@13)` if a matching variant is present in the test app's `package.json`. + +For example, if you have the following variants in your test app's `package.json`: + +```json +"sentryTest": { + "variants": [ + { + "build-command": "pnpm test:build-13", + "label": "nextjs-pages-dir (next@13)" + }, + { + "build-command": "pnpm test:build-13-canary", + "label": "nextjs-pages-dir (next@13-canary)" + }, + { + "build-command": "pnpm test:build-15", + "label": "nextjs-pages-dir (next@15)" + } + ] +} +``` + +If you run `yarn test:run nextjs-pages-dir --variant 13`, it will match against the very first matching variant, which is `nextjs-pages-dir (next@13)`. If you need to target the second variant in the example, you need to be more specific and use `--variant 13-canary`. + ## How they work Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index e0331f0694f8..443ccf806b73 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -1,13 +1,26 @@ /* eslint-disable no-console */ import { spawn } from 'child_process'; import * as dotenv from 'dotenv'; -import { mkdtemp, rm } from 'fs/promises'; +import { mkdtemp, readFile, rm } from 'fs/promises'; import { sync as globSync } from 'glob'; import { tmpdir } from 'os'; import { join, resolve } from 'path'; import { copyToTemp } from './lib/copyToTemp'; import { registrySetup } from './registrySetup'; +interface SentryTestVariant { + 'build-command': string; + 'assert-command'?: string; + label?: string; +} + +interface PackageJson { + sentryTest?: { + variants?: SentryTestVariant[]; + optionalVariants?: SentryTestVariant[]; + }; +} + const DEFAULT_DSN = 'https://username@domain/123'; const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks'; const DEFAULT_SENTRY_PROJECT = 'sentry-javascript-e2e-tests'; @@ -58,6 +71,49 @@ function asyncExec( }); } +function findMatchingVariant(variants: SentryTestVariant[], variantLabel: string): SentryTestVariant | undefined { + const variantLabelLower = variantLabel.toLowerCase(); + + return variants.find(variant => variant.label?.toLowerCase().includes(variantLabelLower)); +} + +async function getVariantBuildCommand( + packageJsonPath: string, + variantLabel: string, + testAppPath: string, +): Promise<{ buildCommand: string; assertCommand: string; testLabel: string; matchedVariantLabel?: string }> { + try { + const packageJsonContent = await readFile(packageJsonPath, 'utf-8'); + const packageJson: PackageJson = JSON.parse(packageJsonContent); + + const allVariants = [ + ...(packageJson.sentryTest?.variants || []), + ...(packageJson.sentryTest?.optionalVariants || []), + ]; + + const matchingVariant = findMatchingVariant(allVariants, variantLabel); + + if (matchingVariant) { + return { + buildCommand: matchingVariant['build-command'] || 'pnpm test:build', + assertCommand: matchingVariant['assert-command'] || 'pnpm test:assert', + testLabel: matchingVariant.label || testAppPath, + matchedVariantLabel: matchingVariant.label, + }; + } + + console.log(`No matching variant found for "${variantLabel}" in ${testAppPath}, using default build`); + } catch { + console.log(`Could not read variants from package.json for ${testAppPath}, using default build`); + } + + return { + buildCommand: 'pnpm test:build', + assertCommand: 'pnpm test:assert', + testLabel: testAppPath, + }; +} + async function run(): Promise { // Load environment variables from .env file locally dotenv.config(); @@ -65,7 +121,50 @@ async function run(): Promise { // Allow to run a single app only via `yarn test:run ` const appName = process.argv[2] || ''; // Forward any additional flags to the test command - const testFlags = process.argv.slice(3); + const allTestFlags = process.argv.slice(3); + + // Check for --variant flag + let variantLabel: string | undefined; + let skipNextFlag = false; + + const testFlags = allTestFlags.filter((flag, index) => { + // Skip this flag if it was marked to skip (variant value after --variant) + if (skipNextFlag) { + skipNextFlag = false; + return false; + } + + // Handle --variant= format + if (flag.startsWith('--variant=')) { + const value = flag.slice('--variant='.length); + const trimmedValue = value?.trim(); + if (trimmedValue) { + variantLabel = trimmedValue; + } else { + console.warn('Warning: --variant= specified but no value provided. Ignoring variant flag.'); + } + return false; // Remove this flag from testFlags + } + + // Handle --variant format + if (flag === '--variant') { + if (index + 1 < allTestFlags.length) { + const value = allTestFlags[index + 1]; + const trimmedValue = value?.trim(); + if (trimmedValue) { + variantLabel = trimmedValue; + skipNextFlag = true; // Mark next flag to be skipped + } else { + console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.'); + } + } else { + console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.'); + } + return false; + } + + return true; + }); const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN; @@ -107,13 +206,42 @@ async function run(): Promise { await copyToTemp(originalPath, tmpDirPath); const cwd = tmpDirPath; + // Resolve variant if needed + const { buildCommand, assertCommand, testLabel, matchedVariantLabel } = variantLabel + ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath) + : { + buildCommand: 'pnpm test:build', + assertCommand: 'pnpm test:assert', + testLabel: testAppPath, + }; + + // Print which variant we're using if found + if (matchedVariantLabel) { + console.log(`\n\nUsing variant: "${matchedVariantLabel}"\n\n`); + } - console.log(`Building ${testAppPath} in ${tmpDirPath}...`); - await asyncExec('volta run pnpm test:build', { env, cwd }); + console.log(`Building ${testLabel} in ${tmpDirPath}...`); + await asyncExec(`volta run ${buildCommand}`, { env, cwd }); - console.log(`Testing ${testAppPath}...`); - // Pass command and arguments as an array to prevent command injection - const testCommand = ['volta', 'run', 'pnpm', 'test:assert', ...testFlags]; + console.log(`Testing ${testLabel}...`); + // Pass command as a string to support shell features (env vars, operators like &&) + // This matches how buildCommand is handled for consistency + // Properly quote test flags to preserve spaces and special characters + const quotedTestFlags = testFlags.map(flag => { + // If flag contains spaces or special shell characters, quote it + if ( + flag.includes(' ') || + flag.includes('"') || + flag.includes("'") || + flag.includes('$') || + flag.includes('`') + ) { + // Escape single quotes and wrap in single quotes (safest for shell) + return `'${flag.replace(/'/g, "'\\''")}'`; + } + return flag; + }); + const testCommand = `volta run ${assertCommand}${quotedTestFlags.length > 0 ? ` ${quotedTestFlags.join(' ')}` : ''}`; await asyncExec(testCommand, { env, cwd }); // clean up (although this is tmp, still nice to do)