From e52d51e0ab55265ca701e472ed5bfab4c34a18fe Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 14:02:50 +0200 Subject: [PATCH 1/7] chore(tests): Added variant flag --- dev-packages/e2e-tests/README.md | 8 +++ dev-packages/e2e-tests/run.ts | 101 +++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 2c793fa05df0..906149b2358d 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -25,6 +25,14 @@ Or run only a single E2E test app: yarn test:run ``` +Or you can run a single E2E test app with a specific variant: + +```bash +yarn test:run --variant +``` + +Variant name matching is case-insensitive and partial. For example, `--variant 13` will match `nextjs-pages-dir (next@13)` if a matching variant is present in the test app's `package.json`. + ## How they work Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index e0331f0694f8..62c6eee50944 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -1,13 +1,25 @@ /* eslint-disable no-console */ import { spawn } from 'child_process'; import * as dotenv from 'dotenv'; -import { mkdtemp, rm } from 'fs/promises'; +import { mkdtemp, readFile, rm } from 'fs/promises'; import { sync as globSync } from 'glob'; import { tmpdir } from 'os'; import { join, resolve } from 'path'; import { copyToTemp } from './lib/copyToTemp'; import { registrySetup } from './registrySetup'; +interface SentryTestVariant { + 'build-command': string; + label: string; +} + +interface PackageJson { + sentryTest?: { + variants?: SentryTestVariant[]; + optionalVariants?: SentryTestVariant[]; + }; +} + const DEFAULT_DSN = 'https://username@domain/123'; const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks'; const DEFAULT_SENTRY_PROJECT = 'sentry-javascript-e2e-tests'; @@ -58,6 +70,47 @@ function asyncExec( }); } +function findMatchingVariant(variants: SentryTestVariant[], variantLabel: string): SentryTestVariant | undefined { + const variantLabelLower = variantLabel.toLowerCase(); + + return variants.find(variant => variant.label.toLowerCase().includes(variantLabelLower)); +} + +async function getVariantBuildCommand( + packageJsonPath: string, + variantLabel: string, + testAppPath: string, +): Promise<{ buildCommand: string; testLabel: string; matchedVariantLabel?: string }> { + try { + const packageJsonContent = await readFile(packageJsonPath, 'utf-8'); + const packageJson: PackageJson = JSON.parse(packageJsonContent); + + const allVariants = [ + ...(packageJson.sentryTest?.variants || []), + ...(packageJson.sentryTest?.optionalVariants || []), + ]; + + const matchingVariant = findMatchingVariant(allVariants, variantLabel); + + if (matchingVariant) { + return { + buildCommand: `volta run ${matchingVariant['build-command']}`, + testLabel: matchingVariant.label, + matchedVariantLabel: matchingVariant.label, + }; + } + + console.log(`No matching variant found for "${variantLabel}" in ${testAppPath}, using default build`); + } catch (error) { + console.log(`Could not read variants from package.json for ${testAppPath}, using default build`); + } + + return { + buildCommand: 'volta run pnpm test:build', + testLabel: testAppPath, + }; +} + async function run(): Promise { // Load environment variables from .env file locally dotenv.config(); @@ -65,7 +118,36 @@ async function run(): Promise { // Allow to run a single app only via `yarn test:run ` const appName = process.argv[2] || ''; // Forward any additional flags to the test command - const testFlags = process.argv.slice(3); + const allTestFlags = process.argv.slice(3); + + // Check for --variant flag + let variantLabel: string | undefined; + let skipNextFlag = false; + + const testFlags = allTestFlags.filter((flag, index) => { + // Skip this flag if it was marked to skip (variant value after --variant) + if (skipNextFlag) { + skipNextFlag = false; + return false; + } + + // Handle --variant= format + if (flag.startsWith('--variant=')) { + variantLabel = flag.split('=')[1]; + return false; // Remove this flag from testFlags + } + + // Handle --variant format + if (flag === '--variant') { + if (index + 1 < allTestFlags.length) { + variantLabel = allTestFlags[index + 1]; + skipNextFlag = true; // Mark next flag to be skipped + } + return false; + } + + return true; + }); const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN; @@ -107,11 +189,20 @@ async function run(): Promise { await copyToTemp(originalPath, tmpDirPath); const cwd = tmpDirPath; + // Resolve variant if needed + const { buildCommand, testLabel, matchedVariantLabel } = variantLabel + ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath) + : { buildCommand: 'volta run pnpm test:build', testLabel: testAppPath }; + + // Print which variant we're using if found + if (matchedVariantLabel) { + console.log(`Using variant: "${matchedVariantLabel}"`); + } - console.log(`Building ${testAppPath} in ${tmpDirPath}...`); - await asyncExec('volta run pnpm test:build', { env, cwd }); + console.log(`Building ${testLabel} in ${tmpDirPath}...`); + await asyncExec(buildCommand, { env, cwd }); - console.log(`Testing ${testAppPath}...`); + console.log(`Testing ${testLabel}...`); // Pass command and arguments as an array to prevent command injection const testCommand = ['volta', 'run', 'pnpm', 'test:assert', ...testFlags]; await asyncExec(testCommand, { env, cwd }); From ad3cb010fe3a8187522ab04de794d78fea74e754 Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 15:49:43 +0200 Subject: [PATCH 2/7] fix: cover custom variant assertion commands --- dev-packages/e2e-tests/run.ts | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 62c6eee50944..9e9b52c099a9 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -10,6 +10,7 @@ import { registrySetup } from './registrySetup'; interface SentryTestVariant { 'build-command': string; + 'assert-command'?: string; label: string; } @@ -80,7 +81,7 @@ async function getVariantBuildCommand( packageJsonPath: string, variantLabel: string, testAppPath: string, -): Promise<{ buildCommand: string; testLabel: string; matchedVariantLabel?: string }> { +): Promise<{ buildCommand: string; assertCommand: string; testLabel: string; matchedVariantLabel?: string }> { try { const packageJsonContent = await readFile(packageJsonPath, 'utf-8'); const packageJson: PackageJson = JSON.parse(packageJsonContent); @@ -95,18 +96,22 @@ async function getVariantBuildCommand( if (matchingVariant) { return { buildCommand: `volta run ${matchingVariant['build-command']}`, + assertCommand: matchingVariant['assert-command'] + ? `volta run ${matchingVariant['assert-command']}` + : 'volta run pnpm test:assert', testLabel: matchingVariant.label, matchedVariantLabel: matchingVariant.label, }; } console.log(`No matching variant found for "${variantLabel}" in ${testAppPath}, using default build`); - } catch (error) { + } catch { console.log(`Could not read variants from package.json for ${testAppPath}, using default build`); } return { buildCommand: 'volta run pnpm test:build', + assertCommand: 'volta run pnpm test:assert', testLabel: testAppPath, }; } @@ -190,9 +195,13 @@ async function run(): Promise { await copyToTemp(originalPath, tmpDirPath); const cwd = tmpDirPath; // Resolve variant if needed - const { buildCommand, testLabel, matchedVariantLabel } = variantLabel + const { buildCommand, assertCommand, testLabel, matchedVariantLabel } = variantLabel ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath) - : { buildCommand: 'volta run pnpm test:build', testLabel: testAppPath }; + : { + buildCommand: 'volta run pnpm test:build', + assertCommand: 'volta run pnpm test:assert', + testLabel: testAppPath, + }; // Print which variant we're using if found if (matchedVariantLabel) { @@ -203,9 +212,11 @@ async function run(): Promise { await asyncExec(buildCommand, { env, cwd }); console.log(`Testing ${testLabel}...`); - // Pass command and arguments as an array to prevent command injection - const testCommand = ['volta', 'run', 'pnpm', 'test:assert', ...testFlags]; - await asyncExec(testCommand, { env, cwd }); + // Use the variant's assert command if available, otherwise use default + // Construct the full command as a string to handle environment variables (e.g., NODE_VERSION=20) + // and append test flags + const fullAssertCommand = testFlags.length > 0 ? `${assertCommand} ${testFlags.join(' ')}` : assertCommand; + await asyncExec(fullAssertCommand, { env, cwd }); // clean up (although this is tmp, still nice to do) await rm(tmpDirPath, { recursive: true }); From d2b18b7197f11aa0c12499d4683274f8af4a172a Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 15:52:41 +0200 Subject: [PATCH 3/7] fix: drop volta prefix --- dev-packages/e2e-tests/run.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 9e9b52c099a9..821f6e9a67cf 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -95,10 +95,8 @@ async function getVariantBuildCommand( if (matchingVariant) { return { - buildCommand: `volta run ${matchingVariant['build-command']}`, - assertCommand: matchingVariant['assert-command'] - ? `volta run ${matchingVariant['assert-command']}` - : 'volta run pnpm test:assert', + buildCommand: matchingVariant['build-command'] || 'pnpm test:build', + assertCommand: matchingVariant['assert-command'] || 'pnpm test:assert', testLabel: matchingVariant.label, matchedVariantLabel: matchingVariant.label, }; @@ -110,8 +108,8 @@ async function getVariantBuildCommand( } return { - buildCommand: 'volta run pnpm test:build', - assertCommand: 'volta run pnpm test:assert', + buildCommand: 'pnpm test:build', + assertCommand: 'pnpm test:assert', testLabel: testAppPath, }; } @@ -198,8 +196,8 @@ async function run(): Promise { const { buildCommand, assertCommand, testLabel, matchedVariantLabel } = variantLabel ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath) : { - buildCommand: 'volta run pnpm test:build', - assertCommand: 'volta run pnpm test:assert', + buildCommand: 'pnpm test:build', + assertCommand: 'pnpm test:assert', testLabel: testAppPath, }; @@ -212,9 +210,7 @@ async function run(): Promise { await asyncExec(buildCommand, { env, cwd }); console.log(`Testing ${testLabel}...`); - // Use the variant's assert command if available, otherwise use default - // Construct the full command as a string to handle environment variables (e.g., NODE_VERSION=20) - // and append test flags + // Append test flags to assert command const fullAssertCommand = testFlags.length > 0 ? `${assertCommand} ${testFlags.join(' ')}` : assertCommand; await asyncExec(fullAssertCommand, { env, cwd }); From a44ad64698be3bdceaaf9bcca80784a75b563380 Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 15:54:59 +0200 Subject: [PATCH 4/7] fix: variant flag validation --- dev-packages/e2e-tests/run.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 821f6e9a67cf..bd29ec5dbe13 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -136,15 +136,29 @@ async function run(): Promise { // Handle --variant= format if (flag.startsWith('--variant=')) { - variantLabel = flag.split('=')[1]; + const value = flag.split('=')[1]; + const trimmedValue = value?.trim(); + if (trimmedValue) { + variantLabel = value; + } else { + console.warn('Warning: --variant= specified but no value provided. Ignoring variant flag.'); + } return false; // Remove this flag from testFlags } // Handle --variant format if (flag === '--variant') { if (index + 1 < allTestFlags.length) { - variantLabel = allTestFlags[index + 1]; - skipNextFlag = true; // Mark next flag to be skipped + const value = allTestFlags[index + 1]; + const trimmedValue = value?.trim(); + if (trimmedValue) { + variantLabel = value; + skipNextFlag = true; // Mark next flag to be skipped + } else { + console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.'); + } + } else { + console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.'); } return false; } From bc5b9237981e3cd0ea7a1ce37b76a74134a57be7 Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 16:10:08 +0200 Subject: [PATCH 5/7] fix: bring back volta --- dev-packages/e2e-tests/run.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index bd29ec5dbe13..2fcd2b6959ef 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -139,7 +139,7 @@ async function run(): Promise { const value = flag.split('=')[1]; const trimmedValue = value?.trim(); if (trimmedValue) { - variantLabel = value; + variantLabel = trimmedValue; } else { console.warn('Warning: --variant= specified but no value provided. Ignoring variant flag.'); } @@ -152,7 +152,7 @@ async function run(): Promise { const value = allTestFlags[index + 1]; const trimmedValue = value?.trim(); if (trimmedValue) { - variantLabel = value; + variantLabel = trimmedValue; skipNextFlag = true; // Mark next flag to be skipped } else { console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.'); @@ -217,16 +217,16 @@ async function run(): Promise { // Print which variant we're using if found if (matchedVariantLabel) { - console.log(`Using variant: "${matchedVariantLabel}"`); + console.log(`\n\nUsing variant: "${matchedVariantLabel}"\n\n`); } console.log(`Building ${testLabel} in ${tmpDirPath}...`); - await asyncExec(buildCommand, { env, cwd }); + await asyncExec(`volta run ${buildCommand}`, { env, cwd }); console.log(`Testing ${testLabel}...`); - // Append test flags to assert command - const fullAssertCommand = testFlags.length > 0 ? `${assertCommand} ${testFlags.join(' ')}` : assertCommand; - await asyncExec(fullAssertCommand, { env, cwd }); + // Pass command and arguments as an array to prevent command injection + const testCommand = ['volta', 'run', ...assertCommand.split(' '), ...testFlags]; + await asyncExec(testCommand, { env, cwd }); // clean up (although this is tmp, still nice to do) await rm(tmpDirPath, { recursive: true }); From 86126e5c70b3c7eb2c0e0c5c6738f18a4fb32a3e Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Wed, 10 Dec 2025 17:26:51 +0200 Subject: [PATCH 6/7] fix: command construction --- dev-packages/e2e-tests/run.ts | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/dev-packages/e2e-tests/run.ts b/dev-packages/e2e-tests/run.ts index 2fcd2b6959ef..443ccf806b73 100644 --- a/dev-packages/e2e-tests/run.ts +++ b/dev-packages/e2e-tests/run.ts @@ -11,7 +11,7 @@ import { registrySetup } from './registrySetup'; interface SentryTestVariant { 'build-command': string; 'assert-command'?: string; - label: string; + label?: string; } interface PackageJson { @@ -74,7 +74,7 @@ function asyncExec( function findMatchingVariant(variants: SentryTestVariant[], variantLabel: string): SentryTestVariant | undefined { const variantLabelLower = variantLabel.toLowerCase(); - return variants.find(variant => variant.label.toLowerCase().includes(variantLabelLower)); + return variants.find(variant => variant.label?.toLowerCase().includes(variantLabelLower)); } async function getVariantBuildCommand( @@ -97,7 +97,7 @@ async function getVariantBuildCommand( return { buildCommand: matchingVariant['build-command'] || 'pnpm test:build', assertCommand: matchingVariant['assert-command'] || 'pnpm test:assert', - testLabel: matchingVariant.label, + testLabel: matchingVariant.label || testAppPath, matchedVariantLabel: matchingVariant.label, }; } @@ -136,7 +136,7 @@ async function run(): Promise { // Handle --variant= format if (flag.startsWith('--variant=')) { - const value = flag.split('=')[1]; + const value = flag.slice('--variant='.length); const trimmedValue = value?.trim(); if (trimmedValue) { variantLabel = trimmedValue; @@ -224,8 +224,24 @@ async function run(): Promise { await asyncExec(`volta run ${buildCommand}`, { env, cwd }); console.log(`Testing ${testLabel}...`); - // Pass command and arguments as an array to prevent command injection - const testCommand = ['volta', 'run', ...assertCommand.split(' '), ...testFlags]; + // Pass command as a string to support shell features (env vars, operators like &&) + // This matches how buildCommand is handled for consistency + // Properly quote test flags to preserve spaces and special characters + const quotedTestFlags = testFlags.map(flag => { + // If flag contains spaces or special shell characters, quote it + if ( + flag.includes(' ') || + flag.includes('"') || + flag.includes("'") || + flag.includes('$') || + flag.includes('`') + ) { + // Escape single quotes and wrap in single quotes (safest for shell) + return `'${flag.replace(/'/g, "'\\''")}'`; + } + return flag; + }); + const testCommand = `volta run ${assertCommand}${quotedTestFlags.length > 0 ? ` ${quotedTestFlags.join(' ')}` : ''}`; await asyncExec(testCommand, { env, cwd }); // clean up (although this is tmp, still nice to do) From cf891169f1f066a9c4bcce05b568331eb1e1bed4 Mon Sep 17 00:00:00 2001 From: Abdelrahman Awad Date: Thu, 11 Dec 2025 16:06:01 +0200 Subject: [PATCH 7/7] docs: added an example --- dev-packages/e2e-tests/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index 906149b2358d..133b53268d52 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -33,6 +33,29 @@ yarn test:run --variant Variant name matching is case-insensitive and partial. For example, `--variant 13` will match `nextjs-pages-dir (next@13)` if a matching variant is present in the test app's `package.json`. +For example, if you have the following variants in your test app's `package.json`: + +```json +"sentryTest": { + "variants": [ + { + "build-command": "pnpm test:build-13", + "label": "nextjs-pages-dir (next@13)" + }, + { + "build-command": "pnpm test:build-13-canary", + "label": "nextjs-pages-dir (next@13-canary)" + }, + { + "build-command": "pnpm test:build-15", + "label": "nextjs-pages-dir (next@15)" + } + ] +} +``` + +If you run `yarn test:run nextjs-pages-dir --variant 13`, it will match against the very first matching variant, which is `nextjs-pages-dir (next@13)`. If you need to target the second variant in the example, you need to be more specific and use `--variant 13-canary`. + ## How they work Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we