diff --git a/package-lock.json b/package-lock.json index 819ed4d..5cac6f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@computesdk/s3": "^1.1.1", "@computesdk/sprites": "^0.1.5", "@computesdk/tigris": "^1.1.1", - "@computesdk/vercel": "^1.7.21", + "@computesdk/vercel": "1.7.15", "computesdk": "^2.5.4", "dotenv": "^17.4.0", "playwright-core": "^1.59.1" @@ -1401,17 +1401,36 @@ } }, "node_modules/@computesdk/vercel": { - "version": "1.7.21", - "resolved": "https://registry.npmjs.org/@computesdk/vercel/-/vercel-1.7.21.tgz", - "integrity": "sha512-PCHOqpR32qM9bQ72Wbx5c63AdsKtVeR54bhIimKaRNLDR7whMZs2guBNroKxNCcdxfOZhOVBpOR8xlh0I/YMTg==", + "version": "1.7.15", + "resolved": "https://registry.npmjs.org/@computesdk/vercel/-/vercel-1.7.15.tgz", + "integrity": "sha512-QRmQSSa5Rr1/IlbxY0uPf8tTEod72zkxhdp2+zwsHVfVVfSABo5bFDWJbQa/ns9NzKC+ZNC5cBo4pQlivxHNtg==", "license": "MIT", "dependencies": { - "@computesdk/provider": "1.1.0", + "@computesdk/provider": "1.0.28", "@vercel/sandbox": "^1.6.0", - "computesdk": "2.5.4", + "computesdk": "2.3.0", "ms": "^2.1.3" } }, + "node_modules/@computesdk/vercel/node_modules/@computesdk/provider": { + "version": "1.0.28", + "resolved": "https://registry.npmjs.org/@computesdk/provider/-/provider-1.0.28.tgz", + "integrity": "sha512-ecn/AFiLpr5GCU0r6VkwBrwYnFULIoJ2K1oiJ7TBqj0pAYmQsFgv8i5b4VRbKab1FGG5zCXthaFY/3liTYjzCg==", + "license": "MIT", + "dependencies": { + "@computesdk/cmd": "0.4.1", + "computesdk": "2.3.0" + } + }, + "node_modules/@computesdk/vercel/node_modules/computesdk": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/computesdk/-/computesdk-2.3.0.tgz", + "integrity": "sha512-4B7CRN2qB6XkuAnN7dZ0aMYqHaFrh2qdSuh02lM+cgMEQ7wZy9v44FAjBGfWebHXuPNA/nZRx7211U6CEiGdTw==", + "license": "MIT", + "dependencies": { + "@computesdk/cmd": "0.4.1" + } + }, "node_modules/@connectrpc/connect": { "version": "2.0.0-rc.3", "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.0.0-rc.3.tgz", diff --git a/package.json b/package.json index 1eccef1..6326356 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@computesdk/s3": "^1.1.1", "@computesdk/sprites": "^0.1.5", "@computesdk/tigris": "^1.1.1", - "@computesdk/vercel": "^1.7.21", + "@computesdk/vercel": "1.7.15", "computesdk": "^2.5.4", "dotenv": "^17.4.0", "playwright-core": "^1.59.1" diff --git a/src/browser/benchmark.ts b/src/browser/benchmark.ts index 006ff96..ae9df9e 100644 --- a/src/browser/benchmark.ts +++ b/src/browser/benchmark.ts @@ -135,24 +135,6 @@ export async function runBrowserBenchmark(config: BrowserProviderConfig): Promis const successful = results.filter(r => !r.error); - // If every iteration failed, mark as skipped - if (successful.length === 0) { - return { - provider: name, - mode: 'browser', - iterations: results, - summary: { - createMs: { median: 0, p95: 0, p99: 0 }, - connectMs: { median: 0, p95: 0, p99: 0 }, - navigateMs: { median: 0, p95: 0, p99: 0 }, - releaseMs: { median: 0, p95: 0, p99: 0 }, - totalMs: { median: 0, p95: 0, p99: 0 }, - }, - skipped: true, - skipReason: 'All iterations failed', - }; - } - return { provider: name, mode: 'browser', diff --git a/src/merge-results.ts b/src/merge-results.ts index 18bbe32..a75d651 100644 --- a/src/merge-results.ts +++ b/src/merge-results.ts @@ -184,10 +184,6 @@ function printStorageResultsTable(results: StorageBenchmarkResult[], fileSize: s } const ok = r.iterations.filter(i => !i.error).length; const total = r.iterations.length; - if (ok === 0 && total > 0) { - console.log([r.provider.padEnd(14), '--'.padEnd(8), '--'.padEnd(14), '--'.padEnd(14), '--'.padEnd(14), 'FAILED'.padEnd(10)].join(' | ')); - continue; - } const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; const dl = (r.summary.downloadMs.median / 1000).toFixed(2) + 's'; const tp = r.summary.throughputMbps.median.toFixed(1) + ' Mbps'; @@ -300,10 +296,6 @@ function printBrowserResultsTable(results: BrowserBenchmarkResult[]): void { } const ok = r.iterations.filter(i => !i.error).length; const total = r.iterations.length; - if (ok === 0 && total > 0) { - console.log([r.provider.padEnd(14), '--'.padEnd(8), '--'.padEnd(12), '--'.padEnd(12), '--'.padEnd(12), '--'.padEnd(12), '--'.padEnd(12), 'FAILED'.padEnd(10)].join(' | ')); - continue; - } const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; const create = (r.summary.createMs.median / 1000).toFixed(2) + 's'; const connect = (r.summary.connectMs.median / 1000).toFixed(2) + 's'; diff --git a/src/run.ts b/src/run.ts index 67c1263..160b810 100644 --- a/src/run.ts +++ b/src/run.ts @@ -150,9 +150,11 @@ async function runStorage(toRun: typeof storageProviders, fileSizeLabel: string) console.log(`${r.provider}: SKIPPED (${r.skipReason})`); continue; } + const ok = r.iterations.filter(i => !i.error).length; + const total = r.iterations.length; console.log(`${r.provider}:`); console.log(` Download: ${(r.summary.downloadMs.median / 1000).toFixed(2)}s (median), ${r.summary.throughputMbps.median.toFixed(2)} Mbps`); - console.log(` Score: ${r.compositeScore?.toFixed(1) || '--'}`); + console.log(` Score: ${r.compositeScore?.toFixed(1) || '--'} (${ok}/${total} OK)`); } // Write JSON results to storage subdirectory with file size @@ -193,9 +195,11 @@ async function runBrowser(toRun: typeof browserProviders): Promise { console.log(`${r.provider}: SKIPPED (${r.skipReason})`); continue; } + const ok = r.iterations.filter(i => !i.error).length; + const total = r.iterations.length; console.log(`${r.provider}:`); console.log(` Total: ${(r.summary.totalMs.median / 1000).toFixed(2)}s (median) — create ${(r.summary.createMs.median / 1000).toFixed(2)}s + connect ${(r.summary.connectMs.median / 1000).toFixed(2)}s + navigate ${(r.summary.navigateMs.median / 1000).toFixed(2)}s + release ${(r.summary.releaseMs.median / 1000).toFixed(2)}s`); - console.log(` Score: ${r.compositeScore?.toFixed(1) || '--'}`); + console.log(` Score: ${r.compositeScore?.toFixed(1) || '--'} (${ok}/${total} OK)`); } // Write JSON results to browser subdirectory diff --git a/src/sandbox/benchmark.ts b/src/sandbox/benchmark.ts index b51af0a..fcb73bf 100644 --- a/src/sandbox/benchmark.ts +++ b/src/sandbox/benchmark.ts @@ -38,22 +38,13 @@ export async function runBenchmark(config: ProviderConfig): Promise !r.error); - // If every iteration failed, mark as skipped so it doesn't show 0.00s - if (successful.length === 0) { - return { - provider: name, - iterations: results, - summary: { ttiMs: { median: 0, p95: 0, p99: 0 } }, - skipped: true, - skipReason: 'All iterations failed', - }; - } - return { provider: name, iterations: results, summary: { - ttiMs: computeStats(successful.map(r => r.ttiMs)), + ttiMs: successful.length > 0 + ? computeStats(successful.map(r => r.ttiMs)) + : { median: 0, p95: 0, p99: 0 }, }, }; } diff --git a/src/sandbox/concurrent.ts b/src/sandbox/concurrent.ts index 9183592..16b2650 100644 --- a/src/sandbox/concurrent.ts +++ b/src/sandbox/concurrent.ts @@ -50,22 +50,8 @@ export async function runConcurrentBenchmark(config: ConcurrentConfig): Promise< const successful = results.filter(r => !r.error); - if (successful.length === 0) { - return { - provider: name, - mode: 'concurrent', - concurrency, - iterations: results, - summary: { ttiMs: { median: 0, p95: 0, p99: 0 } }, - wallClockMs, - timeToFirstReadyMs: 0, - skipped: true, - skipReason: 'All sandboxes failed', - }; - } - const successfulTimes = successful.map(r => r.ttiMs); - const timeToFirstReadyMs = Math.min(...successfulTimes); + const timeToFirstReadyMs = successful.length > 0 ? Math.min(...successfulTimes) : 0; console.log(` Wall clock: ${(wallClockMs / 1000).toFixed(2)}s | First ready: ${(timeToFirstReadyMs / 1000).toFixed(2)}s | Success: ${successful.length}/${concurrency}`); @@ -75,7 +61,9 @@ export async function runConcurrentBenchmark(config: ConcurrentConfig): Promise< concurrency, iterations: results, summary: { - ttiMs: computeStats(successfulTimes), + ttiMs: successful.length > 0 + ? computeStats(successfulTimes) + : { median: 0, p95: 0, p99: 0 }, }, wallClockMs, timeToFirstReadyMs, diff --git a/src/sandbox/staggered.ts b/src/sandbox/staggered.ts index b3c885c..78e63af 100644 --- a/src/sandbox/staggered.ts +++ b/src/sandbox/staggered.ts @@ -65,24 +65,8 @@ export async function runStaggeredBenchmark(config: StaggeredConfig): Promise !r.error); - if (successful.length === 0) { - return { - provider: name, - mode: 'staggered', - concurrency, - staggerDelayMs, - iterations: results, - summary: { ttiMs: { median: 0, p95: 0, p99: 0 } }, - wallClockMs, - timeToFirstReadyMs: 0, - rampProfile: rampProfile.sort((a, b) => a.launchedAt - b.launchedAt), - skipped: true, - skipReason: 'All sandboxes failed', - }; - } - const successfulTimes = successful.map(r => r.ttiMs); - const timeToFirstReadyMs = Math.min(...successfulTimes); + const timeToFirstReadyMs = successful.length > 0 ? Math.min(...successfulTimes) : 0; console.log(` Wall clock: ${(wallClockMs / 1000).toFixed(2)}s | First ready: ${(timeToFirstReadyMs / 1000).toFixed(2)}s | Success: ${successful.length}/${concurrency}`); @@ -93,7 +77,9 @@ export async function runStaggeredBenchmark(config: StaggeredConfig): Promise 0 + ? computeStats(successfulTimes) + : { median: 0, p95: 0, p99: 0 }, }, wallClockMs, timeToFirstReadyMs, diff --git a/src/sandbox/table.ts b/src/sandbox/table.ts index b251db4..ecf72ae 100644 --- a/src/sandbox/table.ts +++ b/src/sandbox/table.ts @@ -47,22 +47,6 @@ export function printResultsTable(results: BenchmarkResult[]): void { const successful = result.iterations.filter(r => !r.error).length; const total = result.iterations.length; - // If all iterations failed, show as FAILED with error reason - if (successful === 0 && total > 0) { - const firstError = result.iterations.find(r => r.error)?.error || 'Unknown error'; - const shortError = firstError.length > 40 ? firstError.slice(0, 37) + '...' : firstError; - console.log([ - pad(result.provider, nameWidth), - pad('--', 8), - pad('--', colWidth), - pad('--', colWidth), - pad('--', colWidth), - pad(`FAILED`, 10), - ].join(' | ')); - continue; - } - - // Truly skipped (missing env vars, etc.) if (result.skipped) { console.log([ pad(result.provider, nameWidth), diff --git a/src/storage/benchmark.ts b/src/storage/benchmark.ts index afad131..dfd4256 100644 --- a/src/storage/benchmark.ts +++ b/src/storage/benchmark.ts @@ -144,24 +144,6 @@ export async function runStorageBenchmark(config: StorageProviderConfig, fileSiz const successful = results.filter(r => !r.error); - // If every iteration failed, mark as skipped - if (successful.length === 0) { - return { - provider: name, - mode: 'storage', - bucket, - fileSizeBytes, - iterations: results, - summary: { - uploadMs: { median: 0, p95: 0, p99: 0 }, - downloadMs: { median: 0, p95: 0, p99: 0 }, - throughputMbps: { median: 0, p95: 0, p99: 0 }, - }, - skipped: true, - skipReason: 'All iterations failed', - }; - } - const uploadTimes = successful.map(r => r.uploadMs); const downloadTimes = successful.map(r => r.downloadMs); const throughputs = successful.map(r => r.throughputMbps);