|
| 1 | +#!/usr/bin/env node |
| 2 | +// Runs the bench_test.js scenario suite N times against each engine's config, timing the whole |
| 3 | +// `codecept run` process wall-clock (helper/browser startup included, since that is part of |
| 4 | +// real-world speed) and reporting the median per engine plus the ratio between them. |
| 5 | +import { spawnSync } from 'child_process' |
| 6 | +import { fileURLToPath } from 'url' |
| 7 | +import { dirname, join } from 'path' |
| 8 | + |
| 9 | +const __dirname = dirname(fileURLToPath(import.meta.url)) |
| 10 | +const repoRoot = join(__dirname, '..', '..') |
| 11 | +const codeceptBin = join(repoRoot, 'bin', 'codecept.js') |
| 12 | + |
| 13 | +const RUNS = Number(process.env.BENCH_RUNS || 3) |
| 14 | +const SITE_URL = process.env.SITE_URL || 'http://127.0.0.1:8000' |
| 15 | + |
| 16 | +const engines = [ |
| 17 | + { name: 'Obscura', config: 'codecept.obscura.bench.js' }, |
| 18 | + { name: 'Playwright', config: 'codecept.playwright.bench.js' }, |
| 19 | +] |
| 20 | + |
| 21 | +function median(nums) { |
| 22 | + const sorted = [...nums].sort((a, b) => a - b) |
| 23 | + const mid = Math.floor(sorted.length / 2) |
| 24 | + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2 |
| 25 | +} |
| 26 | + |
| 27 | +async function siteIsUp() { |
| 28 | + try { |
| 29 | + const res = await fetch(SITE_URL + '/') |
| 30 | + return res.status < 500 |
| 31 | + } catch (e) { |
| 32 | + return false |
| 33 | + } |
| 34 | +} |
| 35 | + |
| 36 | +function runOnce(config) { |
| 37 | + const start = Date.now() |
| 38 | + const result = spawnSync(codeceptBin, ['run', '-c', config], { |
| 39 | + cwd: __dirname, |
| 40 | + env: { ...process.env, SITE_URL }, |
| 41 | + encoding: 'utf8', |
| 42 | + }) |
| 43 | + const elapsed = Date.now() - start |
| 44 | + return { elapsed, exitCode: result.status, stdout: result.stdout, stderr: result.stderr } |
| 45 | +} |
| 46 | + |
| 47 | +async function main() { |
| 48 | + if (!(await siteIsUp())) { |
| 49 | + console.error(`Test app is not reachable at ${SITE_URL}. Start it first:\n php -S 127.0.0.1:8000 -t test/data/app`) |
| 50 | + process.exit(1) |
| 51 | + } |
| 52 | + |
| 53 | + const results = {} |
| 54 | + for (const engine of engines) { |
| 55 | + console.log(`\n=== ${engine.name} (${RUNS} runs) ===`) |
| 56 | + const times = [] |
| 57 | + for (let i = 1; i <= RUNS; i++) { |
| 58 | + const { elapsed, exitCode, stdout, stderr } = runOnce(engine.config) |
| 59 | + if (exitCode !== 0) { |
| 60 | + console.error(` run ${i}: FAILED (exit ${exitCode}) in ${elapsed}ms — excluded from median`) |
| 61 | + console.error(stdout?.slice(-2000)) |
| 62 | + console.error(stderr?.slice(-2000)) |
| 63 | + continue |
| 64 | + } |
| 65 | + console.log(` run ${i}: ${elapsed}ms`) |
| 66 | + times.push(elapsed) |
| 67 | + } |
| 68 | + if (!times.length) { |
| 69 | + console.error(` no successful runs for ${engine.name}`) |
| 70 | + process.exit(1) |
| 71 | + } |
| 72 | + results[engine.name] = { times, median: median(times) } |
| 73 | + } |
| 74 | + |
| 75 | + console.log('\n=== Summary ===') |
| 76 | + for (const engine of engines) { |
| 77 | + const r = results[engine.name] |
| 78 | + console.log(`${engine.name}: median ${r.median}ms (runs: ${r.times.join(', ')}ms)`) |
| 79 | + } |
| 80 | + |
| 81 | + const playwrightMedian = results.Playwright.median |
| 82 | + const obscuraMedian = results.Obscura.median |
| 83 | + const ratio = playwrightMedian / obscuraMedian |
| 84 | + console.log(`\nRatio (Playwright median / Obscura median): ${ratio.toFixed(2)}x`) |
| 85 | + console.log(ratio >= 2 ? 'GATE MET: Obscura is at least 2x faster than Playwright.' : 'GATE NOT MET: Obscura is less than 2x faster than Playwright.') |
| 86 | +} |
| 87 | + |
| 88 | +main() |
0 commit comments