Skip to content
E2E Tests

feat(e2e): #157 basePath-served Playwright project + signup smoke #860

Sign in to view logs

feat(e2e): #157 basePath-served Playwright project + signup smoke

feat(e2e): #157 basePath-served Playwright project + signup smoke #860

Workflow file for this run

.github/workflows/e2e.yml at 22dda48
name: E2E Tests
on:
push:
branches: [main, develop]
# Lever 4: docs / planning / editor / license-only changes can't affect the
# browser build or any E2E flow, so they skip the (~1hr) suite. `schedule`
# and `workflow_dispatch` below ignore path filters → the weekly full run and
# the manual button remain the safety net. NOT ignored on purpose: `.github/**`
# (this workflow lives here — a workflow edit must still be E2E-tested) and
# `supabase/functions/**` (an E2E flow could indirectly hit a deployed function).
paths-ignore:
- '**/*.md'
- 'docs/**'
- '.claude/**'
- '.specify/**'
- 'features/**'
- 'LICENSE'
- '.gitignore'
- '.vscode/**'
pull_request:
branches: [main]
paths-ignore:
- '**/*.md'
- 'docs/**'
- '.claude/**'
- '.specify/**'
- 'features/**'
- 'LICENSE'
- '.gitignore'
- '.vscode/**'
schedule:
- cron: '0 6 * * 1' # Monday 6 AM UTC — full 3-browser run
workflow_dispatch:
concurrency:
# Repo-wide mutex: every E2E run shares one Supabase project, and concurrent
# runs race each other's beforeAll cleanupOldMessages hooks (run 25769955636
# on 2026-05-13 was cancelled at 60 min after PR #88's run, started 3 min
# later, wiped its data). Serialize across all refs; do not cancel a running
# job to make room for a queued one.
group: e2e-supabase-${{ github.repository }}
cancel-in-progress: false
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ vars.NEXT_PUBLIC_SUPABASE_URL }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ vars.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
jobs:
# ============================================
# STEP 1: Build once, cache for all shards
# ============================================
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build application
run: pnpm build
env:
# The E2E suite serves this build from the ROOT (`serve out -l 3000`).
# DISABLE_BASE_PATH forces a basePath-less build so /_next/*.js assets
# resolve at /. If the build baked in the GitHub Pages basePath
# (/ScriptHammer), every /ScriptHammer/_next/*.js would 404 under
# serve-at-root, React would never hydrate, and the whole suite fails.
# A step-level `GITHUB_ACTIONS: false` is NOT reliable here — the
# runner re-injects GITHUB_ACTIONS=true into the child processes
# `pnpm build` spawns (scripts/detect-project.js / next.config execSync).
DISABLE_BASE_PATH: 'true'
NEXT_PUBLIC_EMAILJS_PUBLIC_KEY: ${{ secrets.EMAILJS_PUBLIC_KEY }}
NEXT_PUBLIC_EMAILJS_SERVICE_ID: ${{ vars.NEXT_PUBLIC_EMAILJS_SERVICE_ID }}
NEXT_PUBLIC_EMAILJS_TEMPLATE_ID: ${{ vars.NEXT_PUBLIC_EMAILJS_TEMPLATE_ID }}
# Payment publishable keys must be baked into the static build so
# featureFlags.stripeEnabled / paypalEnabled are true and the
# PaymentButton / provider tabs render. Commit 77e409d added a
# "graceful missing-payment-config" branch that hides these controls
# when env vars are empty, which broke all payment E2E tests. Real
# keys aren't required: tests that redirect to Stripe/PayPal are
# test.skip'd; the rest only assert UI render + consent flow.
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: pk_test_e2e_dummy_not_a_real_key
NEXT_PUBLIC_PAYPAL_CLIENT_ID: e2e_dummy_paypal_client_id
- name: Upload build artifacts
uses: actions/upload-artifact@v6
with:
name: build-output
path: out/
retention-days: 1
# ============================================
# STEP 2: Smoke tests (~2 min) - FAST FEEDBACK
# ============================================
smoke:
name: Smoke Tests
runs-on: ubuntu-latest
needs: build
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright (chromium only)
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps chromium && break
echo "Retry $i: playwright install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
- name: Run smoke tests (sign-up only - auth tests run after auth-setup)
run: |
pnpm exec playwright test \
tests/e2e/auth/sign-up.spec.ts \
--project=signup \
--no-deps \
--reporter=list \
--timeout=30000
env:
CI: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
- name: Smoke test summary
if: always()
run: echo "## Smoke Tests Complete" >> "$GITHUB_STEP_SUMMARY"
# ============================================
# STEP 2b: basePath project-site smoke (#157)
# A GitHub Pages project-site fork needs a /RepoName basePath; the class of
# bug where auth redirects / hand-built URLs escape that prefix is invisible
# to the root-anchored suite. This job builds a DEDICATED basePath-prefixed
# export and serves it under /ScriptHammer, then runs the `basepath` project
# (tests/e2e/basepath/**) against it. Self-contained: builds its own artifact
# (the shared build-output is root-path), chromium-only, one fast spec.
# ============================================
basepath:
name: basePath Project-Site Smoke
runs-on: ubuntu-latest
needs: build
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright (chromium only)
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps chromium && break
echo "Retry $i: playwright install failed, retrying in 10s..."
sleep 10
done
- name: Build basePath-prefixed export
run: bash scripts/serve-basepath.sh build
env:
BASE_PATH: /ScriptHammer
# The prefixed build must NOT be zeroed by the CNAME/DISABLE_BASE_PATH
# path — NEXT_PUBLIC_BASE_PATH (set by the script) wins in next.config.
NEXT_PUBLIC_EMAILJS_PUBLIC_KEY: ${{ secrets.EMAILJS_PUBLIC_KEY }}
NEXT_PUBLIC_EMAILJS_SERVICE_ID: ${{ vars.NEXT_PUBLIC_EMAILJS_SERVICE_ID }}
NEXT_PUBLIC_EMAILJS_TEMPLATE_ID: ${{ vars.NEXT_PUBLIC_EMAILJS_TEMPLATE_ID }}
NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: pk_test_e2e_dummy_not_a_real_key
NEXT_PUBLIC_PAYPAL_CLIENT_ID: e2e_dummy_paypal_client_id
- name: Start server (symlink under /ScriptHammer)
run: |
set -euo pipefail
BASE_PATH=/ScriptHammer bash scripts/serve-basepath.sh serve 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
sleep 2
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
cat serve.log || true
exit 1
fi
if ! npx wait-on http://localhost:3000/ScriptHammer/sign-up/ --timeout 60000 --verbose; then
echo "::error::wait-on timed out — prefixed serve never bound within 60s"
cat serve.log || true
ss -tlnp 2>/dev/null || true
exit 1
fi
echo "serve is responding on http://localhost:3000/ScriptHammer/"
- name: Run basePath project
run: |
pnpm exec playwright test \
--project=basepath \
--no-deps \
--reporter=list \
--timeout=30000
env:
CI: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000/ScriptHammer
# The smoke spec intercepts the signup POST (no real user/email), but
# the shared globalSetup hard-requires these to validate prerequisites
# before ANY project runs — so supply the same set the smoke job uses.
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
- name: basePath summary
if: always()
run: echo "## basePath Project-Site Smoke Complete" >> "$GITHUB_STEP_SUMMARY"
# ============================================
# STEP 3: Rate-limiting tests (ordered, clean IP)
# ============================================
rate-limiting:
name: Rate-Limiting Tests
runs-on: ubuntu-latest
needs: smoke
timeout-minutes: 10
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright (chromium only)
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps chromium && break
echo "Retry $i: playwright install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
- name: Run rate-limiting tests (ordered)
run: pnpm test:e2e --project=rate-limiting --project=brute-force --project=signup --reporter=list --trace=on-first-retry
env:
CI: true
PLAYWRIGHT_BROWSER: chromium
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
# ============================================
# STEP 4: Auth Setup (runs ONCE, shares state)
# ============================================
auth-setup:
name: Auth Setup
runs-on: ubuntu-latest
needs: smoke
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright (chromium only for auth)
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps chromium && break
echo "Retry $i: playwright install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
- name: Run auth setup
run: pnpm exec playwright test --project=setup --reporter=list --timeout=180000
env:
CI: true
AUTH_SETUP_JOB: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
- name: Upload auth state
uses: actions/upload-artifact@v6
with:
name: auth-state
# Include both the primary (USER_A / PRIMARY) and User B (TERTIARY)
# pre-authenticated fixtures. Multi-user messaging and security
# specs load storage-state-auth-b.json into a second browser
# context instead of calling performSignIn live, which cumulatively
# exceeded Supabase GoTrue's 5-attempt brute-force lockout.
path: |
tests/e2e/fixtures/storage-state-auth.json
tests/e2e/fixtures/storage-state-auth-b.json
retention-days: 1
# ============================================
# STEP 5: Full E2E tests (sharded, parallel)
# ============================================
e2e:
name: E2E (${{ matrix.project }} ${{ matrix.shard }})
runs-on: ubuntu-latest
needs: [smoke, rate-limiting, auth-setup]
timeout-minutes: 60
strategy:
fail-fast: false
# Lever 1: the 3 browser jobs now run CONCURRENTLY (the firefox/webkit
# `needs:` chain was dropped — #116 per-test fixture isolation removed the
# cross-test data races the serialization protected against). max-parallel
# is 2 (not 3) so peak concurrent Supabase load is 2 shards × 3 browsers = 6,
# half the historically-dangerous 9 that overwhelmed the free tier
# ("Sign-in failed: Request rate limit reached" on webkit). Validated via
# workflow_dispatch before merge; if rate limits recur, restore one `needs:`
# (re-serialize webkit) rather than raising max-parallel.
max-parallel: 2
matrix:
include:
# ===== CHROMIUM (7 shards) =====
# msg runs as a single shard, not 2, because two concurrent msg
# shards hit the same shared user-pair + conversation and race
# each other's beforeAll `messages.delete()`. Serializing the
# messaging tests within ONE shard per browser eliminates the
# cross-shard data race we spent 9 rounds patching piecemeal.
- { project: chromium-msg, browser: chromium, shard: 1/1 }
# #116 Phase 2 — per-test fixture-isolation specs run as their OWN
# parallel slice at workers>1 (each test owns its data; no shared-user
# race). This is the CI validation gate: prove the cloud free-tier
# sustains parallel signups before converting the rest. workers:2 is
# conservative — raise only after this proves green & flake-free.
- {
project: chromium-msg-iso,
browser: chromium,
shard: 1/1,
workers: 2,
}
- { project: chromium-gen, browser: chromium, shard: 1/6 }
- { project: chromium-gen, browser: chromium, shard: 2/6 }
- { project: chromium-gen, browser: chromium, shard: 3/6 }
- { project: chromium-gen, browser: chromium, shard: 4/6 }
- { project: chromium-gen, browser: chromium, shard: 5/6 }
- { project: chromium-gen, browser: chromium, shard: 6/6 }
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright browsers
env:
BROWSER: ${{ matrix.browser }}
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps "$BROWSER" && break
echo "Retry $i: $BROWSER install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Download auth state
uses: actions/download-artifact@v8
with:
name: auth-state
path: tests/e2e/fixtures/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
env:
CI: true
- name: Prime Supabase connection pool
env:
SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }}
SUPABASE_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
run: |
# Wake Supabase free tier and warm the connection pool.
# Without this, the first query from each shard takes 10-30s.
curl -sf "${SUPABASE_URL}/rest/v1/user_profiles?select=id&limit=1" \
-H "apikey: ${SUPABASE_KEY}" \
-H "Authorization: Bearer ${SUPABASE_KEY}" \
> /dev/null 2>&1 || true
echo "✓ Supabase connection pool primed"
- name: Stagger shards within batch to reduce Supabase contention
env:
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
run: |
SHARD_NUM=$(echo "$SHARD" | cut -d/ -f1)
# max-parallel: 8 already serializes browsers. Within each batch
# of 8, stagger by 5-10 seconds to spread the initial connection
# burst on Supabase.
if echo "$PROJECT" | grep -q "msg"; then
DELAY=$(((SHARD_NUM - 1) * 10))
else
DELAY=$(((SHARD_NUM - 1) * 5))
fi
echo "$PROJECT shard $SHARD_NUM: waiting ${DELAY}s"
sleep $DELAY
- name: Run E2E tests (${{ matrix.project }} ${{ matrix.shard }})
env:
CI: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
DEBUG: pw:api
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
# Per-job worker override (only set for the *-msg-iso slice). Empty
# for every other job → Playwright uses the config default (workers:1
# on CI). Routed via env per workflow-injection guidance.
WORKERS: ${{ matrix.workers }}
run: |
# --no-deps: skip the 'setup' project dependency. The auth-setup
# job already ran setup once (with chromium) and uploaded the
# auth-state artifact, which we downloaded above. Without --no-deps,
# Playwright tries to re-run setup with the matrix browser, which
# fails because firefox/webkit shards don't install chromium.
WORKERS_FLAG=""
if [ -n "$WORKERS" ]; then
WORKERS_FLAG="--workers=$WORKERS"
fi
pnpm exec playwright test \
--project="$PROJECT" \
--shard="$SHARD" \
--reporter=blob \
--trace=on-first-retry \
--no-deps \
$WORKERS_FLAG
- name: Upload blob report
if: always()
uses: actions/upload-artifact@v6
with:
name: blob-report-${{ matrix.project }}-${{ strategy.job-index }}
path: blob-report/
retention-days: 1
# ============================================
# STEP 5b: Firefox E2E (runs concurrently with chromium + webkit — Lever 1)
# ============================================
e2e-firefox:
name: E2E (${{ matrix.project }} ${{ matrix.shard }})
runs-on: ubuntu-latest
# Lever 1: depends only on the build/auth artifacts (same as e2e/chromium),
# NOT on the chromium job — so all 3 browsers run in parallel.
needs: [smoke, rate-limiting, auth-setup]
# Lever 2: chromium gates every PR; firefox + webkit run on push-to-main,
# the weekly cron, and workflow_dispatch — keeping PR feedback fast. A PR
# touching WebKit/Firefox-sensitive code can opt back in with the `full-e2e`
# label (e.g. messaging error-string handling — see the real Safari "Load
# failed" retry bug fixed in commit 7fb7563).
if: >-
always() && !cancelled() &&
(github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, 'full-e2e'))
timeout-minutes: 60
strategy:
fail-fast: false
# Lever 1: max-parallel 2 (was 3) — browsers now run concurrently, so the
# per-browser cap is lower to keep peak Supabase load at 6 shards, not 9.
max-parallel: 2
matrix:
include:
# msg runs as a single shard (see chromium matrix note above).
- { project: firefox-msg, browser: firefox, shard: 1/1 }
# #116 Phase 2 — per-test fixture-isolation slice at workers>1.
- {
project: firefox-msg-iso,
browser: firefox,
shard: 1/1,
workers: 2,
}
- { project: firefox-gen, browser: firefox, shard: 1/6 }
- { project: firefox-gen, browser: firefox, shard: 2/6 }
- { project: firefox-gen, browser: firefox, shard: 3/6 }
- { project: firefox-gen, browser: firefox, shard: 4/6 }
- { project: firefox-gen, browser: firefox, shard: 5/6 }
- { project: firefox-gen, browser: firefox, shard: 6/6 }
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright browsers
env:
BROWSER: ${{ matrix.browser }}
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps "$BROWSER" && break
echo "Retry $i: $BROWSER install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Download auth state
uses: actions/download-artifact@v8
with:
name: auth-state
path: tests/e2e/fixtures/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
env:
CI: true
- name: Prime Supabase connection pool
env:
SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }}
SUPABASE_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
run: |
curl -sf "${SUPABASE_URL}/rest/v1/user_profiles?select=id&limit=1" \
-H "apikey: ${SUPABASE_KEY}" \
-H "Authorization: Bearer ${SUPABASE_KEY}" \
> /dev/null 2>&1 || true
- name: Stagger shards within batch
env:
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
run: |
SHARD_NUM=$(echo "$SHARD" | cut -d/ -f1)
if echo "$PROJECT" | grep -q "msg"; then
DELAY=$(((SHARD_NUM - 1) * 10))
else
DELAY=$(((SHARD_NUM - 1) * 5))
fi
sleep $DELAY
- name: Run E2E tests
env:
CI: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
DEBUG: pw:api
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
# Per-job worker override (set only for the *-msg-iso slice). Empty
# elsewhere → config default (workers:1 on CI). Routed via env.
WORKERS: ${{ matrix.workers }}
run: |
WORKERS_FLAG=""
if [ -n "$WORKERS" ]; then
WORKERS_FLAG="--workers=$WORKERS"
fi
pnpm exec playwright test \
--project="$PROJECT" \
--shard="$SHARD" \
--reporter=blob \
--trace=on-first-retry \
--no-deps \
$WORKERS_FLAG
- name: Upload blob report
if: always()
uses: actions/upload-artifact@v6
with:
name: blob-report-${{ matrix.project }}-${{ strategy.job-index }}
path: blob-report/
retention-days: 1
# ============================================
# STEP 5c: WebKit E2E (runs concurrently with chromium + firefox — Lever 1)
# ============================================
e2e-webkit:
name: E2E (${{ matrix.project }} ${{ matrix.shard }})
runs-on: ubuntu-latest
# Lever 1: depends only on build/auth artifacts, NOT firefox — all 3 parallel.
needs: [smoke, rate-limiting, auth-setup]
# Lever 2: see e2e-firefox above — skipped on PRs (chromium gates those)
# unless the PR carries the `full-e2e` label; runs on push-to-main + cron.
if: >-
always() && !cancelled() &&
(github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, 'full-e2e'))
timeout-minutes: 60
strategy:
fail-fast: false
# Lever 1: max-parallel 2 (was 3) — browsers now run concurrently, so the
# per-browser cap is lower to keep peak Supabase load at 6 shards, not 9.
max-parallel: 2
matrix:
include:
# msg runs as a single shard (see chromium matrix note above).
- { project: webkit-msg, browser: webkit, shard: 1/1 }
# #116 Phase 2 — per-test fixture-isolation slice at workers>1.
- {
project: webkit-msg-iso,
browser: webkit,
shard: 1/1,
workers: 2,
}
- { project: webkit-gen, browser: webkit, shard: 1/6 }
- { project: webkit-gen, browser: webkit, shard: 2/6 }
- { project: webkit-gen, browser: webkit, shard: 3/6 }
- { project: webkit-gen, browser: webkit, shard: 4/6 }
- { project: webkit-gen, browser: webkit, shard: 5/6 }
- { project: webkit-gen, browser: webkit, shard: 6/6 }
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install Playwright browsers
env:
BROWSER: ${{ matrix.browser }}
run: |
for i in 1 2 3; do
pnpm exec playwright install --with-deps "$BROWSER" && break
echo "Retry $i: $BROWSER install failed, retrying in 10s..."
sleep 10
done
- name: Download build
uses: actions/download-artifact@v8
with:
name: build-output
path: out/
- name: Download auth state
uses: actions/download-artifact@v8
with:
name: auth-state
path: tests/e2e/fixtures/
- name: Start server
run: |
set -euo pipefail
npx serve out -l 3000 > serve.log 2>&1 &
SERVE_PID=$!
echo "serve started, PID=$SERVE_PID"
# Brief grace period for the child to spawn; wait-on does the real polling.
sleep 2
# Verify serve didn't already exit (port in use, missing out/, etc.).
if ! kill -0 "$SERVE_PID" 2>/dev/null; then
echo "::error::serve exited immediately after launch"
echo "--- serve.log ---"
cat serve.log || true
exit 1
fi
# Explicit exit-code propagation. Don't rely on -e interacting cleanly
# with backgrounded jobs.
if ! npx wait-on http://localhost:3000 --timeout 60000 --verbose; then
echo "::error::wait-on timed out — serve never bound to port 3000 within 60s"
echo "--- serve.log ---"
cat serve.log || true
echo "--- listening sockets ---"
ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true
echo "--- serve process state ---"
ps -fp "$SERVE_PID" 2>/dev/null || echo "serve process $SERVE_PID is gone"
echo "--- direct curl attempt ---"
curl -v --max-time 5 http://localhost:3000/ 2>&1 || true
exit 1
fi
echo "serve is responding on http://localhost:3000"
env:
CI: true
- name: Prime Supabase connection pool
env:
SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }}
SUPABASE_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
run: |
curl -sf "${SUPABASE_URL}/rest/v1/user_profiles?select=id&limit=1" \
-H "apikey: ${SUPABASE_KEY}" \
-H "Authorization: Bearer ${SUPABASE_KEY}" \
> /dev/null 2>&1 || true
- name: Stagger shards within batch
env:
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
run: |
SHARD_NUM=$(echo "$SHARD" | cut -d/ -f1)
if echo "$PROJECT" | grep -q "msg"; then
DELAY=$(((SHARD_NUM - 1) * 10))
else
DELAY=$(((SHARD_NUM - 1) * 5))
fi
sleep $DELAY
- name: Run E2E tests
env:
CI: true
SKIP_WEBSERVER: true
BASE_URL: http://localhost:3000
DEBUG: pw:api
SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
TEST_USER_PRIMARY_EMAIL: ${{ vars.TEST_USER_PRIMARY_EMAIL }}
TEST_USER_PRIMARY_PASSWORD: ${{ secrets.TEST_USER_PRIMARY_PASSWORD }}
TEST_USER_SECONDARY_EMAIL: ${{ vars.TEST_USER_SECONDARY_EMAIL }}
TEST_USER_SECONDARY_PASSWORD: ${{ secrets.TEST_USER_SECONDARY_PASSWORD }}
TEST_USER_TERTIARY_EMAIL: ${{ vars.TEST_USER_TERTIARY_EMAIL }}
TEST_USER_TERTIARY_PASSWORD: ${{ secrets.TEST_USER_TERTIARY_PASSWORD }}
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
# Per-job worker override (set only for the *-msg-iso slice). Empty
# elsewhere → config default (workers:1 on CI). Routed via env.
WORKERS: ${{ matrix.workers }}
run: |
WORKERS_FLAG=""
if [ -n "$WORKERS" ]; then
WORKERS_FLAG="--workers=$WORKERS"
fi
pnpm exec playwright test \
--project="$PROJECT" \
--shard="$SHARD" \
--reporter=blob \
--trace=on-first-retry \
--no-deps \
$WORKERS_FLAG
- name: Upload blob report
if: always()
uses: actions/upload-artifact@v6
with:
name: blob-report-${{ matrix.project }}-${{ strategy.job-index }}
path: blob-report/
retention-days: 1
# ============================================
# STEP 6: Merge reports and generate summary
# ============================================
report:
name: Test Report
runs-on: ubuntu-latest
needs: [e2e, e2e-firefox, e2e-webkit]
if: always()
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Install pnpm
uses: pnpm/action-setup@v6
with:
version: 10.16.1
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version: 20.x
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Download blob reports
uses: actions/download-artifact@v8
with:
pattern: blob-report-*
path: all-blob-reports
merge-multiple: true
- name: Merge reports
run: pnpm exec playwright merge-reports --reporter=html,github ./all-blob-reports
- name: Upload HTML report
uses: actions/upload-artifact@v6
with:
name: playwright-report
path: playwright-report/
retention-days: 7
- name: Generate summary
if: always()
env:
EVENT_NAME: ${{ github.event_name }}
HAS_FULL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
run: |
echo "## E2E Test Results" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
if [ "$EVENT_NAME" = "pull_request" ] && [ "$HAS_FULL_LABEL" != "true" ]; then
echo "**PR Mode**: Chromium only (8 jobs). Add the \`full-e2e\` label to also run Firefox + WebKit." >> "$GITHUB_STEP_SUMMARY"
else
echo "**Full Mode**: Chromium + Firefox + WebKit (24 jobs total)" >> "$GITHUB_STEP_SUMMARY"
fi
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "Download the **playwright-report** artifact for detailed results." >> "$GITHUB_STEP_SUMMARY"