diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 95979ac8d..733e21a32 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,6 +113,51 @@ jobs: done if [ "$FAILED" = "1" ]; then exit 1; fi + visual-tests: + name: Visual regression + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.58.2-noble + options: --ipc=host + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install unzip + run: apt-get update && apt-get install -y unzip + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build design system + run: bun run design-system:build + + - name: Run visual tests + working-directory: ./app + run: npx playwright test --grep @visual + env: + HOME: /root + + - name: Upload HTML report + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: app/playwright-report/ + retention-days: 14 + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: failure() + with: + name: visual-test-results + path: app/test-results/ + retention-days: 7 + build: name: Build runs-on: ubuntu-latest diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index d786e0d03..551f687fc 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -104,6 +104,51 @@ jobs: done if [ "$FAILED" = "1" ]; then exit 1; fi + visual-tests: + name: Visual regression + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.58.2-noble + options: --ipc=host + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install unzip + run: apt-get update && apt-get install -y unzip + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build design system + run: bun run design-system:build + + - name: Run visual tests + working-directory: ./app + run: npx playwright test --grep @visual + env: + HOME: /root + + - name: Upload HTML report + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: app/playwright-report/ + retention-days: 14 + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: failure() + with: + name: visual-test-results + path: app/test-results/ + retention-days: 7 + build: name: Build runs-on: ubuntu-latest diff --git a/.github/workflows/update-visual-baselines.yml b/.github/workflows/update-visual-baselines.yml new file mode 100644 index 000000000..9e1650716 --- /dev/null +++ b/.github/workflows/update-visual-baselines.yml @@ -0,0 +1,84 @@ +name: Update visual baselines + +on: + # Pattern 1: Comment "/update-snapshots" on a PR + issue_comment: + types: [created] + # Pattern 2: Manual trigger from Actions tab (pick a branch) + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-baselines: + name: Update visual baselines + # For issue_comment: only run on PRs when comment is "/update-snapshots" + # For workflow_dispatch: always run + if: > + github.event_name == 'workflow_dispatch' || + ( + github.event.issue.pull_request && + github.event.comment.body == '/update-snapshots' + ) + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.58.2-noble + options: --ipc=host + + steps: + # For issue_comment, we need to resolve the PR branch name + - name: Get PR branch + if: github.event_name == 'issue_comment' + id: pr + run: | + apt-get update && apt-get install -y jq curl + PR_DATA=$(curl -s \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}") + echo "branch=$(echo "$PR_DATA" | jq -r '.head.ref')" >> "$GITHUB_OUTPUT" + + - name: Check out repository + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.branch || github.ref }} + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install unzip + run: | + if ! command -v unzip &> /dev/null; then + apt-get update && apt-get install -y unzip + fi + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build design system + run: bun run design-system:build + + - name: Update visual baselines + working-directory: ./app + run: npx playwright test --grep @visual --update-snapshots --reporter=list + env: + HOME: /root + + - name: Commit updated baselines + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "chore: update visual baselines" + file_pattern: "app/e2e/__screenshots__/**/*.png" + + # Add a rocket reaction so the commenter knows it worked + - name: React to comment + if: github.event_name == 'issue_comment' + run: | + curl -s -X POST \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" \ + -d '{"content": "rocket"}' diff --git a/.gitignore b/.gitignore index 0938796c4..bd64c908b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ coverage/ *.lcov .nyc_output +# Playwright +app/test-results/ +app/playwright-report/ +app/blob-report/ + # Production dist/ dist-*/ diff --git a/Makefile b/Makefile index e40b0203f..a551ba986 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev build test lint format clean deploy +.PHONY: help install dev build test lint format clean deploy pw-test pw-update pw-report help: @echo "Available commands:" @@ -11,6 +11,9 @@ help: @echo " make format - Format code" @echo " make clean - Clean build artifacts" @echo " make deploy - Build and deploy to GitHub Pages" + @echo " make pw-test - Run Playwright visual tests" + @echo " make pw-update - Update visual test baselines" + @echo " make pw-report - Open Playwright HTML report" install: bun install @@ -41,3 +44,12 @@ clean: deploy: build @echo "Build complete. GitHub Actions will handle deployment" + +pw-test: + cd app && npx playwright test --grep @visual + +pw-update: + cd app && npx playwright test --grep @visual --update-snapshots + +pw-report: + cd app && bunx playwright show-report diff --git a/app/e2e/__screenshots__/visual/homepage.visual.spec.ts/homepage-chromium.png b/app/e2e/__screenshots__/visual/homepage.visual.spec.ts/homepage-chromium.png new file mode 100644 index 000000000..7ceb5edcc Binary files /dev/null and b/app/e2e/__screenshots__/visual/homepage.visual.spec.ts/homepage-chromium.png differ diff --git a/app/e2e/visual/homepage.visual.spec.ts b/app/e2e/visual/homepage.visual.spec.ts new file mode 100644 index 000000000..bcaba9c9a --- /dev/null +++ b/app/e2e/visual/homepage.visual.spec.ts @@ -0,0 +1,11 @@ +import { expect, test } from '@playwright/test'; + +test.describe('Homepage', () => { + test('renders correctly @visual', async ({ page }) => { + await page.goto('/'); + await page.waitForLoadState('networkidle'); + await expect(page).toHaveScreenshot('homepage.png', { + fullPage: true, + }); + }); +}); diff --git a/app/e2e/visual/screenshot.css b/app/e2e/visual/screenshot.css new file mode 100644 index 000000000..8beeb8a73 --- /dev/null +++ b/app/e2e/visual/screenshot.css @@ -0,0 +1,11 @@ +/* Applied during Playwright visual screenshots to eliminate flaky dynamic content. */ + +*, +*::before, +*::after { + animation-duration: 0s !important; + animation-delay: 0s !important; + transition-duration: 0s !important; + transition-delay: 0s !important; + scroll-behavior: auto !important; +} diff --git a/app/package.json b/app/package.json index c5859c8a7..a1d27eedc 100644 --- a/app/package.json +++ b/app/package.json @@ -71,6 +71,7 @@ "devDependencies": { "@eslint/js": "^9.29.0", "@ianvs/prettier-plugin-sort-imports": "^4.4.2", + "@playwright/test": "^1.58.2", "@storybook/addon-essentials": "^8.6.14", "@storybook/react": "^8.6.12", "@storybook/react-vite": "^8.6.12", diff --git a/app/playwright.config.ts b/app/playwright.config.ts new file mode 100644 index 000000000..69cef5533 --- /dev/null +++ b/app/playwright.config.ts @@ -0,0 +1,69 @@ +import { defineConfig, devices } from '@playwright/test'; + +const isCI = !!process.env.CI; +const PW_VERSION = '1.58.2'; +const PW_IMAGE = `mcr.microsoft.com/playwright:v${PW_VERSION}-noble`; +const PW_SERVER_PORT = 3200; + +export default defineConfig({ + testDir: './e2e', + fullyParallel: true, + forbidOnly: isCI, + retries: isCI ? 2 : 0, + workers: isCI ? 1 : undefined, + reporter: isCI ? [['html', { open: 'never' }], ['github']] : [['html', { open: 'on-failure' }]], + + snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}-{projectName}{ext}', + + expect: { + toHaveScreenshot: { + maxDiffPixelRatio: 0.005, + threshold: 0.2, + animations: 'disabled', + caret: 'hide', + scale: 'css', + stylePath: './e2e/visual/screenshot.css', + }, + }, + + use: { + baseURL: isCI ? 'http://localhost:3000' : 'http://host.docker.internal:3000', + trace: 'on-first-retry', + timezoneId: 'America/New_York', + ...(!isCI && { + connectOptions: { + wsEndpoint: `ws://127.0.0.1:${PW_SERVER_PORT}/`, + }, + }), + }, + + webServer: [ + { + command: 'VITE_APP_MODE=website npx vite', + url: 'http://localhost:3000', + reuseExistingServer: !isCI, + timeout: 120_000, + }, + ...(!isCI + ? [ + { + command: `docker run --rm --init -p ${PW_SERVER_PORT}:${PW_SERVER_PORT} ${PW_IMAGE} npx playwright run-server --port ${PW_SERVER_PORT} --host 0.0.0.0`, + url: `http://localhost:${PW_SERVER_PORT}`, + timeout: 120_000, + reuseExistingServer: !isCI, + gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 10_000 }, + }, + ] + : []), + ], + + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + viewport: { width: 1280, height: 720 }, + }, + }, + ], +}); diff --git a/app/vite.config.mjs b/app/vite.config.mjs index 643f6f995..f9c4b069c 100644 --- a/app/vite.config.mjs +++ b/app/vite.config.mjs @@ -123,5 +123,6 @@ export default defineConfig({ globals: true, environment: 'jsdom', setupFiles: './vitest.setup.mjs', + exclude: ['e2e/**', 'node_modules/**'], }, }); diff --git a/bun.lock b/bun.lock index 3c033b2df..716889da4 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "policyengine-monorepo", @@ -57,6 +58,7 @@ "devDependencies": { "@eslint/js": "^9.29.0", "@ianvs/prettier-plugin-sort-imports": "^4.4.2", + "@playwright/test": "^1.58.2", "@storybook/addon-essentials": "^8.6.14", "@storybook/react": "^8.6.12", "@storybook/react-vite": "^8.6.12", @@ -125,7 +127,7 @@ "name": "@policyengine/website", "version": "0.1.0", "dependencies": { - "@policyengine/design-system": "workspace:*", + "@policyengine/design-system": "*", "@tabler/icons-react": "^3.31.0", "framer-motion": "^12.38.0", "fuse.js": "^7.1.0", @@ -462,6 +464,8 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@playwright/test": ["@playwright/test@1.58.2", "", { "dependencies": { "playwright": "1.58.2" }, "bin": { "playwright": "cli.js" } }, "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA=="], + "@plotly/d3": ["@plotly/d3@3.8.2", "", {}, "sha512-wvsNmh1GYjyJfyEBPKJLTMzgf2c2bEbSIL50lmqVUi+o1NHaLPi1Lb4v7VxXXJn043BhNyrxUrWI85Q+zmjOVA=="], "@plotly/d3-sankey": ["@plotly/d3-sankey@0.7.2", "", { "dependencies": { "d3-array": "1", "d3-collection": "1", "d3-shape": "^1.2.0" } }, "sha512-2jdVos1N3mMp3QW0k2q1ph7Gd6j5PY1YihBrwpkFnKqO+cqtZq3AdEYUeSGXMeLsBDQYiqTVcihYfk8vr5tqhw=="], @@ -2052,7 +2056,11 @@ "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], - "plotly.js": ["plotly.js@3.4.0", "", { "dependencies": { "@plotly/d3": "3.8.2", "@plotly/d3-sankey": "0.7.2", "@plotly/d3-sankey-circular": "0.33.1", "@plotly/mapbox-gl": "1.13.4", "@plotly/regl": "^2.1.2", "@turf/area": "^7.1.0", "@turf/bbox": "^7.1.0", "@turf/centroid": "^7.1.0", "base64-arraybuffer": "^1.0.2", "canvas-fit": "^1.5.0", "color-alpha": "1.0.4", "color-normalize": "1.5.0", "color-parse": "2.0.0", "color-rgba": "3.0.0", "country-regex": "^1.1.0", "d3-force": "^1.2.1", "d3-format": "^1.4.5", "d3-geo": "^1.12.1", "d3-geo-projection": "^2.9.0", "d3-hierarchy": "^1.1.9", "d3-interpolate": "^3.0.1", "d3-time": "^1.1.0", "d3-time-format": "^2.2.3", "fast-isnumeric": "^1.1.4", "gl-mat4": "^1.2.0", "gl-text": "^1.4.0", "has-hover": "^1.0.1", "has-passive-events": "^1.0.0", "is-mobile": "^4.0.0", "maplibre-gl": "^4.7.1", "mouse-change": "^1.4.0", "mouse-event-offset": "^3.0.2", "mouse-wheel": "^1.2.0", "native-promise-only": "^0.8.1", "parse-svg-path": "^0.1.2", "point-in-polygon": "^1.1.0", "polybooljs": "^1.2.2", "probe-image-size": "^7.2.3", "regl-error2d": "^2.0.12", "regl-line2d": "^3.1.3", "regl-scatter2d": "^3.3.1", "regl-splom": "^1.0.14", "strongly-connected-components": "^1.0.1", "superscript-text": "^1.0.0", "svg-path-sdf": "^1.1.3", "tinycolor2": "^1.4.2", "to-px": "1.0.1", "topojson-client": "^3.1.0", "webgl-context": "^2.2.0", "world-calendars": "^1.0.4" } }, "sha512-jdWfHLB8AxlGUmVhqJTGEKdwjCKGn9Yi8yg16067/JyqseuSado7F6IOM1XPZspdZyO/cf8IPuy7ROlVhqZZNw=="], + "playwright": ["playwright@1.58.2", "", { "dependencies": { "playwright-core": "1.58.2" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A=="], + + "playwright-core": ["playwright-core@1.58.2", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg=="], + + "plotly.js": ["plotly.js@3.3.1", "", { "dependencies": { "@plotly/d3": "3.8.2", "@plotly/d3-sankey": "0.7.2", "@plotly/d3-sankey-circular": "0.33.1", "@plotly/mapbox-gl": "1.13.4", "@plotly/regl": "^2.1.2", "@turf/area": "^7.1.0", "@turf/bbox": "^7.1.0", "@turf/centroid": "^7.1.0", "base64-arraybuffer": "^1.0.2", "canvas-fit": "^1.5.0", "color-alpha": "1.0.4", "color-normalize": "1.5.0", "color-parse": "2.0.0", "color-rgba": "3.0.0", "country-regex": "^1.1.0", "d3-force": "^1.2.1", "d3-format": "^1.4.5", "d3-geo": "^1.12.1", "d3-geo-projection": "^2.9.0", "d3-hierarchy": "^1.1.9", "d3-interpolate": "^3.0.1", "d3-time": "^1.1.0", "d3-time-format": "^2.2.3", "fast-isnumeric": "^1.1.4", "gl-mat4": "^1.2.0", "gl-text": "^1.4.0", "has-hover": "^1.0.1", "has-passive-events": "^1.0.0", "is-mobile": "^4.0.0", "maplibre-gl": "^4.7.1", "mouse-change": "^1.4.0", "mouse-event-offset": "^3.0.2", "mouse-wheel": "^1.2.0", "native-promise-only": "^0.8.1", "parse-svg-path": "^0.1.2", "point-in-polygon": "^1.1.0", "polybooljs": "^1.2.2", "probe-image-size": "^7.2.3", "regl-error2d": "^2.0.12", "regl-line2d": "^3.1.3", "regl-scatter2d": "^3.3.1", "regl-splom": "^1.0.14", "strongly-connected-components": "^1.0.1", "superscript-text": "^1.0.0", "svg-path-sdf": "^1.1.3", "tinycolor2": "^1.4.2", "to-px": "1.0.1", "topojson-client": "^3.1.0", "webgl-context": "^2.2.0", "world-calendars": "^1.0.4" } }, "sha512-SrGSZ02HvCWQIYsbQX4sgjgGo7k4T+Oz8a+RQwE5Caz+yu1vputBM1UThmiOPY51B5HzO9ajym8Rl4pmLj+i9Q=="], "point-in-polygon": ["point-in-polygon@1.1.0", "", {}, "sha512-3ojrFwjnnw8Q9242TzgXuTD+eKiutbzyslcq1ydfu82Db2y+Ogbmyrkpv0Hgj31qwT3lbS9+QAAO/pIQM35XRw=="], @@ -2804,14 +2812,14 @@ "node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + "plotly.js/d3-geo": ["d3-geo@1.12.1", "", { "dependencies": { "d3-array": "1" } }, "sha512-XG4d1c/UJSEX9NfU02KwBL6BYPj8YKHxgBEw5om2ZnTRSbIcego6dhHwcxuSR3clxh0EpE38os1DVPOmnYtTPg=="], "policyengine-app-v2/framer-motion": ["framer-motion@12.29.0", "", { "dependencies": { "motion-dom": "^12.29.0", "motion-utils": "^12.27.2", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-1gEFGXHYV2BD42ZPTFmSU9buehppU+bCuOnHU0AD18DKh9j4DuTx47MvqY5ax+NNWRtK32qIcJf1UxKo1WwjWg=="], "policyengine-app-v2/react-markdown": ["react-markdown@9.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-xaijuJB0kzGiUdG7nc2MOMDUDBWPyGAjZtUrow9XxUeua8IqeP+VlIfAZ3bphpcLTnSZXz6z9jcVC/TCwbfgdw=="], - "policyengine-app-v2/tailwindcss": ["tailwindcss@4.2.0", "", {}, "sha512-yYzTZ4++b7fNYxFfpnberEEKu43w44aqDMNM9MHMmcKuCH7lL8jJ4yJ7LGHv7rSwiqM0nkiobF9I6cLlpS2P7Q=="], - "policyengine-app-v2/vite-tsconfig-paths": ["vite-tsconfig-paths@5.1.4", "", { "dependencies": { "debug": "^4.1.1", "globrex": "^0.1.2", "tsconfck": "^3.0.3" }, "peerDependencies": { "vite": "*" }, "optionalPeers": ["vite"] }, "sha512-cYj0LRuLV2c2sMqhqhGpaO3LretdtMn/BVX4cPLanIZuwwrkVl+lK84E/miEXkCHWXuq65rhNN4rXsBcOB3S4w=="], "pretty-format/ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="],