Skip to content

Commit 5f3ea63

Browse files
kantordkantord
authored
test: restore playwright tests (#1340)
* test: restore playwright tests * build: use latest playwright * fixes * simplify e2e runner scripts * do not show confirmation in end to end tests * address review stuff * . * . * make test more stable * fix failing tests * add timeout constants * some cleanup and documentation
1 parent 047ca4e commit 5f3ea63

File tree

10 files changed

+139
-111
lines changed

10 files changed

+139
-111
lines changed

‎.github/workflows/_e2e.yml‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,18 +44,15 @@ jobs:
4444
sudo systemctl start docker
4545
sudo systemctl status docker
4646
47-
- name: Pre-build app
48-
run: |
49-
pnpm tsc -b --clean
50-
pnpm electron-forge package
47+
- name: Package app
48+
run: pnpm package
5149

5250
- name: Install ffmpeg for video recording
53-
run: |
54-
npx playwright install ffmpeg
51+
run: npx playwright install ffmpeg
5552

5653
- name: Start & unlock keyring
5754
run: |
58-
export $(dbus-launch)
55+
export $(dbus-launch)
5956
echo "ci-passphrase" | \
6057
gnome-keyring-daemon --unlock \
6158
--components=secrets,ssh,pkcs11 &
@@ -69,8 +66,7 @@ jobs:
6966
- name: Run tests
7067
env:
7168
ELECTRON_DISABLE_SANDBOX: 'true'
72-
run: |
73-
xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" pnpm run playwright test
69+
run: xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" pnpm e2e:prebuilt
7470

7571
- uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
7672
if: ${{ always() }}

‎.github/workflows/on-main.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ jobs:
1919
uses: ./.github/workflows/_unit-tests.yml
2020
secrets: inherit
2121

22-
# The e2e tests are currently flaky, so we're disabling them for now, needs to be investigated
23-
# e2e-tests:
24-
# name: End-to-end tests
25-
# uses: ./.github/workflows/_e2e.yml
26-
# secrets: inherit
22+
e2e-tests:
23+
name: End-to-end tests
24+
uses: ./.github/workflows/_e2e.yml
25+
secrets: inherit

‎.github/workflows/on-pr.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ jobs:
2222
uses: ./.github/workflows/_unit-tests.yml
2323
secrets: inherit
2424

25-
# The e2e tests are currently flaky, so we're disabling them for now, needs to be investigated
26-
# e2e-tests:
27-
# name: End-to-end tests
28-
# uses: ./.github/workflows/_e2e.yml
29-
# secrets: inherit
25+
e2e-tests:
26+
name: End-to-end tests
27+
uses: ./.github/workflows/_e2e.yml
28+
secrets: inherit

‎docs/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,15 @@ Here are the most common scripts you will use during development:
5252
- `pnpm run test:coverage`: Runs tests with coverage.
5353
- `pnpm run thv`: Run the same `thv` binary that the dev server uses
5454

55+
### End-to-end tests
56+
57+
E2E tests use Playwright to test the packaged Electron application.
58+
59+
- `pnpm run e2e`: Packages the app and runs e2e tests. Use this for a full test
60+
run from scratch.
61+
- `pnpm run e2e:prebuilt`: Runs e2e tests against an already packaged app in
62+
`out/`. Use this when iterating on tests without rebuilding.
63+
5564
### Building and packaging
5665

5766
- `pnpm run package`: Packages the application for the current platform.

‎e2e-tests/fixtures/electron.ts‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import path from 'path'
2+
import { execSync } from 'child_process'
13
import {
24
test as base,
35
_electron as electron,
@@ -10,30 +12,102 @@ type ElectronFixtures = {
1012
window: Page
1113
}
1214

15+
const TEST_GROUP_NAME = 'playwright-automated-test-fixture'
16+
17+
/** Default timeout for most operations (10 seconds) */
18+
export const DEFAULT_TIMEOUT = 10_000
19+
20+
/** Long timeout for operations that may take a while, like server installation (60 seconds) */
21+
export const LONG_TIMEOUT = 60_000
22+
23+
function getExecutablePath(): string {
24+
const platform = process.platform
25+
const arch = process.arch
26+
const basePath = path.join(__dirname, '..', '..', 'out')
27+
28+
if (platform === 'darwin') {
29+
return path.join(
30+
basePath,
31+
`ToolHive-darwin-${arch}`,
32+
'ToolHive.app',
33+
'Contents',
34+
'MacOS',
35+
'ToolHive'
36+
)
37+
} else if (platform === 'win32') {
38+
return path.join(basePath, `ToolHive-win32-${arch}`, 'ToolHive.exe')
39+
} else {
40+
return path.join(basePath, `ToolHive-linux-${arch}`, 'ToolHive')
41+
}
42+
}
43+
44+
function getThvPath(): string {
45+
const platform = process.platform
46+
const arch = process.arch
47+
const binName = platform === 'win32' ? 'thv.exe' : 'thv'
48+
return path.join(__dirname, '..', '..', 'bin', `${platform}-${arch}`, binName)
49+
}
50+
51+
function deleteTestGroupViaCli(): void {
52+
const thvPath = getThvPath()
53+
try {
54+
execSync(`"${thvPath}" group rm "${TEST_GROUP_NAME}" --with-workloads`, {
55+
input: 'y\n',
56+
stdio: ['pipe', 'ignore', 'ignore'],
57+
})
58+
} catch {
59+
// Group doesn't exist, which is fine
60+
}
61+
}
62+
63+
async function createAndActivateTestGroup(window: Page): Promise<void> {
64+
await window.getByRole('button', { name: /add a group/i }).click()
65+
await window.getByRole('dialog').waitFor()
66+
await window.getByLabel(/name/i).fill(TEST_GROUP_NAME)
67+
await window.getByRole('button', { name: /create/i }).click()
68+
await window.getByRole('dialog').waitFor({ state: 'hidden' })
69+
await window.getByRole('link', { name: TEST_GROUP_NAME }).click()
70+
await window
71+
.getByRole('heading', { name: /add your first mcp server/i })
72+
.waitFor()
73+
}
74+
1375
export const test = base.extend<ElectronFixtures>({
1476
// eslint-disable-next-line no-empty-pattern
1577
electronApp: async ({}, use) => {
78+
deleteTestGroupViaCli()
79+
1680
const app = await electron.launch({
17-
args: ['.'],
81+
executablePath: getExecutablePath(),
1882
recordVideo: { dir: 'test-videos' },
83+
args: ['--no-sandbox'],
1984
})
2085

2186
await use(app)
2287

88+
// Disable quit confirmation dialog to prevent hang on close
2389
const window = await app.firstWindow()
2490
await window.evaluate(() => {
25-
// mock confirm quit
2691
localStorage.setItem('doNotShowAgain_confirm_quit', 'true')
2792
})
2893

29-
// Ensure app is closed and video is recorded
30-
const appToClose = app.close()
31-
await appToClose
94+
await app.close()
3295
},
3396

3497
window: async ({ electronApp }, use) => {
3598
const window = await electronApp.firstWindow()
99+
100+
// Disable quit confirmation dialog to prevent hang on close
101+
await window.evaluate(() => {
102+
localStorage.setItem('doNotShowAgain_confirm_quit', 'true')
103+
})
104+
105+
await window.getByRole('link', { name: /mcp servers/i }).waitFor()
106+
await createAndActivateTestGroup(window)
107+
36108
await use(window)
109+
110+
deleteTestGroupViaCli()
37111
},
38112
})
39113

‎e2e-tests/open-toolhive.spec.ts‎

Lines changed: 28 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,44 @@
11
import { test, expect } from './fixtures/electron'
22

3-
test('app starts and stops properly', async ({ window }) => {
4-
const header = window.getByRole('heading', {
5-
name: /add your first mcp server/i,
3+
test('app starts and shows test group', async ({ window }) => {
4+
// Verify the test group was created by the fixture
5+
const groupLink = window.getByRole('link', {
6+
name: /playwright-automated-test-fixture/i,
67
})
7-
await header.waitFor()
8-
await expect(header).toBeVisible()
8+
await expect(groupLink).toBeVisible()
99
})
1010

11-
test('install & uninstall server', async ({ window }) => {
12-
await window.getByRole('link', { name: /browse registry/i }).click()
13-
await window
14-
.getByRole('button', {
15-
name: /everything/i,
16-
})
17-
.click()
18-
await window
19-
.getByRole('button', {
20-
name: /install server/i,
21-
})
22-
.click()
23-
24-
await window.getByRole('tab', { name: /network isolation/i }).click()
25-
26-
await window
27-
.getByRole('switch', { name: /enable outbound network filtering/i })
28-
.click()
29-
30-
await window.getByRole('button', { name: /add a host/i }).click()
31-
await window.getByRole('textbox', { name: /host 1/i }).fill('wikipedia.org')
32-
33-
await window.getByRole('button', { name: /add a host/i }).click()
34-
await window.getByRole('textbox', { name: /host 2/i }).fill('google.com')
35-
36-
await window.getByRole('textbox', { name: /host 2/i }).fill('google')
37-
await expect(window.getByText(/invalid host format/i)).toBeVisible()
38-
39-
await window.getByRole('textbox', { name: /host 2/i }).fill('google.com')
40-
await expect(window.getByText(/invalid host format/i)).not.toBeVisible()
41-
42-
await window.getByRole('textbox', { name: /host 2/i }).fill('google')
43-
await expect(window.getByText(/invalid host format/i)).toBeVisible()
44-
45-
await window.getByRole('tab', { name: /configuration/i }).click()
11+
test('install and uninstall server from registry', async ({ window }) => {
12+
await window.getByRole('button', { name: /add an mcp server/i }).click()
13+
await window.getByRole('menuitem', { name: /from the registry/i }).click()
14+
await window.getByText('everything').click()
15+
await window.getByRole('button', { name: /install server/i }).click()
16+
await window.getByRole('dialog').waitFor()
4617

18+
// Use custom name and test group to avoid conflicts
19+
const serverNameInput = window.getByLabel('Server name')
20+
await serverNameInput.fill('e2e-test-server')
21+
await window.getByRole('combobox', { name: 'Group' }).click()
4722
await window
48-
.getByRole('button', {
49-
name: /install server/i,
50-
})
23+
.getByRole('option', { name: 'playwright-automated-test-fixture' })
5124
.click()
5225

53-
await expect(window.getByText(/invalid host format/i)).toBeVisible()
26+
await window.getByRole('button', { name: /install server/i }).click()
5427

55-
await window.getByRole('textbox', { name: /host 2/i }).fill('google.com')
56-
await expect(window.getByText(/invalid host format/i)).not.toBeVisible()
28+
// The View button is a Link inside a Button, so we use getByRole('link')
29+
const viewButton = window.getByRole('link', { name: /^view$/i })
30+
await viewButton.waitFor()
31+
await viewButton.click()
5732

58-
await window.getByRole('textbox', { name: /host 2/i }).fill('google')
59-
await expect(window.getByText(/invalid host format/i)).toBeVisible()
33+
await window.getByText('Running').waitFor()
6034

61-
await window.getByRole('textbox', { name: /host 2/i }).fill('google.com')
62-
await expect(window.getByText(/invalid.*/i)).not.toBeVisible()
35+
await window.getByRole('button', { name: /more options/i }).click()
36+
await window.getByRole('menuitem', { name: /remove/i }).click()
37+
await window.getByRole('button', { name: /remove/i }).click()
6338

64-
await window
65-
.getByRole('button', {
66-
name: /install server/i,
67-
})
68-
.click()
69-
70-
await window
71-
.getByRole('link', {
72-
name: /view/i,
73-
})
74-
.click()
75-
await window.getByText('Running').waitFor()
76-
await window
77-
.getByRole('button', {
78-
name: /more options/i,
79-
})
80-
.click()
81-
await window
82-
.getByRole('menuitem', {
83-
name: /remove/i,
84-
})
85-
.click()
86-
await window
87-
.getByRole('button', {
88-
name: /remove/i,
89-
})
90-
.click()
91-
const header = window.getByRole('heading', {
39+
const emptyState = window.getByRole('heading', {
9240
name: /add your first mcp server/i,
9341
})
94-
await header.waitFor()
95-
await expect(header).toBeVisible()
42+
await emptyState.waitFor()
43+
await expect(emptyState).toBeVisible()
9644
})

‎forge.config.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,9 @@ const config: ForgeConfig = {
204204
[FuseV1Options.RunAsNode]: false,
205205
[FuseV1Options.EnableCookieEncryption]: true,
206206
[FuseV1Options.EnableNodeOptionsEnvironmentVariable]: false,
207-
[FuseV1Options.EnableNodeCliInspectArguments]: false,
207+
// Enable for e2e tests (Playwright requires this), disable for production releases
208+
[FuseV1Options.EnableNodeCliInspectArguments]:
209+
!process.env.PRODUCTION_BUILD,
208210
[FuseV1Options.EnableEmbeddedAsarIntegrityValidation]: true,
209211
[FuseV1Options.OnlyLoadAppFromAsar]: true,
210212
}),

‎package.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
},
1111
"scripts": {
1212
"start": "electron-forge start",
13-
"e2e": "env CI=true sh -c \"tsc -b --clean && tsc -b && electron-forge package && playwright test\"",
13+
"e2e": "pnpm package && pnpm e2e:prebuilt",
14+
"e2e:prebuilt": "playwright test",
1415
"package": "tsc -b --clean && tsc -b && electron-forge package",
1516
"make": "tsc -b --clean && tsc -b && electron-forge make",
1617
"prettier": "prettier . --check",
@@ -27,8 +28,7 @@
2728
"generate-client": "pnpm run update-api && cd api && openapi-ts",
2829
"generate-client:nofetch": "cd api && openapi-ts",
2930
"generate-icons": "ts-node scripts/generate-icons.ts",
30-
"knip": "knip",
31-
"playwright": "playwright"
31+
"knip": "knip"
3232
},
3333
"keywords": [],
3434
"license": "Apache-2.0",
@@ -48,7 +48,7 @@
4848
"@electron/fuses": "^2.0.0",
4949
"@eslint/js": "^9.25.0",
5050
"@hey-api/openapi-ts": "0.85.2",
51-
"@playwright/test": "^1.54.1",
51+
"@playwright/test": "^1.57.0",
5252
"@tailwindcss/postcss": "^4.1.10",
5353
"@tanstack/router-plugin": "^1.120.11",
5454
"@testing-library/jest-dom": "^6.6.3",

‎playwright.config.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import { defineConfig } from '@playwright/test'
2+
import { LONG_TIMEOUT } from './e2e-tests/fixtures/electron'
23

34
/**
45
* See https://playwright.dev/docs/test-configuration.
56
*/
67
export default defineConfig({
78
testDir: './e2e-tests',
8-
timeout: 2 * 60 * 1000,
9+
timeout: LONG_TIMEOUT,
910
fullyParallel: false,
1011
forbidOnly: !!process.env.CI,
1112
retries: process.env.CI ? 2 : 0,

‎pnpm-lock.yaml‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)