diff --git a/packages/experiments-realm/sample-command-card.gts b/packages/experiments-realm/sample-command-card.gts new file mode 100644 index 00000000000..5fcd2e41764 --- /dev/null +++ b/packages/experiments-realm/sample-command-card.gts @@ -0,0 +1,160 @@ +import { action } from '@ember/object'; +import { on } from '@ember/modifier'; +import { tracked } from '@glimmer/tracking'; + +import { + CardDef, + field, + contains, + Component, +} from 'https://cardstack.com/base/card-api'; +import StringField from 'https://cardstack.com/base/string'; + +import { Command, type CommandContext } from '@cardstack/runtime-common'; +import SaveCardCommand from '@cardstack/boxel-host/commands/save-card'; +import { getDefaultWritableRealmURL } from '@cardstack/boxel-host/test-helpers'; + +// --------------------------------------------------------------------------- +// Input / Output cards for SampleCommand + +class GreetInput extends CardDef { + static displayName = 'Greet Input'; + @field name = contains(StringField); +} + +export class GreetOutput extends CardDef { + static displayName = 'Greet Output'; + @field message = contains(StringField); +} + +// --------------------------------------------------------------------------- +// SampleCommand — greets the given name, falling back to "World" + +export class SampleCommand extends Command< + typeof GreetInput, + typeof GreetOutput +> { + static actionVerb = 'Greet'; + name = 'SampleCommand'; + description = 'Returns a greeting for the given name.'; + + async getInputType() { + return GreetInput; + } + + protected async run(input: GreetInput): Promise { + let name = input?.name?.trim() || 'World'; + return new GreetOutput({ message: `Hello, ${name}!` }); + } +} + +// --------------------------------------------------------------------------- +// SampleCommandCard + +export class SampleCommandCard extends CardDef { + static displayName = 'Sample Command Card'; + + @field name = contains(StringField); + + static isolated = class Isolated extends Component { + @tracked commandOutput: string | null = null; + @tracked isRunning = false; + @tracked commandError: string | null = null; + @tracked savedCardId: string | null = null; + @tracked isSaving = false; + + @action + async runSampleCommand() { + let commandContext = this.args.context?.commandContext as + | CommandContext + | undefined; + if (!commandContext) { + this.commandError = 'No commandContext available'; + return; + } + this.isRunning = true; + this.commandOutput = null; + this.commandError = null; + try { + let cmd = new SampleCommand(commandContext); + let result = await cmd.execute({ name: this.args.model.name }); + this.commandOutput = result?.message ?? null; + } catch (e: unknown) { + this.commandError = e instanceof Error ? e.message : String(e); + } finally { + this.isRunning = false; + } + } + + @action + async runSampleSave() { + let commandContext = this.args.context?.commandContext as + | CommandContext + | undefined; + if (!commandContext) { + this.commandError = 'No commandContext available'; + return; + } + this.isSaving = true; + this.savedCardId = null; + this.commandError = null; + try { + let newCard = new GreetOutput({ + message: `Hello, ${this.args.model.name || 'World'}!`, + }); + let cmd = new SaveCardCommand(commandContext); + await cmd.execute({ + card: newCard, + realm: getDefaultWritableRealmURL(), + }); + this.savedCardId = newCard.id ?? null; + } catch (e: unknown) { + this.commandError = e instanceof Error ? e.message : String(e); + } finally { + this.isSaving = false; + } + } + + + }; + + static embedded = class Embedded extends Component { + + }; +} diff --git a/packages/experiments-realm/sample-command-card.test.gts b/packages/experiments-realm/sample-command-card.test.gts new file mode 100644 index 00000000000..735e6e403e4 --- /dev/null +++ b/packages/experiments-realm/sample-command-card.test.gts @@ -0,0 +1,118 @@ +import { module, test } from 'qunit'; +import { click, waitFor } from '@ember/test-helpers'; +import { + renderCard, + visitOperatorMode, + fetchCard, + setupTestRealm, +} from '@cardstack/boxel-host/test-helpers'; +import { SampleCommand, SampleCommandCard } from './sample-command-card'; + +// --------------------------------------------------------------------------- +// SampleCommand unit-style tests +// --------------------------------------------------------------------------- + +test('SampleCommand returns a greeting for the given name', async (assert) => { + let cmd = new SampleCommand({} as any); + let result = await cmd.execute({ name: 'Alice' }); + assert.strictEqual(result?.message, 'Hello, Alice!'); +}); + +test('SampleCommand falls back to "World" when name is empty', async (assert) => { + let cmd = new SampleCommand({} as any); + let result = await cmd.execute({ name: '' }); + assert.strictEqual(result?.message, 'Hello, World!'); +}); + +test('SampleCommand falls back to "World" when name is whitespace-only', async (assert) => { + let cmd = new SampleCommand({} as any); + let result = await cmd.execute({ name: ' ' }); + assert.strictEqual(result?.message, 'Hello, World!'); +}); + +// --------------------------------------------------------------------------- +// SampleCommandCard field tests +// --------------------------------------------------------------------------- + +test('SampleCommandCard can be instantiated with a name field', async (assert) => { + let card = new SampleCommandCard({ name: 'Bob' }); + assert.strictEqual(card.name, 'Bob'); +}); + +test('SampleCommandCard has a displayName', async (assert) => { + assert.strictEqual(SampleCommandCard.displayName, 'Sample Command Card'); +}); + +// --------------------------------------------------------------------------- +// DOM assertion example +// --------------------------------------------------------------------------- + +test('assert.dom — greeting renders correct text', async (assert) => { + let el = document.createElement('div'); + el.innerHTML = `

Hello, World!

`; + document.body.appendChild(el); + + assert.dom('.greeting', el).exists(); + assert.dom('.greeting', el).hasText('Hello, World!'); + assert.dom('.greeting', el).hasTagName('p'); + + el.remove(); +}); + +// --------------------------------------------------------------------------- +// Acceptance-style test: renders the live app in the iframe +// --------------------------------------------------------------------------- + +test('SampleCommandCard renders name in isolated view', async (assert) => { + let card = new SampleCommandCard({ name: 'Alice' }); + + await renderCard(card); + + assert.dom('[data-test-sample-command-card]').exists(); + assert.dom('[data-test-name]').hasText('Name: Alice'); + assert.dom('[data-test-run-button]').exists(); +}); + +// --------------------------------------------------------------------------- +// Command execution test: render card and run a command via button click +// --------------------------------------------------------------------------- + +test('SampleCommand runs and shows output when button is clicked', async (assert) => { + let card = new SampleCommandCard({ name: 'Alice' }); + + await renderCard(card); + + assert.dom('[data-test-run-button]').exists(); + + await click('[data-test-run-button]'); + + assert.dom('[data-test-output]').hasText('Hello, Alice!'); +}); + +// --------------------------------------------------------------------------- +// Save test: uses an in-memory test realm so nothing is written to the live realm +// --------------------------------------------------------------------------- + +module('SampleCommandCard | save', function (hooks) { + setupTestRealm(hooks, { contents: {}, realmURL: 'http://test-realm/' }); + + test('runSampleSave saves a new card to the realm', async (assert) => { + let card = new SampleCommandCard({ name: 'Alice' }); + + await renderCard(card); + + assert.dom('[data-test-save-button]').exists(); + + await click('[data-test-save-button]'); + await waitFor('[data-test-saved-id]'); + + assert.dom('[data-test-error]').doesNotExist(); + assert.dom('[data-test-saved-id]').exists('saved card id is shown'); + + let savedId = document + .querySelector('[data-test-saved-id]')! + .textContent!.trim(); + let saved = await fetchCard(savedId); + assert.strictEqual(saved.data.attributes.message, 'Hello, Alice!'); + }); +}); diff --git a/packages/experiments-realm/tsconfig.json b/packages/experiments-realm/tsconfig.json index f1208dafd2c..666abf290d2 100644 --- a/packages/experiments-realm/tsconfig.json +++ b/packages/experiments-realm/tsconfig.json @@ -23,6 +23,8 @@ "experimentalDecorators": true, "paths": { "https://cardstack.com/base/*": ["../base/*"], + "@cardstack/boxel-host/commands/*": ["../host/app/commands/*"], + "@cardstack/boxel-host/test-helpers": ["../host/app/test-helpers/index.ts"], "@cardstack/catalog/commands/*": ["../catalog-realm/commands/*"], "@cardstack/catalog/*": ["../catalog-realm/catalog-app/*"] }, diff --git a/packages/host/app/components/operator-mode/code-submode/module-inspector.gts b/packages/host/app/components/operator-mode/code-submode/module-inspector.gts index d08f466c0d8..b9ddad701de 100644 --- a/packages/host/app/components/operator-mode/code-submode/module-inspector.gts +++ b/packages/host/app/components/operator-mode/code-submode/module-inspector.gts @@ -9,6 +9,7 @@ import { tracked } from '@glimmer/tracking'; import Eye from '@cardstack/boxel-icons/eye'; import FileCog from '@cardstack/boxel-icons/file-cog'; +import FlaskConical from '@cardstack/boxel-icons/flask-conical'; import Schema from '@cardstack/boxel-icons/schema'; import { task } from 'ember-concurrency'; @@ -52,6 +53,7 @@ import Playground from '@cardstack/host/components/operator-mode/code-submode/pl import SchemaEditor from '@cardstack/host/components/operator-mode/code-submode/schema-editor'; import SpecPreview from '@cardstack/host/components/operator-mode/code-submode/spec-preview'; import SpecPreviewBadge from '@cardstack/host/components/operator-mode/code-submode/spec-preview-badge'; +import TestRunnerPanel from '@cardstack/host/components/operator-mode/code-submode/test-runner-panel'; import ToggleButton from '@cardstack/host/components/operator-mode/code-submode/toggle-button'; import PreviewPanel from '@cardstack/host/components/operator-mode/preview-panel/index'; @@ -97,6 +99,7 @@ const moduleInspectorPanels: Record = { schema: Schema, preview: Eye, spec: FileCog, + test: FlaskConical, }; interface ModuleInspectorSignature { @@ -541,7 +544,11 @@ export default class ModuleInspector extends Component } get displayInspector() { - return this.args.selectedDeclaration; + return this.args.selectedDeclaration || this.isTestFile; + } + + private get isTestFile() { + return this.args.readyFile?.url?.endsWith('.test.gts') ?? false; } satisfies TemplateOnlyComponent<{ model: TestRunnerModel }>; + +export default RouteTemplate(TestRunner); diff --git a/packages/host/app/test-helpers/index.ts b/packages/host/app/test-helpers/index.ts new file mode 100644 index 00000000000..9c363edd8ab --- /dev/null +++ b/packages/host/app/test-helpers/index.ts @@ -0,0 +1,200 @@ +// @ts-ignore +import { precompileTemplate } from '@ember/template-compilation'; +import { getContext, render, settled } from '@ember/test-helpers'; + +import { provide as provideConsumeContext } from 'ember-provide-consume-context/test-support'; + +import { + CardContextName, + GetCardContextName, + GetCardsContextName, + GetCardCollectionContextName, +} from '@cardstack/runtime-common'; + +import { getCard } from '../resources/card-resource'; +import { getCardCollection } from '../resources/card-collection'; +import PrerenderedCardSearch from '../components/prerendered-card-search'; + +import type LoaderService from '../services/loader-service'; +import type StoreService from '../services/store'; + +function getOwnerServices() { + let context = getContext() as any; + let owner = context?.owner; + if (!owner) throw new Error('No test context owner — was setContext() called?'); + let loaderService = owner.lookup('service:loader-service') as LoaderService; + let commandService = owner.lookup('service:command-service') as any; + let store = owner.lookup('service:store') as StoreService; + let router = owner.lookup('service:router') as any; + let network = owner.lookup('service:network') as any; + return { loaderService, commandService, store, router, network }; +} + +type SetupHooks = { + beforeEach: (fn: () => Promise | void) => void; + afterEach: (fn: () => Promise | void) => void; +}; + +export function setupTestRealm( + hooks: SetupHooks, + { + contents, + realmURL, + }: { contents: Record; realmURL?: string }, +) { + let handler: ((req: Request) => Promise) | null = null; + + hooks.beforeEach(async () => { + // Resolve the realm URL at test-run time so we can fall back to the live + // default writable realm when no explicit URL is provided. This way the + // in-memory handler intercepts the same URL the component would save to, + // without needing a targetRealmUrl field on the card. + let resolvedURL = realmURL ?? getDefaultWritableRealmURL(); + if (!resolvedURL.endsWith('/')) resolvedURL += '/'; + + let store = new Map(); + for (let [path, data] of Object.entries(contents)) { + let key = new URL(path, resolvedURL).href; + store.set(key, data); + } + + handler = async (req: Request) => { + if (!req.url.startsWith(resolvedURL)) return null; + let url = req.url.split('?')[0]; + + if (req.method === 'GET' || req.method === 'HEAD') { + let data = store.get(url) ?? store.get(url + '.json'); + if (!data) return null; + return new Response(JSON.stringify(data), { + headers: { 'Content-Type': 'application/vnd.card+json' }, + }); + } + + if (req.method === 'POST') { + let body = await req.json(); + let typeName = body?.data?.meta?.adoptsFrom?.name ?? 'Card'; + let id = `${resolvedURL}${typeName}/${crypto.randomUUID()}`; + let doc = { ...body, data: { ...body.data, id } }; + store.set(id, doc); + return new Response(JSON.stringify(doc), { + status: 201, + headers: { 'Content-Type': 'application/vnd.card+json' }, + }); + } + + if (req.method === 'PATCH') { + let body = await req.json(); + let doc = { ...body, data: { ...body.data, id: url } }; + store.set(url, doc); + return new Response(JSON.stringify(doc), { + headers: { 'Content-Type': 'application/vnd.card+json' }, + }); + } + + if (req.method === 'DELETE') { + store.delete(url); + return new Response(null, { status: 204 }); + } + + return null; + }; + + (getContext() as any).__testRealmURL = resolvedURL; + let { network } = getOwnerServices(); + network.virtualNetwork.mount(handler, { prepend: true }); + }); + + hooks.afterEach(async () => { + await settled(); + (getContext() as any).__testRealmURL = null; + if (handler) { + let { network } = getOwnerServices(); + network.virtualNetwork.unmount(handler); + handler = null; + } + }); +} + +export function getDefaultWritableRealmURL(): string { + let testRealmURL = (getContext() as any)?.__testRealmURL; + if (testRealmURL) return testRealmURL; + + let { store } = getOwnerServices(); + // Access allRealmsInfo directly to avoid going through defaultWritableRealm, + // which calls matrixService.userName → matrixService.client and throws if the + // matrix SDK hasn't finished loading yet. + let allRealmsInfo: Record = + (store as any).realm?.allRealmsInfo ?? {}; + let writable = Object.entries(allRealmsInfo) + .filter(([, meta]) => meta.canWrite) + .sort(([, a], [, b]) => a.info.name.localeCompare(b.info.name)); + let first = writable[0]; + if (!first) throw new Error('No default writable realm found'); + return first[0]; +} + +export async function fetchCard(url: string): Promise { + let { loaderService } = getOwnerServices(); + let response = await loaderService.loader.fetch(url, { + headers: { Accept: 'application/vnd.card+json' }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch card ${url}: ${response.status} ${response.statusText}`); + } + return response.json(); +} + +export async function renderCard( + card: any, + opts?: { commandContext?: any; format?: string }, +) { + let { loaderService, commandService, store } = getOwnerServices(); + let api = await loaderService.loader.import('https://cardstack.com/base/card-api'); + let Component = api.getComponent(card); + let format = opts?.format ?? 'isolated'; + let resolvedCommandContext = opts?.commandContext ?? commandService.commandContext; + + provideConsumeContext(CardContextName, { + commandContext: resolvedCommandContext, + getCard, + getCards: store.getSearchResource.bind(store), + getCardCollection, + store, + prerenderedCardSearchComponent: PrerenderedCardSearch, + mode: 'operator', + submode: 'interact', + }); + provideConsumeContext(GetCardContextName, getCard); + provideConsumeContext(GetCardsContextName, store.getSearchResource.bind(store)); + provideConsumeContext(GetCardCollectionContextName, getCardCollection); + + let context = { commandContext: resolvedCommandContext }; + await render( + precompileTemplate('', { + strictMode: true, + scope: () => ({ Component, format, context }), + }), + ); + await settled(); +} + +export async function visitOperatorMode( + state: { + stacks?: Array>; + submode?: string; + [key: string]: unknown; + } = {}, +) { + let { router } = getOwnerServices(); + let operatorModeState = { + stacks: state.stacks ?? [], + submode: state.submode ?? 'interact', + workspaceChooserOpened: false, + aiAssistantOpen: false, + ...state, + }; + await router.transitionTo('index', { + queryParams: { operatorModeState: JSON.stringify(operatorModeState) }, + }); + await settled(); +} diff --git a/packages/host/docs/test-runner.md b/packages/host/docs/test-runner.md new file mode 100644 index 00000000000..a329f5de4c9 --- /dev/null +++ b/packages/host/docs/test-runner.md @@ -0,0 +1,223 @@ +# Card Test Runner — Host App API + +This document explains how to run `.test.gts` realm files via the prerender server's `/run-tests` endpoint, and how to reproduce a test run manually using `curl` or a fetch call. + +--- + +## Overview + +``` +.test.gts file in realm + → POST /run-tests (prerender server) + └── PagePool.getPage() + └── page.goto('/_test-runner?module=&nonce=[&filter=]') + └── /_test-runner route boots in Puppeteer page + └── test module imported via loaderService + └── tests filtered by name (if filter provided), then run sequentially + └── results written to DOM → [data-test-results] + → JSON response with pass/fail/counts +``` + +--- + +## Prerender Server — `/run-tests` endpoint + +**Method:** `POST` +**Content-Type:** `application/json` + +### Request body + +```json +{ + "data": { + "attributes": { + "moduleUrl": "http://localhost:4201/experiments/sample-command-card.test.gts", + "auth": "", + "realm": "http://localhost:4201/experiments/", + "affinityType": "realm", + "affinityValue": "http://localhost:4201/experiments/" + } + } +} +``` + +| Field | Type | Required | Description | +|---|---|---|---| +| `moduleUrl` | string | yes | Full URL of the `.test.gts` file in the realm | +| `auth` | string | yes | JWT or session auth for the realm | +| `realm` | string | yes | Realm base URL | +| `affinityType` | `"realm"` or `"user"` | yes | Page pool affinity — use `"realm"` for test runs | +| `affinityValue` | string | yes | Realm URL (when `affinityType = "realm"`) | +| `filter` | string | no | Exact test name to run. Omit to run all tests. | + +### Response body (HTTP 201) + +```json +{ + "data": { + "type": "test-result", + "id": "http://localhost:4201/experiments/sample-command-card.test.gts", + "attributes": { + "status": "pass", + "total": 5, + "passed": 5, + "failed": 0, + "duration": 1240, + "tests": [ + { + "name": "SampleCommand returns a greeting for the given name", + "status": "pass", + "duration": 38 + }, + { + "name": "SampleCommand falls back to World when name is empty", + "status": "pass", + "duration": 12 + } + ] + } + }, + "meta": { + "timing": { "launchMs": 320, "renderMs": 920, "totalMs": 1240 }, + "pool": { "pageId": "...", "affinityType": "realm", "affinityValue": "...", "reused": false, "evicted": false, "timedOut": false } + } +} +``` + +### curl example + +```sh +PRERENDER_URL="http://localhost:4221" +REALM_URL="http://localhost:4201/experiments/" +AUTH_TOKEN="" + +curl -s -X POST "${PRERENDER_URL}/run-tests" \ + -H "Content-Type: application/json" \ + -d "{ + \"data\": { + \"attributes\": { + \"moduleUrl\": \"${REALM_URL}sample-command-card.test.gts\", + \"auth\": \"${AUTH_TOKEN}\", + \"realm\": \"${REALM_URL}\", + \"affinityType\": \"realm\", + \"affinityValue\": \"${REALM_URL}\" + } + } + }" | jq '.data.attributes' +``` + +--- + +## Host App — `/_test-runner` route + +The prerender server navigates a Puppeteer page to: + +``` +/_test-runner?module=&nonce= +``` + +The route: +1. Reads `module` and `nonce` from query params +2. Exposes `globalThis.__boxelTestRegistry` for test modules to register tests +3. Dynamically imports the test module via `loaderService.loader.import(moduleUrl)` +4. Runs collected tests sequentially, catching errors per-test +5. Serialises results to `[data-test-results]` in the DOM (hidden `
`)
+6. Sets `data-prerender-status="ready"` (or `"error"`) on `[data-prerender-id="test-runner"]`
+
+The prerender server polls the `data-prerender-status` attribute until it is set, then reads `[data-test-results]` to get the JSON payload.
+
+### Visiting the route manually in a browser
+
+Navigate to (experiments realm running locally):
+
+```
+http://localhost:4200/_test-runner?module=http%3A%2F%2Flocalhost%3A4201%2Fexperiments%2Fsample-command-card.test.gts&nonce=1
+```
+
+Open DevTools → Elements and inspect:
+
+```html
+
+ +
+``` + +Or in the console: + +```js +JSON.parse(document.querySelector('[data-test-results]').textContent) +``` + +--- + +## Writing a test file + +Test files are `.test.gts` files that live in the realm alongside card definitions. + +```ts +// my-card.test.gts + +// Future: import { test } from '@cardstack/test-support'; +// For now, use the global registry provided by /_test-runner: + +declare const globalThis: { + __boxelTestRegistry?: { + test(name: string, fn: () => Promise | void): void; + }; +}; + +function test(name: string, fn: () => Promise | void): void { + globalThis.__boxelTestRegistry!.test(name, fn); +} + +test('my card has the expected title', async () => { + let { MyCard } = await import('./my-card'); + let card = new MyCard({ title: 'Hello' }); + if (card.title !== 'Hello') { + throw new Error(`Expected 'Hello' but got '${card.title}'`); + } +}); +``` + +See [packages/experiments-realm/sample-command-card.test.gts](../../experiments-realm/sample-command-card.test.gts) for a complete example that tests both a command and a card. + +--- + +## Example: SampleCommandCard + +[packages/experiments-realm/sample-command-card.gts](../../experiments-realm/sample-command-card.gts) demonstrates: + +- A `SampleCommand` that accepts a name and returns a greeting +- A card component with a **Run Sample Command** button +- A `@tracked commandOutput` variable updated with the command result +- `data-test-*` attributes on all interactive elements for DOM assertions + +[packages/experiments-realm/sample-command-card.test.gts](../../experiments-realm/sample-command-card.test.gts) contains runnable tests for that card. + +--- + +## Test result format + +```ts +type TestResult = { + name: string; + status: 'pass' | 'fail' | 'error'; + duration: number; // milliseconds + error?: { + message: string; + stack?: string; + actual?: unknown; + expected?: unknown; + }; +}; + +type RunTestsResponse = { + status: 'pass' | 'fail' | 'error'; + total: number; + passed: number; + failed: number; + duration: number; // total milliseconds + tests: TestResult[]; +}; +``` diff --git a/packages/local-types/index.d.ts b/packages/local-types/index.d.ts index 2e02b1d4e74..6ad875592b5 100644 --- a/packages/local-types/index.d.ts +++ b/packages/local-types/index.d.ts @@ -52,3 +52,27 @@ declare module '@ember/component' { import '../../runtime-common/global'; import './matrix-js-sdk'; + +import 'qunit'; +import 'qunit-dom'; + +declare module '@cardstack/boxel-host/test-helpers' { + export function renderCard( + card: any, + opts?: { commandContext?: any; format?: string }, + ): Promise; + export function visitOperatorMode(state?: { + stacks?: Array>; + submode?: string; + [key: string]: unknown; + }): Promise; + export function setupTestRealm( + hooks: { + beforeEach: (fn: () => Promise | void) => void; + afterEach: (fn: () => Promise | void) => void; + }, + options: { contents: Record; realmURL?: string }, + ): void; + export function fetchCard(url: string): Promise; + export function getDefaultWritableRealmURL(): string; +} diff --git a/packages/local-types/package.json b/packages/local-types/package.json index e81b7aa0040..2aa985e3670 100644 --- a/packages/local-types/package.json +++ b/packages/local-types/package.json @@ -6,6 +6,8 @@ "content-tag": "catalog:", "ember-source": "catalog:", "@glint/template": "^1.3.0", - "ember-provide-consume-context": "^0.7.0" + "ember-provide-consume-context": "^0.7.0", + "@types/qunit": "catalog:", + "qunit-dom": "catalog:" } } diff --git a/packages/realm-server/handlers/handle-run-tests.ts b/packages/realm-server/handlers/handle-run-tests.ts new file mode 100644 index 00000000000..ab9b98fd4a1 --- /dev/null +++ b/packages/realm-server/handlers/handle-run-tests.ts @@ -0,0 +1,133 @@ +import type Koa from 'koa'; + +import { + fetchRealmPermissions, + type DBAdapter, + type RealmPermissions, + type Prerenderer, +} from '@cardstack/runtime-common'; + +import { + fetchRequestFromContext, + sendResponseForBadRequest, + sendResponseForUnauthorizedRequest, + sendResponseForForbiddenRequest, + sendResponseForSystemError, + setContextResponse, +} from '../middleware'; +import type { RealmServerTokenClaim } from '../utils/jwt'; + +export default function handleRunTests({ + prerenderer, + dbAdapter, + createPrerenderAuth, +}: { + prerenderer?: Prerenderer; + dbAdapter: DBAdapter; + createPrerenderAuth: ( + userId: string, + permissions: RealmPermissions, + ) => string; +}) { + return async (ctxt: Koa.Context) => { + if (!prerenderer) { + await sendResponseForSystemError( + ctxt, + 'Prerender proxy is not configured on this realm server', + ); + return; + } + + if (ctxt.method !== 'POST') { + await sendResponseForBadRequest(ctxt, 'Only POST is supported'); + return; + } + + let token = ctxt.state.token as RealmServerTokenClaim | undefined; + if (!token?.user) { + await sendResponseForUnauthorizedRequest( + ctxt, + 'Missing or invalid realm token', + ); + return; + } + + let request = await fetchRequestFromContext(ctxt); + let rawBody = await request.text(); + let json: any; + try { + json = rawBody ? JSON.parse(rawBody) : undefined; + } catch { + await sendResponseForBadRequest(ctxt, 'Body must be valid JSON'); + return; + } + + let attrs = json?.data?.attributes; + if (!attrs) { + await sendResponseForBadRequest( + ctxt, + 'Request body must include data.attributes', + ); + return; + } + if (!attrs.moduleUrl) { + await sendResponseForBadRequest(ctxt, 'Missing moduleUrl in attributes'); + return; + } + if (!attrs.realm) { + await sendResponseForBadRequest(ctxt, 'Missing realm in attributes'); + return; + } + + let permissionsByUser = await fetchRealmPermissions( + dbAdapter, + new URL(attrs.realm), + ); + let userPermissions = permissionsByUser[token.user]; + if (!userPermissions?.length) { + await sendResponseForForbiddenRequest( + ctxt, + `${token.user} does not have permissions in ${attrs.realm}`, + ); + return; + } + + let permissions: RealmPermissions = { + [attrs.realm]: userPermissions, + }; + let auth = createPrerenderAuth(token.user, permissions); + + let result; + try { + result = await prerenderer.runTests({ + moduleUrl: attrs.moduleUrl, + realm: attrs.realm, + affinityType: 'realm', + affinityValue: attrs.realm, + auth, + filter: attrs.filter, + }); + } catch (err) { + let msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err); + await sendResponseForSystemError(ctxt, `Error running tests: ${msg}`); + return; + } + + await setContextResponse( + ctxt, + new Response( + JSON.stringify({ + data: { + type: 'test-result', + id: attrs.moduleUrl, + attributes: result, + }, + }), + { + status: 201, + headers: { 'content-type': 'application/vnd.api+json' }, + }, + ), + ); + }; +} diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index 0291f353a40..b6694ebf44e 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -13,6 +13,7 @@ import { type FileExtractResponse, type FileRenderResponse, type RunCommandResponse, + type RunTestsResponse, } from '@cardstack/runtime-common'; import { ecsMetadata, @@ -100,6 +101,14 @@ export function buildPrerenderApp(options: { commandInput?: unknown; }; + type RunTestsRouteArgs = RouteBaseArgs & { + affinityType: AffinityType; + affinityValue: string; + realm: string; + moduleUrl: string; + filter?: string; + }; + type RouteParseResult = { args?: A; missing: string[]; @@ -223,6 +232,54 @@ export function buildPrerenderApp(options: { }; }; + let parseRunTestsAttributes = ( + attrs: any, + ): RouteParseResult => { + let rawAuth = attrs.auth; + let rawAffinityType = attrs.affinityType; + let rawAffinityValue = attrs.affinityValue; + let rawRealm = attrs.realm; + let moduleUrl = attrs.moduleUrl; + let filter = + typeof attrs.filter === 'string' && attrs.filter.trim().length > 0 + ? (attrs.filter as string) + : undefined; + let renderOptions = parseRenderOptions(attrs); + let missing: string[] = []; + if (!isNonEmptyString(rawAuth)) missing.push('auth'); + if (rawAffinityType !== 'realm' && rawAffinityType !== 'user') + missing.push('affinityType'); + if (!isNonEmptyString(rawAffinityValue)) missing.push('affinityValue'); + if (!isNonEmptyString(rawRealm)) missing.push('realm'); + if (!isNonEmptyString(moduleUrl)) missing.push('moduleUrl'); + return { + args: + missing.length > 0 + ? undefined + : { + affinityType: rawAffinityType as AffinityType, + affinityValue: rawAffinityValue as string, + auth: rawAuth as string, + realm: rawRealm as string, + moduleUrl: moduleUrl as string, + filter, + renderOptions, + }, + missing, + missingMessage: + 'Missing or invalid required attributes: auth, moduleUrl, realm, affinityType, affinityValue', + logTarget: (moduleUrl as string | undefined) ?? '', + responseId: (moduleUrl as string | undefined) ?? 'test-runner', + rejectionLogDetails: `affinityType=${ + (rawAffinityType as string | undefined) ?? '' + } affinityValue=${(rawAffinityValue as string | undefined) ?? ''} realm=${ + (rawRealm as string | undefined) ?? '' + } moduleUrlProvided=${Boolean(isNonEmptyString(moduleUrl))} authProvided=${ + typeof rawAuth === 'string' && rawAuth.trim().length > 0 + }`, + }; + }; + function registerPrerenderRoute( path: string, options: { @@ -472,6 +529,28 @@ export function buildPrerenderApp(options: { }, ); + registerPrerenderRoute( + '/run-tests', + { + requestDescription: 'test-runner', + responseType: 'test-result', + infoLabel: 'test-runner', + warnTimeoutMessage: (target) => `test run of ${target} timed out`, + errorContext: '/run-tests', + errorMessage: 'Error running tests', + parseAttributes: parseRunTestsAttributes, + execute: (args) => + prerenderer.runTests({ + affinityType: args.affinityType, + affinityValue: args.affinityValue, + auth: args.auth, + moduleUrl: args.moduleUrl, + filter: args.filter, + }), + drainingPromise: options.drainingPromise, + }, + ); + // File render route needs additional attributes (fileData, types) // beyond what registerPrerenderRoute handles, so we register it directly. router.post('/prerender-file-render', async (ctxt: Koa.Context) => { diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 11f9d267c97..fe42e94964b 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -8,6 +8,7 @@ import { type FileRenderArgs, logger, type RunCommandResponse, + type RunTestsResponse, } from '@cardstack/runtime-common'; import { BrowserManager } from './browser-manager'; import { PagePool } from './page-pool'; @@ -377,6 +378,43 @@ export class Prerenderer { } } + async runTests({ + affinityType, + affinityValue, + auth, + moduleUrl, + filter, + opts, + }: { + affinityType: AffinityType; + affinityValue: string; + auth: string; + moduleUrl: string; + filter?: string; + opts?: { timeoutMs?: number }; + }): Promise<{ + response: RunTestsResponse; + timings: { launchMs: number; renderMs: number }; + pool: PoolMeta; + }> { + if (this.#stopped) { + throw new Error('Prerenderer has been stopped and cannot be used'); + } + try { + return await this.#renderRunner.runTestsAttempt({ + affinityType, + affinityValue, + auth, + moduleUrl, + filter, + opts, + }); + } catch (e) { + log.error(`test run attempt failed (module ${moduleUrl})`, e); + throw e; + } + } + async prerenderFileExtract({ affinityType, affinityValue, diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index 4053997196e..4b007a76fe3 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -8,6 +8,7 @@ import { type FileRenderArgs, type RenderRouteOptions, type RunCommandResponse, + type RunTestsResponse, logger, } from '@cardstack/runtime-common'; import { @@ -242,6 +243,20 @@ export function createRemotePrerenderer( }, ); }, + async runTests({ affinityType, affinityValue, auth, moduleUrl, realm, filter }) { + return await requestWithRetry( + 'run-tests', + 'run-tests-request', + { + affinityType, + affinityValue, + auth, + moduleUrl, + realm, + ...(filter ? { filter } : {}), + }, + ); + }, }; } @@ -272,6 +287,7 @@ function validatePrerenderAttributes( } if ( requestType !== 'run-command-request' && + requestType !== 'run-tests-request' && (typeof attrs.url !== 'string' || attrs.url.trim().length === 0) ) { missing.add('url'); diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 19939ce0d19..f4808a2bc04 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -8,6 +8,7 @@ import { type FileRenderArgs, type RenderRouteOptions, type RunCommandResponse, + type RunTestsResponse, type AffinityType, serializeRenderRouteOptions, logger, @@ -1451,4 +1452,184 @@ export class RenderRunner { log.warn(`Error disposing affinity %s on %s:`, affinityKey, reason, e); } } + + async runTestsAttempt({ + affinityType, + affinityValue, + auth, + moduleUrl, + filter, + opts, + }: { + affinityType: AffinityType; + affinityValue: string; + auth: string; + moduleUrl: string; + filter?: string; + opts?: { timeoutMs?: number }; + }): Promise<{ + response: RunTestsResponse; + timings: { launchMs: number; renderMs: number }; + pool: { + pageId: string; + affinityType: AffinityType; + affinityValue: string; + reused: boolean; + evicted: boolean; + timedOut: boolean; + }; + }> { + this.#nonce++; + let affinityKey = toAffinityKey({ affinityType, affinityValue }); + log.info( + `running tests for module ${moduleUrl}, nonce=${this.#nonce} affinity=${affinityKey}`, + ); + + const { page, reused, launchMs, pageId, release } = + await this.#getPageForAffinity(affinityKey, auth); + const poolInfo = { + pageId: pageId ?? 'unknown', + affinityType, + affinityValue, + reused, + evicted: false, + timedOut: false, + }; + this.#pagePool.resetConsoleErrors(pageId); + + try { + let renderStart = Date.now(); + let nonce = String(this.#nonce); + + await page.evaluate( + (sessionAuth, nonce, moduleUrl) => { + localStorage.setItem('boxel-session', sessionAuth); + localStorage.setItem('boxel-test-runner-nonce', nonce); + localStorage.setItem('boxel-test-runner-module', moduleUrl); + }, + auth, + nonce, + moduleUrl, + ); + + let queryParams: Record = { module: moduleUrl, nonce }; + if (filter) { + queryParams['filter'] = filter; + } + await transitionTo(page, 'test-runner', { queryParams }); + log.info( + 'test-runner navigated for module: %s nonce: %s filter: %s', + moduleUrl, + nonce, + filter ?? '', + ); + + let waitResult = await withTimeout( + page, + async () => { + await page.waitForFunction( + (expectedNonce: string) => { + let containers = Array.from( + document.querySelectorAll( + '[data-prerender][data-prerender-id="test-runner"]', + ), + ) as HTMLElement[]; + let container = + containers.find( + (candidate) => + candidate.dataset.prerenderNonce === expectedNonce, + ) ?? null; + if (!container) { + return false; + } + let status = container.dataset.prerenderStatus ?? ''; + return ['ready', 'error'].includes(status); + }, + {}, + nonce, + ); + return true; + }, + opts?.timeoutMs, + ); + + if (isRenderError(waitResult)) { + let errorResponse: RunTestsResponse = { + status: 'error', + total: 0, + passed: 0, + failed: 0, + duration: 0, + tests: [], + }; + return { + response: errorResponse, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } + + let payload = await page.evaluate((expectedNonce: string) => { + let containers = Array.from( + document.querySelectorAll( + '[data-prerender][data-prerender-id="test-runner"]', + ), + ) as HTMLElement[]; + let container = + containers.find( + (candidate) => candidate.dataset.prerenderNonce === expectedNonce, + ) ?? null; + let resultsElement = container?.querySelector( + '[data-test-results]', + ) as HTMLElement | null; + let raw = (resultsElement?.textContent ?? '').trim(); + return { raw }; + }, nonce); + + let response: RunTestsResponse; + try { + response = JSON.parse(payload.raw) as RunTestsResponse; + } catch { + response = { + status: 'error', + total: 0, + passed: 0, + failed: 0, + duration: 0, + tests: [], + }; + } + + return { + response, + timings: { launchMs, renderMs: Date.now() - renderStart }, + pool: poolInfo, + }; + } catch (e) { + log.error('Error running tests in headless chrome:', e); + let errorResponse: RunTestsResponse = { + status: 'error', + total: 0, + passed: 0, + failed: 0, + duration: 0, + tests: [], + }; + return { + response: errorResponse, + timings: { launchMs, renderMs: 0 }, + pool: poolInfo, + }; + } finally { + // Close the page after tests — each test run gets an isolated page + try { + await page.close(); + } catch (e) { + log.warn('Error closing test-runner page:', e); + } + release(); + // Evict the closed page from the pool so the next test run gets a fresh page + void this.#pagePool.disposeAffinity(affinityKey, { awaitIdle: false }); + } + } } diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index 94516331f42..2b4a4f1ef01 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -38,6 +38,7 @@ import handleGetBoxelClaimedDomainRequest from './handlers/handle-get-boxel-clai import handleClaimBoxelDomainRequest from './handlers/handle-claim-boxel-domain'; import handleDeleteBoxelClaimedDomainRequest from './handlers/handle-delete-boxel-claimed-domain'; import handlePrerenderProxy from './handlers/handle-prerender-proxy'; +import handleRunTests from './handlers/handle-run-tests'; import handleSearch from './handlers/handle-search'; import handleSearchPrerendered from './handlers/handle-search-prerendered'; import handleRealmInfo from './handlers/handle-realm-info'; @@ -200,6 +201,15 @@ export function createRoutes(args: CreateRoutesArgs) { createPrerenderAuth, }), ); + router.post( + '/_run-tests', + jwtMiddleware(args.realmSecretSeed), + handleRunTests({ + prerenderer: args.prerenderer, + dbAdapter: args.dbAdapter, + createPrerenderAuth, + }), + ); router.post( '/_publish-realm', jwtMiddleware(args.realmSecretSeed), diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index a1df4bef82a..18a25fff769 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -151,12 +151,44 @@ export type RunCommandResponse = { error?: string | null; }; +export type TestResult = { + name: string; + status: 'pass' | 'fail' | 'error'; + duration: number; + error?: { + message: string; + stack?: string; + actual?: unknown; + expected?: unknown; + }; +}; + +export type RunTestsResponse = { + status: 'pass' | 'fail' | 'error'; + total: number; + passed: number; + failed: number; + duration: number; + tests: TestResult[]; +}; + +export type RunTestsArgs = { + moduleUrl: string; + auth: string; + affinityType: AffinityType; + affinityValue: string; + realm: string; + /** Run only the test whose name exactly matches this string. Omit to run all tests. */ + filter?: string; +}; + export interface Prerenderer { prerenderCard(args: PrerenderCardArgs): Promise; prerenderModule(args: ModulePrerenderArgs): Promise; prerenderFileExtract(args: ModulePrerenderArgs): Promise; prerenderFileRender(args: FileRenderArgs): Promise; runCommand(args: RunCommandArgs): Promise; + runTests(args: RunTestsArgs): Promise; } export type RealmAction = 'read' | 'write' | 'realm-owner' | 'assume-user'; diff --git a/packages/runtime-common/module-syntax.ts b/packages/runtime-common/module-syntax.ts index 5e8a77b2f7e..f313a8732e5 100644 --- a/packages/runtime-common/module-syntax.ts +++ b/packages/runtime-common/module-syntax.ts @@ -11,6 +11,7 @@ import { type FunctionDeclaration, type ClassDeclaration, type Reexport, + type TestDeclaration, isInternalReference, } from './schema-analysis-plugin'; import type { Options as RemoveOptions } from './remove-field-plugin'; @@ -49,6 +50,7 @@ export type { FunctionDeclaration, ClassDeclaration, Reexport, + TestDeclaration, }; export { isInternalReference }; diff --git a/packages/runtime-common/schema-analysis-plugin.ts b/packages/runtime-common/schema-analysis-plugin.ts index 83ddd8fbd84..22b892628dc 100644 --- a/packages/runtime-common/schema-analysis-plugin.ts +++ b/packages/runtime-common/schema-analysis-plugin.ts @@ -43,6 +43,12 @@ export interface Reexport extends BaseDeclaration { type: 'reexport'; } +export interface TestDeclaration extends BaseDeclaration { + type: 'test'; + testName: string; + path: NodePath; +} + export interface PossibleField { card: ClassReference; type: ExternalReference | InternalReference; @@ -54,7 +60,8 @@ export type Declaration = | PossibleCardOrFieldDeclaration | FunctionDeclaration | ClassDeclaration - | Reexport; + | Reexport + | TestDeclaration; export interface Options { possibleCardsOrFields: PossibleCardOrFieldDeclaration[]; //cards may not be exports @@ -181,6 +188,33 @@ const coreVisitor = { state.insideCard = false; }, }, + ExpressionStatement: { + enter(path: NodePath, state: State) { + if (!path.parentPath.isProgram()) { + return; + } + let expr = path.node.expression; + if ( + expr.type !== 'CallExpression' || + expr.callee.type !== 'Identifier' || + expr.callee.name !== 'test' + ) { + return; + } + let [firstArg] = expr.arguments; + if (!firstArg || firstArg.type !== 'StringLiteral') { + return; + } + let testName = firstArg.value; + state.opts.declarations.push({ + localName: testName, + exportName: undefined, + path, + type: 'test', + testName, + }); + }, + }, Decorator(path: NodePath, state: State) { if (!state.insideCard) { return; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e256d11a86d..2d17b3771dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2216,6 +2216,9 @@ importers: '@glint/template': specifier: ^1.3.0 version: 1.3.0 + '@types/qunit': + specifier: 'catalog:' + version: 2.19.13 content-tag: specifier: 'catalog:' version: 4.1.0 @@ -2225,6 +2228,9 @@ importers: ember-source: specifier: 'catalog:' version: 5.12.0(patch_hash=2586bd032e74105d65b7953ccaae1cd1d1175047a58ba89ec0057f1f8b7b7fe1)(@glimmer/component@2.0.0)(@glint/template@1.3.0)(rsvp@4.8.5)(webpack@5.104.1) + qunit-dom: + specifier: 'catalog:' + version: 3.5.0 packages/matrix: devDependencies: diff --git a/testing-cards-scope.md b/testing-cards-scope.md new file mode 100644 index 00000000000..a92e81d3365 --- /dev/null +++ b/testing-cards-scope.md @@ -0,0 +1,224 @@ +# Testing Cards in Code Mode — Scope & Requirements + +## Summary + +Card authors can write **integration test** files (`.test.gts`) that live in the realm alongside their card definitions. Tests run against a fresh `MemoryRealm` seeded with data — covering real realm interactions such as card creation, search, and linked card resolution. Opening a test file in code mode shows a Test Runner panel in the RHS. Tests run in an isolated browser context so they cannot mutate host app state. The user writes assertions only — setup infrastructure is provided by the framework. + +--- + +## Functional Requirements + +### 1. LHS — File Detection & Test Indicators +- The LHS file tree distinguishes `.test.gts` files visually (e.g. a test icon or badge) +- When a `.test.gts` file is opened, code mode detects the file type and switches to test mode +- The LHS shows a per-test status indicator next to each `test()` call in the file after a run: + - No indicator — not yet run + - Green dot — passed + - Red dot — failed +- Indicators update after each run and clear when the file is modified + +### 2. Test Execution — Isolated Puppeteer Page per Test File +- Each test file run gets a dedicated Puppeteer page via the prerender server (`/run-tests` endpoint) +- The page navigates to `/_test-runner?module=` on the host app, boots a fresh `MemoryRealm` (SQLite WASM + `TestRealmAdapter`), and runs all tests in the file +- Tests must **not** mutate the state of the host app — the Puppeteer page is a separate process with its own JS context, services, and localStorage +- The page is closed and discarded after the run — no cleanup needed, no data persisted to the live realm +- Results are returned to code mode as a single JSON response from the prerender server once all tests complete + +### 3. Test Runner Lifecycle — Tracked by Code Mode +- Code mode keeps track of the active test runner instance (one per open test file) +- The runner has explicit states: `idle | running | pass | fail | error` +- Switching to a different test file tears down the previous runner and initialises a new one +- The runner does **not** auto-run on file open — the user triggers it manually + +### 4. Test Result Format + +Results are returned as a single JSON payload once all tests complete: + +```ts +type TestResult = { + name: string; + status: 'pass' | 'fail' | 'error'; + duration: number; + error?: { + message: string; + stack?: string; + actual?: unknown; + expected?: unknown; + }; +}; + +type TestRunResult = { + status: 'pass' | 'fail' | 'error'; + total: number; + passed: number; + failed: number; + duration: number; + tests: TestResult[]; +}; +``` + +### 6. Manual Re-run After Editor Change +- After the user edits the test file in Monaco, they can manually trigger a re-run via the **Run** button +- The runner reloads the test module (not cached) on each run to pick up the latest editor content +- Auto-run on save is a future enhancement, not in scope + +### 7. Available Imports for Test Authors +The test file has access to a set of provided helpers — no Ember/QUnit internals exposed: + +```ts +import { test, setupRealm } from '@cardstack/test-support'; +import { render, assert } from '@cardstack/test-support'; +``` + +| Helper | Description | +|---|---| +| `test(name, fn)` | Register a test case | +| `setupRealm(hooks, files)` | Seed the MemoryRealm with initial card files before the test run | +| `render(component, container)` | Render a Glimmer component into a DOM container | +| `assert` | QUnit assert instance — gives access to `qunit-dom` assertions | + +### 8. Assertions — DOM Only +- Assertions are DOM-based only — no network mocking, no service stubbing +- `qunit-dom` is the assertion library (already in project): `assert.dom('[data-test-name]').hasText('Alice')` +- Test authors are not expected to import from `@ember/test-helpers`, QUnit, or any Ember internals +- Any Ember internals needed (e.g. `settled()`) are used internally by the `render` helper — not exposed +- **QUnit is the underlying test runner** inside the isolated Puppeteer page — `QUnit.module`, `QUnit.test`, and `QUnit.done()` are used internally to collect the result payload. The global state collision problem does not apply here because the Puppeteer page owns its own isolated JS context with no host app QUnit instance present. The pattern mirrors the existing setup (`Testem → Chrome → QUnit`) but replaces Testem with Puppeteer: `Puppeteer (prerender server) → Chrome → /_test-runner → QUnit` + +### 9. Test File Lives in the Realm +- Test files are stored in the realm as `.test.gts` files alongside card definitions +- They are editable in Monaco like any other realm file +- They are indexed by the realm as a `GtsFileDef` (same pattern as other `.gts` source files) — not as cards +- Indexing gives each test file a stable realm URL (e.g. `https://my-realm/my-card.test.gts`) which is passed to the prerender server's `/run-tests` endpoint +- The `/_test-runner` page on the host app receives this URL as a query param and dynamically imports the module via `loaderService` +- The existing Testem-based test runner (`tests/index.html`) is **not** used — it requires test files to be pre-bundled at build time and cannot load dynamic realm modules + +### 10. Setup Is Framework-Provided +- The test author does not write realm boot code — `setupRealm` handles it +- `setupRealm` accepts an initial file map to seed the MemoryRealm: + +```ts +setupRealm(hooks, { + 'person.gts': `...`, + 'PersonCard/alice.json': `{ "data": { ... } }`, +}); +``` + +- Between tests, the realm is reset to the seeded state (no bleed between tests) +- Base cards (`https://cardstack.com/base/`) are available automatically + +--- + +## UI Requirements + +### 1. LHS — File Tree +- `.test.gts` files shown with a distinct icon in the file tree +- After a run, each file shows a summary badge (e.g. `3/5` passed) next to its name in the tree +- Per-test line indicators shown inline in Monaco (green/red gutter markers next to each `test()` call) + +### 2. RHS — Test Runner Panel +- The Test Runner appears as a **tab alongside Spec and Playground** in the RHS panel — not a replacement +- The tab is only visible when a `.test.gts` file is open +- Switching to a non-test file hides the Test Runner tab +- A **Run** button is always visible regardless of runner state +- Any edit to the file in Monaco immediately invalidates results — all test bullets turn grey, tab resets to neutral +- Results remain visible (greyed out) after invalidation until the user manually triggers a new run + +### 3. RHS — Runner State Display + +| State | UI | +|---|---| +| `idle` | "Run Tests" prompt, no results, Run button prominent | +| `running` | Spinner, Run button disabled | +| `pass` | Green summary (e.g. "5 passed in 1.2s"), per-test pass indicators | +| `fail` | Red summary, failing tests expanded with assertion message and actual/expected diff | +| `error` | Module load or setup error displayed prominently with stack trace | + +### 4. RHS — Test Result List +- Each test case is listed individually with its name and status icon +- Failing tests show: + - The `qunit-dom` assertion message + - Actual vs expected diff inline + - Collapsible stack trace +- The panel retains the last run's results until the next run starts +- Results are displayed all at once when the full run completes + +--- + +## Out of Scope + +- Auto-run on save / watch mode (future) +- Test file indexing or search in the realm +- Snapshot testing +- Network/API mocking +- Multi-file test suites (one `.test.gts` per run) +- CI integration (separate concern) + +--- + +## Suggested Improvements to the Spec + +### 1. Isolated browser context — one Puppeteer page per test file +Each test file run gets its own dedicated Puppeteer page via the prerender server. The page is created fresh for the run, boots a `MemoryRealm`, runs all tests in that file, then is closed and discarded. + +``` +Code mode opens my-card.test.gts + → POST /run-tests { module: "https://my-realm/my-card.test.gts", auth } + → Prerender server + └── PagePool.getPage() ← borrows or creates a Puppeteer page + └── page.goto('/_test-runner?module=...') + └── MemoryRealm boots in page + └── tests run, results streamed back + └── page closed after run + → RHS panel renders streamed results +``` + +This decision should be reflected in the functional requirements — update Requirement 2 to specify Puppeteer (not iframe) as the isolation mechanism. The iframe approach is ruled out because: +- Shares the Chrome process with the host app — a runaway test can still affect host performance +- `postMessage` protocol adds complexity for streaming results +- Puppeteer page gives true process-level isolation consistent with how prerendering already works in Boxel + +### 2. Define the reset boundary between tests +The reset boundary is determined by what the test author writes, not imposed by the framework. The framework provides lifecycle hooks — the test author chooses the granularity: + +```ts +// realm seeded once for the whole file +setupRealm(hooks, { + 'person.gts': `...`, +}); + +// per-test setup +beforeEach(async () => { + await createCard(PersonCard, { firstName: 'Alice' }); +}); + +// per-test teardown +afterEach(async () => { + await deleteAllCards(PersonCard); +}); +``` + +Available hooks: `beforeAll`, `afterAll`, `beforeEach`, `afterEach` — same model as QUnit/Mocha. `setupRealm` is always `beforeAll` scope (boots the MemoryRealm once per file run). Per-test isolation is opt-in via `beforeEach`/`afterEach`. + +### 3. Scope is integration tests only +Unit-style tests (constructing cards in memory without a realm) are explicitly out of scope. All tests in `.test.gts` files are integration tests — they require a `MemoryRealm` and cover real realm interactions: + +- Card creation and retrieval +- Search and query results +- Linked card resolution +- Rendered output of components that read from the realm + +The MemoryRealm boot adds ~1-2s per run. This is acceptable for integration tests and avoids the complexity of supporting two different test styles with different infrastructure needs. + +### 4. Test Runner tab — placement and invalidation behaviour +The Test Runner appears as a **tab at the same level as Spec and Playground** in the RHS panel. It is only visible when a `.test.gts` file is open. + +**Tab states:** + +| Condition | Tab appearance | Test list | +|---|---|---| +| Not yet run | Neutral tab label | Empty / "Run to see results" | +| Run completed, all pass | Green tab indicator | All green bullet points | +| Run completed, some fail | Red tab indicator | Red bullets on failing tests | +| Module edited after last run | Tab resets to neutral | All bullets turn grey — results invalidated | + +**Invalidation rule:** any edit to the open `.test.gts` file in Monaco immediately invalidates the previous results — all bullets turn grey and the tab indicator resets. Results are stale until the user manually clicks Run. The framework does not auto-run on change. diff --git a/testing-cards-tasks.md b/testing-cards-tasks.md new file mode 100644 index 00000000000..69a5d2364da --- /dev/null +++ b/testing-cards-tasks.md @@ -0,0 +1,65 @@ +# Testing Cards in Code Mode — Implementation Tasks + +Commit-sized tasks grouped by layer. Work top-to-bottom — each group roughly depends on the one above it. + +--- + +## 1. Prerender Server + +- [ ] Add `/run-tests` endpoint to prerender server (Koa route, mirrors `/run-command` pattern) +- [ ] Integrate `PagePool` for test runs — borrow page, navigate to `/_test-runner?module=`, wait for `window.__testResults`, return JSON, close page + +--- + +## 2. Host App — `/_test-runner` Route + +- [ ] Add `/_test-runner` Ember route that accepts `?module=` query param +- [ ] Boot `MemoryRealm` (SQLite WASM + `TestRealmAdapter`) inside `/_test-runner` on route load +- [ ] Dynamically import test module via `loaderService` from the `module` query param +- [ ] Wire QUnit (`QUnit.module`, `QUnit.done`) to run imported tests and write result to `window.__testResults` + +--- + +## 3. Test Support Package + +- [ ] Create `@cardstack/test-support` package with `test`, `setupRealm`, `render`, `assert`, `beforeEach`, `afterEach`, `beforeAll`, `afterAll` exports +- [ ] Implement `setupRealm(hooks, files)` — wraps QUnit `beforeAll`, boots `MemoryRealm`, seeds with provided file map +- [ ] Implement `render(component, container)` — renders Glimmer component into DOM container, calls `settled()` internally +- [ ] Implement lifecycle hook exports — `beforeEach`, `afterEach`, `beforeAll`, `afterAll` delegating to QUnit module hooks + +--- + +## 4. Realm — Test File Indexing + +- [ ] Index `.test.gts` files as `GtsFileDef` in the realm — detect by extension, not treated as card definitions + +--- + +## 5. Code Mode — LHS + +- [ ] Detect `.test.gts` when opened in Monaco and switch code mode to test mode +- [ ] Show distinct icon for `.test.gts` files in the LHS file tree +- [ ] Show summary badge (e.g. `3/5 passed`) next to test file name in tree after a run +- [ ] Add Monaco gutter markers (green/red) next to each `test()` call after a run +- [ ] Grey out gutter markers on any Monaco edit after a run (invalidation) + +--- + +## 6. Code Mode — `TestRunnerService` + +- [ ] Create `TestRunnerService` — stores `{ moduleUrl, status, results }` keyed by test file URL +- [ ] Implement `runTests(moduleUrl)` — calls `POST /run-tests` on prerender server, updates state +- [ ] Implement invalidation — any Monaco content change sets status to `idle` and greys results without clearing them + +--- + +## 7. Code Mode — RHS Test Runner Tab + +- [ ] Add Test Runner tab to RHS panel alongside Spec and Playground tabs +- [ ] Show/hide Test Runner tab based on whether the open file is `.test.gts` +- [ ] Implement idle state UI — "Run Tests" prompt, prominent Run button, no results +- [ ] Implement running state UI — spinner, Run button disabled +- [ ] Implement pass state UI — green summary line, green bullet per test +- [ ] Implement fail state UI — red summary, failing tests expanded with `qunit-dom` message and actual/expected diff, collapsible stack trace +- [ ] Implement error state UI — module load or setup error with stack trace +- [ ] Implement invalidated state UI — all bullets grey, tab indicator neutral, results still visible diff --git a/testing-cards.md b/testing-cards.md new file mode 100644 index 00000000000..540a225f919 --- /dev/null +++ b/testing-cards.md @@ -0,0 +1,162 @@ +# Testing Cards in Code Mode + +## Context + +The goal is to allow card authors to write and run tests directly from code mode, with results shown in the RHS panel. Tests should deal with DOM assertions only and not expose Ember/QUnit internals. + +--- + +## Option 1: `.test.gts` Module + In-App Mini Runner + +When a `.test.gts` file is opened in code mode, the RHS switches from the Playground panel to a **Test Runner panel**. + +**Runner:** Custom micro-runner (~30 lines) — no global state, fully isolated instance, owned by the test panel service. + +**Assertions:** [`@testing-library/dom`](https://testing-library.com/docs/dom-testing-library/intro/) — browser-native, framework-agnostic, encourages accessible queries (`getByRole`, `getByText`). + +**Rendering:** Thin wrapper around `renderComponent` (from `@glimmer/core`) using the app's owner. `settled()` from `@ember/test-helpers` used internally — not exposed to test authors. + +```ts +// example test file +test('shows name', async (container) => { + const card = new PersonCard({ firstName: 'Alice' }); + await render(, container); + getByText(container, 'Alice'); +}); +``` + +**Pros:** +- Clean API, no framework leakage to test authors +- Uses existing `loaderService` for dynamic module import +- RHS panel is a natural extension of the existing playground panel + +**Cons:** +- Rendering Glimmer components requires careful owner/context injection +- `settled()` internals still needed under the hood +- No test isolation from host app services + +--- + +## Option 2: Iframe-Sandboxed Test Environment + +The RHS renders a sandboxed `