From 7cfd9674b8a9c04758075351c5565fb4c8632700 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 10 Mar 2026 15:49:02 -0700 Subject: [PATCH 01/22] Replace Volta with mise --- .github/actions/init/action.yml | 6 +- .gitignore | 1 + .mise.toml | 7 ++ README.md | 2 +- mise-tasks/lib/env-vars.sh | 74 ++++++++++++++++++++++ package.json | 4 -- packages/ai-bot/Dockerfile | 7 +- packages/ai-bot/package.json | 3 - packages/base/package.json | 3 - packages/billing/package.json | 3 - packages/bot-runner/Dockerfile | 7 +- packages/bot-runner/package.json | 3 - packages/boxel-homepage-realm/package.json | 3 - packages/boxel-icons/package.json | 3 - packages/boxel-ui/addon/package.json | 3 - packages/boxel-ui/test-app/package.json | 3 - packages/catalog-realm/package.json | 3 - packages/catalog/package.json | 3 - packages/eslint-plugin-boxel/package.json | 3 - packages/experiments-realm/package.json | 3 - packages/host/package.json | 3 - packages/matrix/package.json | 3 - packages/postgres/Dockerfile | 7 +- packages/postgres/package.json | 3 - packages/realm-server/package.json | 3 - packages/runtime-common/package.json | 3 - packages/skills-realm/package.json | 3 - packages/template-lint/package.json | 5 +- packages/vscode-boxel-tools/package.json | 3 - packages/workspace-sync-cli/package.json | 3 - 30 files changed, 101 insertions(+), 79 deletions(-) create mode 100644 .mise.toml create mode 100755 mise-tasks/lib/env-vars.sh diff --git a/.github/actions/init/action.yml b/.github/actions/init/action.yml index e567ffd2241..fdd1ede4c47 100644 --- a/.github/actions/init/action.yml +++ b/.github/actions/init/action.yml @@ -4,11 +4,9 @@ description: Setup common dependencies runs: using: composite steps: - - uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # 4.3.0 + - uses: jdx/mise-action@v2 with: - node-version-file: "package.json" - - - uses: cardstack/gh-actions/setup-pnpm-volta@main + install: true - name: Get pnpm store directory id: pnpm-cache diff --git a/.gitignore b/.gitignore index 70f6926c7eb..a580deb4c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ schema_tmp.sql test-results/.last-run.json .claude/settings.local.json traefik/dynamic/*.yml +.mise.local.toml diff --git a/.mise.toml b/.mise.toml new file mode 100644 index 00000000000..4c63370cf39 --- /dev/null +++ b/.mise.toml @@ -0,0 +1,7 @@ +[tools] +node = "24.13.1" +pnpm = "10.30.0" + +[env] +_.source = "./mise-tasks/lib/env-vars.sh" +NODE_NO_WARNINGS = "1" diff --git a/README.md b/README.md index 683145cfb9a..277d10ab272 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ For a quickstart, see [here](./QUICKSTART.md) - you will want the [Glint](https://marketplace.visualstudio.com/items?itemName=typed-ember.glint-vscode) vscode extension - you will want the [vscode-glimmer](https://marketplace.visualstudio.com/items?itemName=chiragpat.vscode-glimmer) vscode extension - you will want the [Playwright](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright) vscode extension -- this project uses [volta](https://volta.sh/) for Javascript toolchain version management. Make sure you have the latest verison of volta on your system and have define [the ENV var described here](https://docs.volta.sh/advanced/pnpm). +- this project uses [mise](https://mise.jdx.dev/) for tool version management. Install mise and run `mise install` from the repo root to get the correct Node.js and pnpm versions. - this project uses [pnpm](https://pnpm.io/) for package management. run `pnpm install` to install the project dependencies first. - this project uses [docker](https://docker.com). Make sure to install docker on your system. - Ensure that node_modules/.bin is in your path. e.g. include `export PATH="./node_modules/.bin:$PATH"` in your .zshrc diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh new file mode 100755 index 00000000000..20b22c975dd --- /dev/null +++ b/mise-tasks/lib/env-vars.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Shared environment computation for mise tasks. +# Sourced via .mise.toml [env] _.source — every mise task gets these variables automatically. + +compute_env_slug() { + echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g' +} + +export PGPORT="${PGPORT:-5435}" + +if [ -n "${BOXEL_ENVIRONMENT:-}" ]; then + ENV_SLUG=$(compute_env_slug "$BOXEL_ENVIRONMENT") + export ENV_SLUG + export ENV_MODE=true + + # Service URLs (Traefik hostnames) + export REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" + export REALM_TEST_URL="http://realm-test.${ENV_SLUG}.localhost" + export MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" + export WORKER_MGR_URL="http://worker.${ENV_SLUG}.localhost" + export WORKER_TEST_MGR_URL="http://worker-test.${ENV_SLUG}.localhost" + export PRERENDER_URL="${PRERENDER_URL:-http://prerender.${ENV_SLUG}.localhost}" + export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://prerender-mgr.${ENV_SLUG}.localhost}" + export ICONS_URL="http://icons.${ENV_SLUG}.localhost" + export HOST_URL="http://host.${ENV_SLUG}.localhost" + + # Database + export PGDATABASE="${PGDATABASE:-boxel_${ENV_SLUG}}" + export PGDATABASE_TEST="boxel_test_${ENV_SLUG}" + + # Ports (dynamic in env mode) + export REALM_PORT=0 + export TEST_PORT=0 + export WORKER_PORT=0 + export WORKER_TEST_PORT=0 + export PRERENDER_PORT=0 + export PRERENDER_MGR_PORT=0 + export ICONS_PORT=0 + + # Paths + export REALMS_ROOT="./realms/${ENV_SLUG}" + export REALMS_TEST_ROOT="./realms/${ENV_SLUG}_test" +else + export ENV_SLUG="" + export ENV_MODE="" + + # Service URLs (fixed localhost ports) + export REALM_BASE_URL="http://localhost:4201" + export REALM_TEST_URL="http://localhost:4202" + export MATRIX_URL_VAL="http://localhost:8008" + export WORKER_MGR_URL="http://localhost:4210" + export WORKER_TEST_MGR_URL="http://localhost:4211" + export PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" + export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://localhost:4222}" + export ICONS_URL="http://localhost:4206" + export HOST_URL="http://localhost:4200" + + # Database + export PGDATABASE="${PGDATABASE:-boxel}" + export PGDATABASE_TEST="boxel_test" + + # Ports (fixed in standard mode) + export REALM_PORT=4201 + export TEST_PORT=4202 + export WORKER_PORT=4210 + export WORKER_TEST_PORT=4211 + export PRERENDER_PORT=4221 + export PRERENDER_MGR_PORT=4222 + export ICONS_PORT=4206 + + # Paths + export REALMS_ROOT="./realms/localhost_4201" + export REALMS_TEST_ROOT="./realms/localhost_4202" +fi diff --git a/package.json b/package.json index 32e066fb6e1..0d4b17467c1 100644 --- a/package.json +++ b/package.json @@ -88,10 +88,6 @@ "engines": { "pnpm": "^10" }, - "volta": { - "node": "24.13.1", - "pnpm": "10.30.0" - }, "dependencies": { "matrix-js-sdk": "38.3.0" } diff --git a/packages/ai-bot/Dockerfile b/packages/ai-bot/Dockerfile index 4c073a7d596..f896a787444 100644 --- a/packages/ai-bot/Dockerfile +++ b/packages/ai-bot/Dockerfile @@ -1,7 +1,10 @@ -FROM node:24.13.1-slim +ARG NODE_VERSION=24.13.1 +ARG PNPM_VERSION=10.30.0 +FROM node:${NODE_VERSION}-slim ARG CI=1 +ARG PNPM_VERSION RUN apt-get update && apt-get install -y python3 build-essential -RUN npm install -g pnpm@10.30.0 +RUN npm install -g pnpm@${PNPM_VERSION} WORKDIR /boxel COPY . . RUN pnpm install --frozen-lockfile diff --git a/packages/ai-bot/package.json b/packages/ai-bot/package.json index 6fe396e714f..0d3ba4839f6 100644 --- a/packages/ai-bot/package.json +++ b/packages/ai-bot/package.json @@ -42,8 +42,5 @@ "start:development": "sh scripts/start-development.sh", "test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts", "get-chat": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/get_chat.ts" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/base/package.json b/packages/base/package.json index 87e4e242939..5fcad843a84 100644 --- a/packages/base/package.json +++ b/packages/base/package.json @@ -33,8 +33,5 @@ "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:hbs": "ember-template-lint .", "lint:hbs:fix": "ember-template-lint . --fix" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/billing/package.json b/packages/billing/package.json index 714b41f0a8b..5f2a7fb86ec 100644 --- a/packages/billing/package.json +++ b/packages/billing/package.json @@ -18,8 +18,5 @@ "lint:js": "eslint . --report-unused-disable-directives --cache", "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:glint": "glint" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/bot-runner/Dockerfile b/packages/bot-runner/Dockerfile index fb45e54f816..b6045778d2d 100644 --- a/packages/bot-runner/Dockerfile +++ b/packages/bot-runner/Dockerfile @@ -1,7 +1,10 @@ -FROM node:24.13.1-slim +ARG NODE_VERSION=24.13.1 +ARG PNPM_VERSION=10.30.0 +FROM node:${NODE_VERSION}-slim ARG CI=1 +ARG PNPM_VERSION RUN apt-get update && apt-get install -y python3 build-essential -RUN npm install -g pnpm@10.30.0 +RUN npm install -g pnpm@${PNPM_VERSION} WORKDIR /boxel COPY . . RUN pnpm install --frozen-lockfile diff --git a/packages/bot-runner/package.json b/packages/bot-runner/package.json index cc14b22209b..d3bc87c19d8 100644 --- a/packages/bot-runner/package.json +++ b/packages/bot-runner/package.json @@ -19,8 +19,5 @@ "start:development": "NODE_NO_WARNINGS=1 PGDATABASE=boxel PGPORT=5435 ts-node --transpileOnly main", "test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts", "lint": "glint" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/boxel-homepage-realm/package.json b/packages/boxel-homepage-realm/package.json index 0782e791003..962b78d1f9b 100644 --- a/packages/boxel-homepage-realm/package.json +++ b/packages/boxel-homepage-realm/package.json @@ -6,8 +6,5 @@ "boxel-homepage:setup": "sh ./scripts/boxel-homepage-setup.sh", "boxel-homepage:update": "pnpm boxel-homepage:setup && cd contents && git pull", "boxel-homepage:reset": "rm -rf contents && pnpm boxel-homepage:setup" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/boxel-icons/package.json b/packages/boxel-icons/package.json index e5b7ed67193..119745e5643 100644 --- a/packages/boxel-icons/package.json +++ b/packages/boxel-icons/package.json @@ -82,9 +82,6 @@ "engines": { "node": "24.13.1" }, - "volta": { - "extends": "../../package.json" - }, "ember": { "edition": "octane" }, diff --git a/packages/boxel-ui/addon/package.json b/packages/boxel-ui/addon/package.json index fcd211c269c..ea6c84cd094 100644 --- a/packages/boxel-ui/addon/package.json +++ b/packages/boxel-ui/addon/package.json @@ -113,9 +113,6 @@ "engines": { "node": "24.13.1" }, - "volta": { - "extends": "../../../package.json" - }, "ember": { "edition": "octane" }, diff --git a/packages/boxel-ui/test-app/package.json b/packages/boxel-ui/test-app/package.json index 283f85fd02e..ea87b85efd2 100644 --- a/packages/boxel-ui/test-app/package.json +++ b/packages/boxel-ui/test-app/package.json @@ -110,9 +110,6 @@ "engines": { "node": ">= 20" }, - "volta": { - "extends": "../../../package.json" - }, "ember": { "edition": "octane" }, diff --git a/packages/catalog-realm/package.json b/packages/catalog-realm/package.json index f2c8de0fd1c..670cf126375 100644 --- a/packages/catalog-realm/package.json +++ b/packages/catalog-realm/package.json @@ -24,8 +24,5 @@ "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:hbs": "ember-template-lint . --no-error-on-unmatched-pattern", "lint:hbs:fix": "ember-template-lint . --fix" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/catalog/package.json b/packages/catalog/package.json index 900e64ee3db..255cea2fe60 100644 --- a/packages/catalog/package.json +++ b/packages/catalog/package.json @@ -6,8 +6,5 @@ "catalog:setup": "[ -d contents ] || (git clone git@github.com:cardstack/boxel-catalog.git contents || git clone https://github.com/cardstack/boxel-catalog.git contents)", "catalog:update": "pnpm catalog:setup && cd contents && git pull", "catalog:reset": "rm -rf contents && pnpm catalog:setup" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/eslint-plugin-boxel/package.json b/packages/eslint-plugin-boxel/package.json index cdb2d143fd5..15dfcabf906 100644 --- a/packages/eslint-plugin-boxel/package.json +++ b/packages/eslint-plugin-boxel/package.json @@ -64,8 +64,5 @@ }, "engines": { "node": ">= 20" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/experiments-realm/package.json b/packages/experiments-realm/package.json index a3045ba4613..4be0b2488a7 100644 --- a/packages/experiments-realm/package.json +++ b/packages/experiments-realm/package.json @@ -25,8 +25,5 @@ "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:hbs": "ember-template-lint .", "lint:hbs:fix": "ember-template-lint . --fix" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/host/package.json b/packages/host/package.json index 8193d3fba88..5476e470286 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -199,9 +199,6 @@ "engines": { "node": ">= 20" }, - "volta": { - "extends": "../../package.json" - }, "ember": { "edition": "octane" }, diff --git a/packages/matrix/package.json b/packages/matrix/package.json index bf71b69f13e..e96d8b8b720 100644 --- a/packages/matrix/package.json +++ b/packages/matrix/package.json @@ -48,8 +48,5 @@ "wait": "sleep 10000000", "lint": "glint", "clean": "rm -rf ./synapse-data" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile index 7dd3c6a9d79..8e63ce5586c 100644 --- a/packages/postgres/Dockerfile +++ b/packages/postgres/Dockerfile @@ -1,7 +1,10 @@ -FROM node:24.13.1-slim +ARG NODE_VERSION=24.13.1 +ARG PNPM_VERSION=10.30.0 +FROM node:${NODE_VERSION}-slim ARG CI=1 +ARG PNPM_VERSION RUN apt-get update && apt-get install -y postgresql -RUN npm install -g pnpm@10.30.0 +RUN npm install -g pnpm@${PNPM_VERSION} WORKDIR /boxel COPY . . RUN pnpm install --frozen-lockfile diff --git a/packages/postgres/package.json b/packages/postgres/package.json index 2244782309b..3ad09afd941 100644 --- a/packages/postgres/package.json +++ b/packages/postgres/package.json @@ -29,8 +29,5 @@ "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:migrations": "node ./scripts/lint-migrations.js", "lint:glint": "glint" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json index 23e750fa1a6..6decd7ce976 100644 --- a/packages/realm-server/package.json +++ b/packages/realm-server/package.json @@ -137,9 +137,6 @@ "sync-stripe-products": "NODE_NO_WARNINGS=1 PGDATABASE=boxel PGPORT=5435 ts-node --transpileOnly scripts/sync-stripe-products.ts", "stripe": "docker run --rm --add-host=host.docker.internal:host-gateway -it stripe/stripe-cli:latest" }, - "volta": { - "extends": "../../package.json" - }, "dependencies": { "puppeteer": "^24.8.2" } diff --git a/packages/runtime-common/package.json b/packages/runtime-common/package.json index d9bf786c943..1b1fd5456be 100644 --- a/packages/runtime-common/package.json +++ b/packages/runtime-common/package.json @@ -95,8 +95,5 @@ "lint:js": "eslint . ", "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:types": "glint" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/skills-realm/package.json b/packages/skills-realm/package.json index e4fd3789102..e8c755bd91c 100644 --- a/packages/skills-realm/package.json +++ b/packages/skills-realm/package.json @@ -6,8 +6,5 @@ "skills:setup": "[ -d contents ] || (git clone git@github.com:cardstack/boxel-skills.git contents || git clone https://github.com/cardstack/boxel-skills.git contents)", "skills:update": "pnpm skills:setup && cd contents && git pull", "skills:reset": "rm -rf contents && pnpm skills:setup" - }, - "volta": { - "extends": "../../package.json" } } diff --git a/packages/template-lint/package.json b/packages/template-lint/package.json index adff159b738..fad3f88cf2e 100644 --- a/packages/template-lint/package.json +++ b/packages/template-lint/package.json @@ -12,8 +12,5 @@ "ember-template-recast": "^6.1.5" }, "author": "", - "license": "MIT", - "volta": { - "extends": "../../package.json" - } + "license": "MIT" } diff --git a/packages/vscode-boxel-tools/package.json b/packages/vscode-boxel-tools/package.json index 4c7d3c6ea41..599e61fedf4 100644 --- a/packages/vscode-boxel-tools/package.json +++ b/packages/vscode-boxel-tools/package.json @@ -241,9 +241,6 @@ } ] }, - "volta": { - "extends": "../../package.json" - }, "scripts": { "vscode:prepublish": "pnpm run compile", "vscode:package": "pnpm vsce package --no-dependencies", diff --git a/packages/workspace-sync-cli/package.json b/packages/workspace-sync-cli/package.json index a64d48bd02c..ea0582938e6 100644 --- a/packages/workspace-sync-cli/package.json +++ b/packages/workspace-sync-cli/package.json @@ -60,8 +60,5 @@ }, "publishConfig": { "access": "public" - }, - "volta": { - "extends": "../../package.json" } } From 554219084cadd5413c558d56264ded317786c961 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 10 Mar 2026 15:51:06 -0700 Subject: [PATCH 02/22] Update more references --- AGENTS.md | 2 +- QUICKSTART.md | 5 ++--- docs/card-field-rename.md | 2 +- packages/vscode-boxel-tools/README.md | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e42607b0ed8..e8eb7b832f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Tooling prerequisites -- We pin the toolchain with Volta (`.volta`), using the versions of Node.js and pnpm specified in package.json. Install Volta and set `VOLTA_FEATURE_PNPM=1` so pnpm is managed automatically—avoid global installs outside Volta. +- We pin the toolchain with [mise](https://mise.jdx.dev/) (`.mise.toml`), which manages Node.js and pnpm versions. Run `mise install` from the repo root to get the correct versions—avoid global installs outside mise. - pnpm is required for all scripts; use the pinned version as specified above. - Docker is required (Postgres, Synapse, SMTP, Stripe CLI container). Ensure the daemon is running and you can run `docker` without sudo. diff --git a/QUICKSTART.md b/QUICKSTART.md index 33bfc08b97b..50f9d470403 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -4,9 +4,8 @@ To build the entire repository and run the application, follow these steps: 1. The 2 main system dependencies to install are: - - [volta](https://docs.volta.sh/guide/getting-started) + - [mise](https://mise.jdx.dev/getting-started.html) - [docker](https://docs.docker.com/get-docker/) - - [pnpm](https://docs.volta.sh/advanced/pnpm) Note: If you don't have pnpm already on your system, **DON'T** install pnpm manually (volta will install it for you when you call `pnpm install`). 2. Clone the repo: @@ -17,7 +16,7 @@ To build the entire repository and run the application, follow these steps: 3. Install the package dependencies: ```zsh - echo 'export VOLTA_FEATURE_PNPM=1' >> ~/.profile && source ~/.profile + mise install pnpm install ``` diff --git a/docs/card-field-rename.md b/docs/card-field-rename.md index 6d0b6f11229..43230a86396 100644 --- a/docs/card-field-rename.md +++ b/docs/card-field-rename.md @@ -77,7 +77,7 @@ pnpm install Notes: -- This project uses Volta for Node + pnpm versions (see `package.json`). +- This project uses mise for Node + pnpm versions (see `.mise.toml`). ## AI conversations diff --git a/packages/vscode-boxel-tools/README.md b/packages/vscode-boxel-tools/README.md index 7775e2c46e5..9ab125dd832 100644 --- a/packages/vscode-boxel-tools/README.md +++ b/packages/vscode-boxel-tools/README.md @@ -73,7 +73,7 @@ For further assistance, check the [extension repository](https://github.com/card ## Development -This project uses Volta for toolchain management and pnpm for package management. +This project uses mise for toolchain management and pnpm for package management. Install dependencies with pnpm: From 80dbdf23f56d9f8b3319aec082668b613bba3880 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 10 Mar 2026 15:56:41 -0700 Subject: [PATCH 03/22] Change Actions version --- .github/actions/init/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/init/action.yml b/.github/actions/init/action.yml index fdd1ede4c47..eba9558461e 100644 --- a/.github/actions/init/action.yml +++ b/.github/actions/init/action.yml @@ -4,7 +4,7 @@ description: Setup common dependencies runs: using: composite steps: - - uses: jdx/mise-action@v2 + - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 with: install: true From 03ba46cbe5501547c7804923cc50497f14ba3d6f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 10 Mar 2026 16:41:42 -0700 Subject: [PATCH 04/22] Add Mise tasks --- mise-tasks/dev | 44 ++++++++++ mise-tasks/dev-minimal | 20 +++++ mise-tasks/infra/ensure-db | 18 +++++ mise-tasks/infra/ensure-pg | 18 +++++ mise-tasks/infra/ensure-synapse | 16 ++++ mise-tasks/infra/ensure-traefik | 7 ++ mise-tasks/infra/wait-for-prerender | 21 +++++ mise-tasks/services/icons | 8 ++ mise-tasks/services/prerender | 17 ++++ mise-tasks/services/prerender-mgr | 12 +++ mise-tasks/services/realm-server | 111 ++++++++++++++++++++++++++ mise-tasks/services/realm-server-base | 32 ++++++++ mise-tasks/services/test-realms | 69 ++++++++++++++++ mise-tasks/services/worker | 39 +++++++++ mise-tasks/services/worker-base | 21 +++++ mise-tasks/services/worker-test | 42 ++++++++++ mise-tasks/test-services-host | 70 ++++++++++++++++ mise-tasks/test-services-matrix | 14 ++++ 18 files changed, 579 insertions(+) create mode 100755 mise-tasks/dev create mode 100755 mise-tasks/dev-minimal create mode 100755 mise-tasks/infra/ensure-db create mode 100755 mise-tasks/infra/ensure-pg create mode 100755 mise-tasks/infra/ensure-synapse create mode 100755 mise-tasks/infra/ensure-traefik create mode 100755 mise-tasks/infra/wait-for-prerender create mode 100755 mise-tasks/services/icons create mode 100755 mise-tasks/services/prerender create mode 100755 mise-tasks/services/prerender-mgr create mode 100755 mise-tasks/services/realm-server create mode 100755 mise-tasks/services/realm-server-base create mode 100755 mise-tasks/services/test-realms create mode 100755 mise-tasks/services/worker create mode 100755 mise-tasks/services/worker-base create mode 100755 mise-tasks/services/worker-test create mode 100755 mise-tasks/test-services-host create mode 100755 mise-tasks/test-services-matrix diff --git a/mise-tasks/dev b/mise-tasks/dev new file mode 100755 index 00000000000..dd41e358151 --- /dev/null +++ b/mise-tasks/dev @@ -0,0 +1,44 @@ +#!/bin/sh +#MISE description="Start full dev stack (realm server, workers, test realms)" +#MISE dir="packages/realm-server" + +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" + +# Phase 1 readiness URLs +BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" +SKILLS_READY="http-get://${REALM_BASE_URL#http://}/skills/${READY_PATH}" +WAIT_URLS="${BASE_REALM_READY}|${SKILLS_READY}" + +if [ -z "${SKIP_CATALOG:-}" ]; then + WAIT_URLS="${WAIT_URLS}|http-get://${REALM_BASE_URL#http://}/catalog/${READY_PATH}" +fi +if [ -z "${SKIP_BOXEL_HOMEPAGE:-}" ]; then + WAIT_URLS="${WAIT_URLS}|http-get://${REALM_BASE_URL#http://}/boxel-homepage/${READY_PATH}" +fi +if [ -z "${SKIP_EXPERIMENTS:-}" ]; then + WAIT_URLS="${WAIT_URLS}|http-get://${REALM_BASE_URL#http://}/experiments/${READY_PATH}" +fi + +WAIT_URLS="${WAIT_URLS}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}" + +# Phase 2 readiness URL +NODE_TEST_REALM_READY="http-get://${REALM_TEST_URL#http://}/node-test/${READY_PATH}" + +# In environment mode, bootstrap infra before starting services +if [ -n "$BOXEL_ENVIRONMENT" ]; then + ./scripts/start-pg.sh + echo "Waiting for Postgres to accept connections..." + until docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done + REPO_ROOT="$(cd "../.." && pwd)" + "$REPO_ROOT/scripts/ensure-branch-db.sh" + echo "Running database migrations..." + pnpm migrate + ./scripts/start-matrix.sh +fi + +WAIT_ON_TIMEOUT=2400000 NODE_NO_WARNINGS=1 start-server-and-test \ + 'run-p -ln start:pg start:matrix start:smtp start:prerender-dev start:prerender-manager-dev start:worker-development start:development' \ + "$WAIT_URLS" \ + 'run-p -ln start:worker-test start:test-realms' \ + "$NODE_TEST_REALM_READY" \ + 'wait' diff --git a/mise-tasks/dev-minimal b/mise-tasks/dev-minimal new file mode 100755 index 00000000000..8136cb74815 --- /dev/null +++ b/mise-tasks/dev-minimal @@ -0,0 +1,20 @@ +#!/bin/sh +#MISE description="Start dev stack without optional realms (experiments, catalog, homepage, submission)" +#MISE dir="packages/realm-server" + +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" +BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" +NODE_TEST_REALM_READY="http-get://${REALM_TEST_URL#http://}/node-test/${READY_PATH}" + +WAIT_ON_TIMEOUT=900000 \ + SKIP_EXPERIMENTS=true \ + SKIP_CATALOG=true \ + SKIP_BOXEL_HOMEPAGE=true \ + SKIP_SUBMISSION=true \ + NODE_NO_WARNINGS=1 \ + start-server-and-test \ + 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:matrix start:smtp start:worker-development start:development' \ + "${BASE_REALM_READY}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}" \ + 'run-p -ln start:worker-test start:test-realms' \ + "${NODE_TEST_REALM_READY}" \ + 'wait' diff --git a/mise-tasks/infra/ensure-db b/mise-tasks/infra/ensure-db new file mode 100755 index 00000000000..a9cd94b5198 --- /dev/null +++ b/mise-tasks/infra/ensure-db @@ -0,0 +1,18 @@ +#!/bin/sh +#MISE description="Ensure per-environment database exists" +#MISE depends=["infra:ensure-pg"] + +# In standard mode the database is always 'boxel' — nothing to create. +[ -z "$BOXEL_ENVIRONMENT" ] && exit 0 + +DB_SUFFIX="${1:-$ENV_SLUG}" +DB_NAME="boxel_${DB_SUFFIX}" + +echo "Ensuring database '${DB_NAME}' exists..." + +if docker exec boxel-pg psql -U postgres -lqt | cut -d \| -f 1 | grep -qw "$DB_NAME"; then + echo "Database '${DB_NAME}' already exists." +else + docker exec boxel-pg createdb -U postgres "$DB_NAME" + echo "Created database '${DB_NAME}'." +fi diff --git a/mise-tasks/infra/ensure-pg b/mise-tasks/infra/ensure-pg new file mode 100755 index 00000000000..c5839e9bb83 --- /dev/null +++ b/mise-tasks/infra/ensure-pg @@ -0,0 +1,18 @@ +#!/bin/sh +#MISE description="Wait for PostgreSQL to be ready" + +COUNT=0 +MAX_ATTEMPTS=10 + +while ! docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1; do + if [ $COUNT -eq 0 ]; then + echo "Waiting for postgres" + fi + if [ $COUNT -eq $MAX_ATTEMPTS ]; then + echo "Failed to detect postgres after $MAX_ATTEMPTS attempts." + exit 1 + fi + COUNT=$((COUNT + 1)) + printf '.' + sleep 5 +done diff --git a/mise-tasks/infra/ensure-synapse b/mise-tasks/infra/ensure-synapse new file mode 100755 index 00000000000..644c483b54a --- /dev/null +++ b/mise-tasks/infra/ensure-synapse @@ -0,0 +1,16 @@ +#!/bin/sh +#MISE description="Ensure Synapse is running and users are registered" +#MISE dir="packages/matrix" + +pnpm assert-synapse-running + +# In environment mode, register users once per fresh Synapse data dir. +if [ -n "$BOXEL_ENVIRONMENT" ]; then + MARKER="./synapse-data-${ENV_SLUG}/.users-registered" + if [ ! -f "$MARKER" ]; then + echo "First start for environment — registering users..." + pnpm register-all && touch "$MARKER" + else + echo "Users already registered for this environment, skipping." + fi +fi diff --git a/mise-tasks/infra/ensure-traefik b/mise-tasks/infra/ensure-traefik new file mode 100755 index 00000000000..5fc3ad3f995 --- /dev/null +++ b/mise-tasks/infra/ensure-traefik @@ -0,0 +1,7 @@ +#!/bin/sh +#MISE description="Ensure Traefik is running (env mode only)" + +[ -z "$BOXEL_ENVIRONMENT" ] && exit 0 + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +sh "$REPO_ROOT/scripts/start-traefik.sh" diff --git a/mise-tasks/infra/wait-for-prerender b/mise-tasks/infra/wait-for-prerender new file mode 100755 index 00000000000..f453de24af9 --- /dev/null +++ b/mise-tasks/infra/wait-for-prerender @@ -0,0 +1,21 @@ +#!/bin/sh +#MISE description="Wait for prerender server to be ready" + +URL="${1:-${PRERENDER_MGR_URL}}" +TRIMMED_URL="${URL%/}/" +TIMEOUT_SECONDS=1500 +START_TIME=$(date +%s) + +echo "Waiting for prerender server at ${TRIMMED_URL}" + +while ! curl -sf "$TRIMMED_URL" >/dev/null 2>&1; do + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + if [ "$ELAPSED" -ge "$TIMEOUT_SECONDS" ]; then + echo "Timed out waiting for prerender server after ${TIMEOUT_SECONDS}s" + exit 1 + fi + sleep 1 +done + +echo "Prerender server is ready" diff --git a/mise-tasks/services/icons b/mise-tasks/services/icons new file mode 100755 index 00000000000..62115f614fa --- /dev/null +++ b/mise-tasks/services/icons @@ -0,0 +1,8 @@ +#!/bin/sh +#MISE description="Start boxel-icons HTTP server" +#MISE dir="packages/realm-server" + +# This script delegates to the existing start-icons.sh which handles +# both environment mode (dynamic port + Traefik registration) and +# standard mode (fixed port 4206). +exec sh ./scripts/start-icons.sh diff --git a/mise-tasks/services/prerender b/mise-tasks/services/prerender new file mode 100755 index 00000000000..3e71291cb6f --- /dev/null +++ b/mise-tasks/services/prerender @@ -0,0 +1,17 @@ +#!/bin/sh +#MISE description="Start prerender server for development" +#MISE depends=["infra:ensure-traefik"] +#MISE dir="packages/realm-server" + +if [ -n "$ENV_MODE" ]; then + DEFAULT_HOST_URL="${HOST_URL}" +else + DEFAULT_HOST_URL="http://localhost:4200" +fi + +NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + BOXEL_HOST_URL="${BOXEL_HOST_URL:-$DEFAULT_HOST_URL}" \ + ts-node \ + --transpileOnly prerender/prerender-server \ + --port="${PRERENDER_PORT}" diff --git a/mise-tasks/services/prerender-mgr b/mise-tasks/services/prerender-mgr new file mode 100755 index 00000000000..3f5a837b6cc --- /dev/null +++ b/mise-tasks/services/prerender-mgr @@ -0,0 +1,12 @@ +#!/bin/sh +#MISE description="Start prerender manager for development" +#MISE depends=["infra:ensure-traefik"] +#MISE dir="packages/realm-server" + +NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + PRERENDER_MANAGER_VERBOSE_LOGS=false \ + ts-node \ + --transpileOnly prerender/manager-server \ + --port=${PRERENDER_MANAGER_PORT:-$PRERENDER_MGR_PORT} \ + --exit-on-signal diff --git a/mise-tasks/services/realm-server b/mise-tasks/services/realm-server new file mode 100755 index 00000000000..1e2456242fb --- /dev/null +++ b/mise-tasks/services/realm-server @@ -0,0 +1,111 @@ +#!/bin/sh +#MISE description="Start realm development server" +#MISE depends=["infra:ensure-traefik", "infra:ensure-pg", "infra:ensure-db"] +#MISE dir="packages/realm-server" + +SCRIPTS_DIR="./scripts" + +sh "$SCRIPTS_DIR/start-icons.sh" & +ICONS_PID=$! +cleanup_icons_server() { + if [ -n "$ICONS_PID" ]; then + kill "$ICONS_PID" >/dev/null 2>&1 || true + fi +} +trap cleanup_icons_server EXIT INT TERM + +pnpm --dir=../skills-realm skills:setup + +if [ -z "${SKIP_BOXEL_HOMEPAGE:-}" ]; then + pnpm --dir=../boxel-homepage-realm boxel-homepage:setup +fi + +if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then + MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly "$SCRIPTS_DIR/matrix-registration-secret.ts") + export MATRIX_REGISTRATION_SHARED_SECRET +fi + +START_EXPERIMENTS=$(if [ -z "$SKIP_EXPERIMENTS" ]; then echo "true"; else echo ""; fi) +START_CATALOG=true +if [ "${SKIP_CATALOG:-}" = "true" ]; then + START_CATALOG="" +fi +START_BOXEL_HOMEPAGE=$(if [ -z "$SKIP_BOXEL_HOMEPAGE" ]; then echo "true"; else echo ""; fi) +START_SUBMISSION=$(if [ -z "$SKIP_SUBMISSION" ]; then echo "true"; else echo ""; fi) + +DEFAULT_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog/" +CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}" +DEFAULT_BOXEL_HOMEPAGE_REALM_URL="${REALM_BASE_URL}/boxel-homepage/" +BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HOMEPAGE_REALM_URL}" +DEFAULT_SUBMISSION_REALM_URL="${REALM_BASE_URL}/submissions/" +SUBMISSION_REALM_URL="${RESOLVED_SUBMISSION_REALM_URL:-$DEFAULT_SUBMISSION_REALM_URL}" + +CATALOG_REALM_PATH="${CATALOG_REALM_PATH:-../catalog-realm}" +SUBMISSION_REALM_PATH="${SUBMISSION_REALM_PATH:-${REALMS_ROOT}/submissions}" + +if [ -n "$USE_EXTERNAL_CATALOG" ]; then + pnpm --dir=../catalog catalog:setup + pnpm --dir=../catalog catalog:update + CATALOG_REALM_PATH='../catalog/contents' +fi + +if [ -n "$START_SUBMISSION" ]; then + sh "$SCRIPTS_DIR/setup-submission-realm.sh" "$SUBMISSION_REALM_PATH" +fi + +if [ -n "$ENV_MODE" ]; then + WORKER_MANAGER_ARG="--workerManagerUrl=${WORKER_MGR_URL}" +else + WORKER_MANAGER_ARG="$1" +fi + +LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ + NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE}" \ + LOG_LEVELS='*=info' \ + REALM_SERVER_SECRET_SEED="mum's the word" \ + REALM_SECRET_SEED="shhh! it's a secret" \ + GRAFANA_SECRET="shhh! it's a secret" \ + MATRIX_URL="${MATRIX_URL_VAL}" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + ENABLE_FILE_WATCHER=true \ + ts-node \ + --transpileOnly main \ + --port="${REALM_PORT}" \ + --matrixURL="${MATRIX_URL_VAL}" \ + --realmsRootPath="${REALMS_ROOT}" \ + --prerendererUrl="${PRERENDER_URL}" \ + --migrateDB \ + $WORKER_MANAGER_ARG \ + \ + --path='../base' \ + --username='base_realm' \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl="${REALM_BASE_URL}/base/" \ + \ + ${START_CATALOG:+--path="${CATALOG_REALM_PATH}"} \ + ${START_CATALOG:+--username='catalog_realm'} \ + ${START_CATALOG:+--fromUrl='@cardstack/catalog/'} \ + ${START_CATALOG:+--toUrl="${CATALOG_REALM_URL}"} \ + \ + --path='../skills-realm/contents' \ + --username='skills_realm' \ + --fromUrl="${REALM_BASE_URL}/skills/" \ + --toUrl="${REALM_BASE_URL}/skills/" \ + \ + ${START_SUBMISSION:+--path="${SUBMISSION_REALM_PATH}"} \ + ${START_SUBMISSION:+--username='submission_realm'} \ + ${START_SUBMISSION:+--fromUrl="${SUBMISSION_REALM_URL}"} \ + ${START_SUBMISSION:+--toUrl="${SUBMISSION_REALM_URL}"} \ + \ + ${START_BOXEL_HOMEPAGE:+--path='../boxel-homepage-realm/contents'} \ + ${START_BOXEL_HOMEPAGE:+--username='boxel_homepage_realm'} \ + ${START_BOXEL_HOMEPAGE:+--fromUrl="${BOXEL_HOMEPAGE_REALM_URL}"} \ + ${START_BOXEL_HOMEPAGE:+--toUrl="${BOXEL_HOMEPAGE_REALM_URL}"} \ + \ + ${START_EXPERIMENTS:+--path='../experiments-realm'} \ + ${START_EXPERIMENTS:+--username='experiments_realm'} \ + ${START_EXPERIMENTS:+--fromUrl="${REALM_BASE_URL}/experiments/"} \ + ${START_EXPERIMENTS:+--toUrl="${REALM_BASE_URL}/experiments/"} diff --git a/mise-tasks/services/realm-server-base b/mise-tasks/services/realm-server-base new file mode 100755 index 00000000000..63541c4459d --- /dev/null +++ b/mise-tasks/services/realm-server-base @@ -0,0 +1,32 @@ +#!/bin/sh +#MISE description="Start base realm server only" +#MISE depends=["infra:ensure-pg"] +#MISE dir="packages/realm-server" + +if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then + MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly ./scripts/matrix-registration-secret.ts) + export MATRIX_REGISTRATION_SHARED_SECRET +fi + +NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + PGPORT="${PGPORT}" \ + PGDATABASE=boxel_base \ + REALM_SERVER_SECRET_SEED="mum's the word" \ + REALM_SECRET_SEED="shhh! it's a secret" \ + GRAFANA_SECRET="shhh! it's a secret" \ + MATRIX_URL="${MATRIX_URL_VAL}" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + ts-node \ + --transpileOnly main \ + --port=4201 \ + --matrixURL="${MATRIX_URL_VAL}" \ + --realmsRootPath='./realms/localhost_4201_base' \ + --prerendererUrl="${PRERENDER_URL}" \ + --migrateDB \ + $1 \ + \ + --path='../base' \ + --username='base_realm' \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl='http://localhost:4201/base/' diff --git a/mise-tasks/services/test-realms b/mise-tasks/services/test-realms new file mode 100755 index 00000000000..90b4a6a55be --- /dev/null +++ b/mise-tasks/services/test-realms @@ -0,0 +1,69 @@ +#!/bin/sh +#MISE description="Start test realm servers" +#MISE depends=["infra:ensure-traefik", "infra:ensure-pg"] +#MISE dir="packages/realm-server" + +SCRIPTS_DIR="./scripts" + +# In environment mode, share the dev icons server; otherwise start our own +if [ -z "$BOXEL_ENVIRONMENT" ]; then + sh "$SCRIPTS_DIR/start-icons.sh" & + ICONS_PID=$! + cleanup_icons_server() { + if [ -n "$ICONS_PID" ]; then + kill "$ICONS_PID" >/dev/null 2>&1 || true + fi + } + trap cleanup_icons_server EXIT INT TERM +fi + +if [ -n "$ENV_MODE" ]; then + WORKER_MANAGER_ARG="--workerManagerUrl=${WORKER_TEST_MGR_URL}" + SERVICE_NAME_ARG="--serviceName=realm-test" + + # Ensure per-environment test database exists + REPO_ROOT="$(cd "../.." && pwd)" + sh "$REPO_ROOT/scripts/ensure-branch-db.sh" "test_${ENV_SLUG}" +else + WORKER_MANAGER_ARG="$1" + SERVICE_NAME_ARG="" +fi + +pnpm --dir=../skills-realm skills:setup + +if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then + MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly "$SCRIPTS_DIR/matrix-registration-secret.ts") + export MATRIX_REGISTRATION_SHARED_SECRET +fi + +NODE_ENV=test \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE_TEST}" \ + NODE_NO_WARNINGS=1 \ + REALM_SERVER_SECRET_SEED="mum's the word" \ + REALM_SECRET_SEED="shhh! it's a secret" \ + GRAFANA_SECRET="shhh! it's a secret" \ + MATRIX_URL="${MATRIX_URL_VAL}" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + ts-node \ + --transpileOnly main \ + --port="${TEST_PORT}" \ + --matrixURL="${MATRIX_URL_VAL}" \ + --realmsRootPath="${REALMS_TEST_ROOT}" \ + --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \ + --migrateDB \ + --prerendererUrl="${PRERENDER_URL}" \ + $WORKER_MANAGER_ARG \ + $SERVICE_NAME_ARG \ + \ + --path='./tests/cards' \ + --username='node-test_realm' \ + --fromUrl="${REALM_TEST_URL}/node-test/" \ + --toUrl="${REALM_TEST_URL}/node-test/" \ + \ + --path='../host/tests/cards' \ + --username='test_realm' \ + --fromUrl="${REALM_TEST_URL}/test/" \ + --toUrl="${REALM_TEST_URL}/test/" \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl="${REALM_BASE_URL}/base/" diff --git a/mise-tasks/services/worker b/mise-tasks/services/worker new file mode 100755 index 00000000000..dd5916bcee1 --- /dev/null +++ b/mise-tasks/services/worker @@ -0,0 +1,39 @@ +#!/bin/sh +#MISE description="Start development worker manager" +#MISE depends=["infra:ensure-traefik", "infra:ensure-pg", "infra:wait-for-prerender"] +#MISE dir="packages/realm-server" + +DEFAULT_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog/" +CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}" + +START_EXPERIMENTS=$(if [ -z "${SKIP_EXPERIMENTS:-}" ]; then echo "true"; else echo ""; fi) +START_CATALOG=$(if [ -z "${SKIP_CATALOG:-}" ]; then echo "true"; else echo ""; fi) + +NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE}" \ + LOG_LEVELS='*=info' \ + REALM_SECRET_SEED="shhh! it's a secret" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + LOW_CREDIT_THRESHOLD=2000 \ + ts-node \ + --transpileOnly worker-manager \ + --allPriorityCount="${WORKER_ALL_PRIORITY_COUNT:-1}" \ + --highPriorityCount="${WORKER_HIGH_PRIORITY_COUNT:-0}" \ + --port="${WORKER_PORT}" \ + --matrixURL="${MATRIX_URL_VAL}" \ + --prerendererUrl="${PRERENDER_MGR_URL}" \ + \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl="${REALM_BASE_URL}/base/" \ + \ + ${START_EXPERIMENTS:+--fromUrl="${REALM_BASE_URL}/experiments/"} \ + ${START_EXPERIMENTS:+--toUrl="${REALM_BASE_URL}/experiments/"} \ + \ + ${START_CATALOG:+--fromUrl='@cardstack/catalog/'} \ + ${START_CATALOG:+--toUrl="${CATALOG_REALM_URL}"} \ + \ + --fromUrl="${REALM_BASE_URL}/skills/" \ + --toUrl="${REALM_BASE_URL}/skills/" diff --git a/mise-tasks/services/worker-base b/mise-tasks/services/worker-base new file mode 100755 index 00000000000..0f6aa184d66 --- /dev/null +++ b/mise-tasks/services/worker-base @@ -0,0 +1,21 @@ +#!/bin/sh +#MISE description="Start worker manager for base realm only" +#MISE depends=["infra:ensure-pg", "infra:wait-for-prerender"] +#MISE dir="packages/realm-server" + +NODE_ENV=development \ + NODE_NO_WARNINGS=1 \ + NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ + PGPORT="${PGPORT}" \ + PGDATABASE=boxel_base \ + REALM_SECRET_SEED="shhh! it's a secret" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + LOW_CREDIT_THRESHOLD=2000 \ + ts-node \ + --transpileOnly worker-manager \ + --port=4213 \ + --matrixURL="${MATRIX_URL_VAL}" \ + --prerendererUrl="${PRERENDER_MGR_URL}" \ + \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl='http://localhost:4201/base/' diff --git a/mise-tasks/services/worker-test b/mise-tasks/services/worker-test new file mode 100755 index 00000000000..a40f8b6f83f --- /dev/null +++ b/mise-tasks/services/worker-test @@ -0,0 +1,42 @@ +#!/bin/sh +#MISE description="Start test worker manager" +#MISE depends=["infra:ensure-traefik", "infra:ensure-pg", "infra:wait-for-prerender"] +#MISE dir="packages/realm-server" + +if [ -n "$ENV_MODE" ]; then + SERVICE_NAME_ARG="--serviceName=worker-test" + MIGRATE_ARG="--migrateDB" + + # Ensure per-environment test database exists + REPO_ROOT="$(cd "../.." && pwd)" + sh "$REPO_ROOT/scripts/ensure-branch-db.sh" "test_${ENV_SLUG}" +else + SERVICE_NAME_ARG="" + MIGRATE_ARG="" +fi + +NODE_ENV=test \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE_TEST}" \ + NODE_NO_WARNINGS=1 \ + NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ + REALM_SECRET_SEED="shhh! it's a secret" \ + REALM_SERVER_MATRIX_USERNAME=realm_server \ + LOW_CREDIT_THRESHOLD=2000 \ + ts-node \ + --transpileOnly worker-manager \ + --port="${WORKER_TEST_PORT}" \ + --matrixURL="${MATRIX_URL_VAL}" \ + --prerendererUrl="${PRERENDER_MGR_URL}" \ + $SERVICE_NAME_ARG \ + $MIGRATE_ARG \ + \ + --fromUrl="${REALM_TEST_URL}/node-test/" \ + --toUrl="${REALM_TEST_URL}/node-test/" \ + \ + --fromUrl="${REALM_TEST_URL}/test/" \ + --toUrl="${REALM_TEST_URL}/test/" \ + --fromUrl='https://cardstack.com/base/' \ + --toUrl="${REALM_BASE_URL}/base/" \ + --fromUrl="${REALM_BASE_URL}/skills/" \ + --toUrl="${REALM_BASE_URL}/skills/" diff --git a/mise-tasks/test-services-host b/mise-tasks/test-services-host new file mode 100755 index 00000000000..abb2d7b8358 --- /dev/null +++ b/mise-tasks/test-services-host @@ -0,0 +1,70 @@ +#!/bin/sh +#MISE description="Start services for host test suite (trimmed catalog)" +#MISE dir="packages/realm-server" + +set -eu + +SCRIPTS_DIR="./scripts" +CATALOG_SRC_PATH="$(cd "../../packages/catalog-realm" && pwd)" +CATALOG_TEMP_PATH="$(mktemp -d "${TMPDIR:-/tmp}/catalog-realm.hosttests.XXXXXX")" + +cleanup() { + rm -rf "$CATALOG_TEMP_PATH" +} +trap cleanup EXIT INT TERM + +# Always include key config files so the realm loads correctly. +for f in .realm.json package.json tsconfig.json .gitignore; do + if [ -e "$CATALOG_SRC_PATH/$f" ]; then + cp -a "$CATALOG_SRC_PATH/$f" "$CATALOG_TEMP_PATH/" + fi +done + +# We need the Spec folder for one of the tests +mkdir -p "$CATALOG_TEMP_PATH/Spec" +cp -a "$CATALOG_SRC_PATH/Spec/grid.json" "$CATALOG_TEMP_PATH/Spec/" + +# Explicitly keep only the tested parts of the catalog +KEEP_FOLDERS="fields catalog-app components commands" +for item in $KEEP_FOLDERS; do + if [ -d "$CATALOG_SRC_PATH/$item" ]; then + cp -a "$CATALOG_SRC_PATH/$item" "$CATALOG_TEMP_PATH/" + else + echo "ERROR: required catalog item not found: $item" >&2 + exit 1 + fi +done +# Explicitly keep some files needed for the tests +KEEP_FILES="cloudflare-image.gts index.json Spec/f869024a-cdec-4a73-afca-d8d32f258ead.json" +for item in $KEEP_FILES; do + if [ -f "$CATALOG_SRC_PATH/$item" ]; then + cp -a "$CATALOG_SRC_PATH/$item" "$CATALOG_TEMP_PATH/$item" + else + echo "ERROR: required catalog item not found: $item" >&2 + exit 1 + fi +done + +export CATALOG_REALM_PATH="$CATALOG_TEMP_PATH" + +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" +BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" +NODE_TEST_REALM_READY="http-get://${REALM_TEST_URL#http://}/node-test/${READY_PATH}" + +HOST_TEST_LOG_LEVELS="${HOST_TEST_LOG_LEVELS:-*=info,realm:requests=warn,realm-index-updater=debug,index-runner=debug,index-perf=debug,index-writer=debug,worker=debug,worker-manager=debug}" +SKIP_CATALOG="${SKIP_CATALOG:-}" + +WAIT_ON_TIMEOUT=900000 \ + SKIP_EXPERIMENTS=true \ + SKIP_CATALOG="$SKIP_CATALOG" \ + SKIP_BOXEL_HOMEPAGE=true \ + SKIP_SUBMISSION=true \ + CATALOG_REALM_PATH="$CATALOG_TEMP_PATH" \ + LOG_LEVELS="$HOST_TEST_LOG_LEVELS" \ + NODE_NO_WARNINGS=1 \ + start-server-and-test \ + 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:matrix start:smtp start:worker-development start:development' \ + "${BASE_REALM_READY}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}" \ + 'run-p -ln start:worker-test start:test-realms' \ + "${NODE_TEST_REALM_READY}" \ + 'wait' diff --git a/mise-tasks/test-services-matrix b/mise-tasks/test-services-matrix new file mode 100755 index 00000000000..8154c99b168 --- /dev/null +++ b/mise-tasks/test-services-matrix @@ -0,0 +1,14 @@ +#!/bin/sh +#MISE description="Start services for matrix test suite (base realm only)" +#MISE dir="packages/realm-server" + +pnpm --dir=../skills-realm skills:setup + +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" +BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" + +WAIT_ON_TIMEOUT=600000 NODE_NO_WARNINGS=1 SKIP_SUBMISSION=true \ + start-server-and-test \ + 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:worker-base start:base' \ + "${BASE_REALM_READY}|${ICONS_URL}" \ + 'wait' From a1fc0b777ddb178a54ad11a1c1e2570b22e3c83f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 10 Mar 2026 16:44:01 -0700 Subject: [PATCH 05/22] Update some package.json scripts This seems like not enough. --- packages/realm-server/package.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json index 6decd7ce976..9ab6fad46df 100644 --- a/packages/realm-server/package.json +++ b/packages/realm-server/package.json @@ -94,7 +94,7 @@ "migrate": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/run-migrations.ts", "start:matrix": "./scripts/start-matrix.sh", "start:smtp": "cd ../matrix && pnpm assert-smtp-running", - "start:icons": "sh ./scripts/start-icons.sh", + "start:icons": "mise run services:icons", "start:pg": "./scripts/start-pg.sh", "stop:pg": "./scripts/stop-pg.sh", "test:wait-for-servers": "WAIT_ON_TIMEOUT=900000 NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' 'test-module'", @@ -104,26 +104,26 @@ "setup:skills-in-deployment": "mkdir -p /persistent/skills", "setup:boxel-homepage-in-deployment": "mkdir -p /persistent/boxel-homepage", "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main", - "start:base": "./scripts/start-base.sh --workerManagerPort=4213", - "start:test-realms": "./scripts/start-test-realms.sh --workerManagerPort=4211", - "start:all": "./scripts/start-all.sh", - "start:skip-optional-realms": "./scripts/start-all-except-optional.sh", - "start:services-for-host-tests": "./scripts/start-services-for-host-tests.sh", + "start:base": "mise run services:realm-server-base -- --workerManagerPort=4213", + "start:test-realms": "mise run services:test-realms -- --workerManagerPort=4211", + "start:all": "mise run dev", + "start:skip-optional-realms": "mise run dev-minimal", + "start:services-for-host-tests": "mise run test-services-host", "start:without-matrix": "./scripts/start-without-matrix.sh", "start:staging": "./scripts/start-staging.sh", - "start:development": "./scripts/start-development.sh --workerManagerPort=4210", + "start:development": "mise run services:realm-server -- --workerManagerPort=4210", "start:production": "./scripts/start-production.sh", - "start:prerender-dev": "./scripts/start-prerender-dev.sh", + "start:prerender-dev": "mise run services:prerender", "start:prerender-staging": "./scripts/start-prerender-staging.sh", "start:prerender-production": "./scripts/start-prerender-production.sh", - "start:prerender-manager-dev": "./scripts/start-prerender-manager-dev.sh", + "start:prerender-manager-dev": "mise run services:prerender-mgr", "start:prerender-manager": "./scripts/start-prerender-manager.sh", - "start:worker-development": "./scripts/start-worker-development.sh", - "start:worker-base": "./scripts/start-worker-base.sh", - "start:worker-test": "./scripts/start-worker-test.sh", + "start:worker-development": "mise run services:worker", + "start:worker-base": "mise run services:worker-base", + "start:worker-test": "mise run services:worker-test", "start:worker-staging": "./scripts/start-worker-staging.sh", "start:worker-production": "./scripts/start-worker-production.sh", - "start:services-for-matrix-tests": "./scripts/start-services-for-matrix-tests.sh", + "start:services-for-matrix-tests": "mise run test-services-matrix", "wait": "sleep 10000000", "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"", From f5a656d5140601650c33b37c47b87ad38ce124a3 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 11 Mar 2026 06:56:51 -0700 Subject: [PATCH 06/22] Fix environment variables when deployed --- mise-tasks/lib/env-vars.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index 20b22c975dd..c226e5b20cb 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -44,31 +44,31 @@ else export ENV_SLUG="" export ENV_MODE="" - # Service URLs (fixed localhost ports) - export REALM_BASE_URL="http://localhost:4201" - export REALM_TEST_URL="http://localhost:4202" - export MATRIX_URL_VAL="http://localhost:8008" - export WORKER_MGR_URL="http://localhost:4210" - export WORKER_TEST_MGR_URL="http://localhost:4211" + # Service URLs — use :- so production/staging env vars are not clobbered + export REALM_BASE_URL="${REALM_BASE_URL:-http://localhost:4201}" + export REALM_TEST_URL="${REALM_TEST_URL:-http://localhost:4202}" + export MATRIX_URL_VAL="${MATRIX_URL_VAL:-http://localhost:8008}" + export WORKER_MGR_URL="${WORKER_MGR_URL:-http://localhost:4210}" + export WORKER_TEST_MGR_URL="${WORKER_TEST_MGR_URL:-http://localhost:4211}" export PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://localhost:4222}" - export ICONS_URL="http://localhost:4206" - export HOST_URL="http://localhost:4200" + export ICONS_URL="${ICONS_URL:-http://localhost:4206}" + export HOST_URL="${HOST_URL:-http://localhost:4200}" # Database export PGDATABASE="${PGDATABASE:-boxel}" - export PGDATABASE_TEST="boxel_test" + export PGDATABASE_TEST="${PGDATABASE_TEST:-boxel_test}" # Ports (fixed in standard mode) - export REALM_PORT=4201 - export TEST_PORT=4202 - export WORKER_PORT=4210 - export WORKER_TEST_PORT=4211 - export PRERENDER_PORT=4221 - export PRERENDER_MGR_PORT=4222 - export ICONS_PORT=4206 + export REALM_PORT="${REALM_PORT:-4201}" + export TEST_PORT="${TEST_PORT:-4202}" + export WORKER_PORT="${WORKER_PORT:-4210}" + export WORKER_TEST_PORT="${WORKER_TEST_PORT:-4211}" + export PRERENDER_PORT="${PRERENDER_PORT:-4221}" + export PRERENDER_MGR_PORT="${PRERENDER_MGR_PORT:-4222}" + export ICONS_PORT="${ICONS_PORT:-4206}" # Paths - export REALMS_ROOT="./realms/localhost_4201" - export REALMS_TEST_ROOT="./realms/localhost_4202" + export REALMS_ROOT="${REALMS_ROOT:-./realms/localhost_4201}" + export REALMS_TEST_ROOT="${REALMS_TEST_ROOT:-./realms/localhost_4202}" fi From dd8a7f56ee1230127909f77ebe8a4049b6559bc3 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 11 Mar 2026 08:13:32 -0700 Subject: [PATCH 07/22] Add empty commit From 3276d58097a82f0f09a1c3bb112dc9c655159b36 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 11 Mar 2026 10:25:26 -0700 Subject: [PATCH 08/22] Fix environment mode ish --- mise-tasks/lib/env-vars.sh | 64 +++++--------------------------------- 1 file changed, 8 insertions(+), 56 deletions(-) diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index c226e5b20cb..f4d944beb48 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -1,6 +1,14 @@ #!/bin/sh # Shared environment computation for mise tasks. # Sourced via .mise.toml [env] _.source — every mise task gets these variables automatically. +# +# NOTE: Service URLs, ports, and paths are NOT exported here yet. The existing +# shell scripts in packages/realm-server/scripts/ compute those themselves. +# Exporting them here would conflict because mise evaluates _.source at shell +# activation time (before BOXEL_ENVIRONMENT is set), and the cached values +# would shadow the scripts' own env-mode computation. +# +# These exports will be added when mise tasks replace the shell scripts. compute_env_slug() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g' @@ -12,63 +20,7 @@ if [ -n "${BOXEL_ENVIRONMENT:-}" ]; then ENV_SLUG=$(compute_env_slug "$BOXEL_ENVIRONMENT") export ENV_SLUG export ENV_MODE=true - - # Service URLs (Traefik hostnames) - export REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" - export REALM_TEST_URL="http://realm-test.${ENV_SLUG}.localhost" - export MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" - export WORKER_MGR_URL="http://worker.${ENV_SLUG}.localhost" - export WORKER_TEST_MGR_URL="http://worker-test.${ENV_SLUG}.localhost" - export PRERENDER_URL="${PRERENDER_URL:-http://prerender.${ENV_SLUG}.localhost}" - export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://prerender-mgr.${ENV_SLUG}.localhost}" - export ICONS_URL="http://icons.${ENV_SLUG}.localhost" - export HOST_URL="http://host.${ENV_SLUG}.localhost" - - # Database - export PGDATABASE="${PGDATABASE:-boxel_${ENV_SLUG}}" - export PGDATABASE_TEST="boxel_test_${ENV_SLUG}" - - # Ports (dynamic in env mode) - export REALM_PORT=0 - export TEST_PORT=0 - export WORKER_PORT=0 - export WORKER_TEST_PORT=0 - export PRERENDER_PORT=0 - export PRERENDER_MGR_PORT=0 - export ICONS_PORT=0 - - # Paths - export REALMS_ROOT="./realms/${ENV_SLUG}" - export REALMS_TEST_ROOT="./realms/${ENV_SLUG}_test" else export ENV_SLUG="" export ENV_MODE="" - - # Service URLs — use :- so production/staging env vars are not clobbered - export REALM_BASE_URL="${REALM_BASE_URL:-http://localhost:4201}" - export REALM_TEST_URL="${REALM_TEST_URL:-http://localhost:4202}" - export MATRIX_URL_VAL="${MATRIX_URL_VAL:-http://localhost:8008}" - export WORKER_MGR_URL="${WORKER_MGR_URL:-http://localhost:4210}" - export WORKER_TEST_MGR_URL="${WORKER_TEST_MGR_URL:-http://localhost:4211}" - export PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" - export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://localhost:4222}" - export ICONS_URL="${ICONS_URL:-http://localhost:4206}" - export HOST_URL="${HOST_URL:-http://localhost:4200}" - - # Database - export PGDATABASE="${PGDATABASE:-boxel}" - export PGDATABASE_TEST="${PGDATABASE_TEST:-boxel_test}" - - # Ports (fixed in standard mode) - export REALM_PORT="${REALM_PORT:-4201}" - export TEST_PORT="${TEST_PORT:-4202}" - export WORKER_PORT="${WORKER_PORT:-4210}" - export WORKER_TEST_PORT="${WORKER_TEST_PORT:-4211}" - export PRERENDER_PORT="${PRERENDER_PORT:-4221}" - export PRERENDER_MGR_PORT="${PRERENDER_MGR_PORT:-4222}" - export ICONS_PORT="${ICONS_PORT:-4206}" - - # Paths - export REALMS_ROOT="${REALMS_ROOT:-./realms/localhost_4201}" - export REALMS_TEST_ROOT="${REALMS_TEST_ROOT:-./realms/localhost_4202}" fi From ec790fd0eac76a657b688f192b2f9d5e2c989642 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 11 Mar 2026 19:27:05 -0700 Subject: [PATCH 09/22] Restore env vars script --- mise-tasks/lib/env-vars.sh | 66 ++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index f4d944beb48..1bb85b05712 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -2,13 +2,9 @@ # Shared environment computation for mise tasks. # Sourced via .mise.toml [env] _.source — every mise task gets these variables automatically. # -# NOTE: Service URLs, ports, and paths are NOT exported here yet. The existing -# shell scripts in packages/realm-server/scripts/ compute those themselves. -# Exporting them here would conflict because mise evaluates _.source at shell -# activation time (before BOXEL_ENVIRONMENT is set), and the cached values -# would shadow the scripts' own env-mode computation. -# -# These exports will be added when mise tasks replace the shell scripts. +# `mise run` re-evaluates _.source with the current environment, so env-mode +# variables are correctly set when BOXEL_ENVIRONMENT is present. The standard-mode +# branch uses ${VAR:-default} to avoid clobbering production/staging env vars in CI. compute_env_slug() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g' @@ -20,7 +16,63 @@ if [ -n "${BOXEL_ENVIRONMENT:-}" ]; then ENV_SLUG=$(compute_env_slug "$BOXEL_ENVIRONMENT") export ENV_SLUG export ENV_MODE=true + + # Service URLs (Traefik hostnames) + export REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" + export REALM_TEST_URL="http://realm-test.${ENV_SLUG}.localhost" + export MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" + export WORKER_MGR_URL="http://worker.${ENV_SLUG}.localhost" + export WORKER_TEST_MGR_URL="http://worker-test.${ENV_SLUG}.localhost" + export PRERENDER_URL="http://prerender.${ENV_SLUG}.localhost" + export PRERENDER_MGR_URL="http://prerender-mgr.${ENV_SLUG}.localhost" + export ICONS_URL="http://icons.${ENV_SLUG}.localhost" + export HOST_URL="http://host.${ENV_SLUG}.localhost" + + # Database + export PGDATABASE="${PGDATABASE:-boxel_${ENV_SLUG}}" + export PGDATABASE_TEST="boxel_test_${ENV_SLUG}" + + # Ports (dynamic in env mode) + export REALM_PORT=0 + export TEST_PORT=0 + export WORKER_PORT=0 + export WORKER_TEST_PORT=0 + export PRERENDER_PORT=0 + export PRERENDER_MGR_PORT=0 + export ICONS_PORT=0 + + # Paths + export REALMS_ROOT="./realms/${ENV_SLUG}" + export REALMS_TEST_ROOT="./realms/${ENV_SLUG}_test" else export ENV_SLUG="" export ENV_MODE="" + + # Service URLs — use :- so production/staging env vars are not clobbered + export REALM_BASE_URL="${REALM_BASE_URL:-http://localhost:4201}" + export REALM_TEST_URL="${REALM_TEST_URL:-http://localhost:4202}" + export MATRIX_URL_VAL="${MATRIX_URL_VAL:-http://localhost:8008}" + export WORKER_MGR_URL="${WORKER_MGR_URL:-http://localhost:4210}" + export WORKER_TEST_MGR_URL="${WORKER_TEST_MGR_URL:-http://localhost:4211}" + export PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" + export PRERENDER_MGR_URL="${PRERENDER_MGR_URL:-http://localhost:4222}" + export ICONS_URL="${ICONS_URL:-http://localhost:4206}" + export HOST_URL="${HOST_URL:-http://localhost:4200}" + + # Database + export PGDATABASE="${PGDATABASE:-boxel}" + export PGDATABASE_TEST="${PGDATABASE_TEST:-boxel_test}" + + # Ports (fixed in standard mode) + export REALM_PORT="${REALM_PORT:-4201}" + export TEST_PORT="${TEST_PORT:-4202}" + export WORKER_PORT="${WORKER_PORT:-4210}" + export WORKER_TEST_PORT="${WORKER_TEST_PORT:-4211}" + export PRERENDER_PORT="${PRERENDER_PORT:-4221}" + export PRERENDER_MGR_PORT="${PRERENDER_MGR_PORT:-4222}" + export ICONS_PORT="${ICONS_PORT:-4206}" + + # Paths + export REALMS_ROOT="${REALMS_ROOT:-./realms/localhost_4201}" + export REALMS_TEST_ROOT="${REALMS_TEST_ROOT:-./realms/localhost_4202}" fi From fabad74201db748b29defc9a9473eba57a97ad9d Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 11 Mar 2026 19:32:08 -0700 Subject: [PATCH 10/22] Fix startup of icons server --- packages/realm-server/scripts/start-icons.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/realm-server/scripts/start-icons.sh b/packages/realm-server/scripts/start-icons.sh index f55347377e7..8b2b58aed47 100644 --- a/packages/realm-server/scripts/start-icons.sh +++ b/packages/realm-server/scripts/start-icons.sh @@ -3,7 +3,7 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then # In environment mode, use port 0 (dynamic) and register with Traefik. # http-server doesn't support port 0, so we pick a free port ourselves. - ICONS_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()' 2>/dev/null || node -e 'const s=require("net").createServer();s.listen(0,()=>{console.log(s.address().port);s.close();})') + ICONS_PORT=$(node -e 'const s=require("net").createServer();s.listen(0,()=>{console.log(s.address().port);s.close();})') echo "Starting icons server on dynamic port ${ICONS_PORT}" cd "$(dirname "$0")/../../boxel-icons" && npx http-server --cors=Origin,X-Requested-With,Content-Type,Accept,Range,Authorization,X-Boxel-Assume-User --port "${ICONS_PORT}" dist & ICONS_PID=$! From 3bf303d26f4b9107cb39790044782a860ed0d05d Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 12 Mar 2026 10:15:33 -0700 Subject: [PATCH 11/22] Remove superseded scripts --- .../realm-server/scripts/ensure-traefik.sh | 14 -- .../scripts/start-all-except-optional.sh | 8 - packages/realm-server/scripts/start-all.sh | 74 --------- packages/realm-server/scripts/start-base.sh | 35 ----- .../realm-server/scripts/start-development.sh | 146 ------------------ .../scripts/start-prerender-dev.sh | 23 --- .../scripts/start-prerender-manager-dev.sh | 23 --- .../scripts/start-services-for-host-tests.sh | 69 --------- .../start-services-for-matrix-tests.sh | 13 -- .../realm-server/scripts/start-test-realms.sh | 86 ----------- .../realm-server/scripts/start-worker-base.sh | 25 --- .../scripts/start-worker-development.sh | 66 -------- .../realm-server/scripts/start-worker-test.sh | 64 -------- packages/realm-server/scripts/wait-for-pg.sh | 22 --- .../scripts/wait-for-prerender.sh | 22 --- 15 files changed, 690 deletions(-) delete mode 100644 packages/realm-server/scripts/ensure-traefik.sh delete mode 100755 packages/realm-server/scripts/start-all-except-optional.sh delete mode 100755 packages/realm-server/scripts/start-all.sh delete mode 100755 packages/realm-server/scripts/start-base.sh delete mode 100755 packages/realm-server/scripts/start-development.sh delete mode 100755 packages/realm-server/scripts/start-prerender-dev.sh delete mode 100755 packages/realm-server/scripts/start-prerender-manager-dev.sh delete mode 100755 packages/realm-server/scripts/start-services-for-host-tests.sh delete mode 100755 packages/realm-server/scripts/start-services-for-matrix-tests.sh delete mode 100755 packages/realm-server/scripts/start-test-realms.sh delete mode 100755 packages/realm-server/scripts/start-worker-base.sh delete mode 100755 packages/realm-server/scripts/start-worker-development.sh delete mode 100755 packages/realm-server/scripts/start-worker-test.sh delete mode 100755 packages/realm-server/scripts/wait-for-pg.sh delete mode 100755 packages/realm-server/scripts/wait-for-prerender.sh diff --git a/packages/realm-server/scripts/ensure-traefik.sh b/packages/realm-server/scripts/ensure-traefik.sh deleted file mode 100644 index cef3224b660..00000000000 --- a/packages/realm-server/scripts/ensure-traefik.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/sh - -# Ensures Traefik is running when in environment mode (BOXEL_ENVIRONMENT is set). -# Sources like wait-for-pg.sh: `. "$SCRIPTS_DIR/ensure-traefik.sh"` -# Call ensure_traefik after sourcing. - -ensure_traefik() { - if [ -z "$BOXEL_ENVIRONMENT" ]; then - return 0 - fi - - REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - sh "$REPO_ROOT/scripts/start-traefik.sh" -} diff --git a/packages/realm-server/scripts/start-all-except-optional.sh b/packages/realm-server/scripts/start-all-except-optional.sh deleted file mode 100755 index e8119162177..00000000000 --- a/packages/realm-server/scripts/start-all-except-optional.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh - -WAIT_ON_TIMEOUT=900000 SKIP_EXPERIMENTS=true SKIP_CATALOG=true SKIP_BOXEL_HOMEPAGE=true SKIP_SUBMISSION=true NODE_NO_WARNINGS=1 start-server-and-test \ - 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:matrix start:smtp start:worker-development start:development' \ - 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001|http://localhost:4206' \ - 'run-p -ln start:worker-test start:test-realms' \ - 'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \ - 'wait' diff --git a/packages/realm-server/scripts/start-all.sh b/packages/realm-server/scripts/start-all.sh deleted file mode 100755 index a0bcfcb05cd..00000000000 --- a/packages/realm-server/scripts/start-all.sh +++ /dev/null @@ -1,74 +0,0 @@ -#! /bin/sh - -# Environment-mode: when BOXEL_ENVIRONMENT is set, use Traefik hostnames for readiness checks -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - REALM_HOST="realm-server.${ENV_SLUG}.localhost" - BASE_REALM="http-get://${REALM_HOST}/base/" - CATALOG_REALM="http-get://${REALM_HOST}/catalog/" - SKILLS_REALM="http-get://${REALM_HOST}/skills/" - BOXEL_HOMEPAGE_REALM="http-get://${REALM_HOST}/boxel-homepage/" - EXPERIMENTS_REALM="http-get://${REALM_HOST}/experiments/" - REALM_TEST_HOST="realm-test.${ENV_SLUG}.localhost" - NODE_TEST_REALM="http-get://${REALM_TEST_HOST}/node-test/" - - ICONS_URL="http://icons.${ENV_SLUG}.localhost" -else - BASE_REALM="http-get://localhost:4201/base/" - CATALOG_REALM="http-get://localhost:4201/catalog/" - SKILLS_REALM="http-get://localhost:4201/skills/" - BOXEL_HOMEPAGE_REALM="http-get://localhost:4201/boxel-homepage/" - EXPERIMENTS_REALM="http-get://localhost:4201/experiments/" - NODE_TEST_REALM="http-get://localhost:4202/node-test/" - ICONS_URL="http://localhost:4206" -fi - -READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" - -BASE_REALM_READY="$BASE_REALM$READY_PATH" -NODE_TEST_REALM_READY="$NODE_TEST_REALM$READY_PATH" - -# Build the wait-on URL list, respecting SKIP_* flags (same as start-development.sh) -WAIT_URLS="$BASE_REALM_READY" -WAIT_URLS="$WAIT_URLS|${SKILLS_REALM}${READY_PATH}" -if [ -z "${SKIP_CATALOG:-}" ]; then - WAIT_URLS="$WAIT_URLS|${CATALOG_REALM}${READY_PATH}" -fi -if [ -z "${SKIP_BOXEL_HOMEPAGE:-}" ]; then - WAIT_URLS="$WAIT_URLS|${BOXEL_HOMEPAGE_REALM}${READY_PATH}" -fi -if [ -z "${SKIP_EXPERIMENTS:-}" ]; then - WAIT_URLS="$WAIT_URLS|${EXPERIMENTS_REALM}${READY_PATH}" -fi - -if [ -n "$BOXEL_ENVIRONMENT" ]; then - SYNAPSE_URL="http://matrix.${ENV_SLUG}.localhost" -else - SYNAPSE_URL="http://localhost:8008" -fi -SMTP_4_DEV_URL="http://localhost:5001" -WAIT_URLS="$WAIT_URLS|$SYNAPSE_URL|$SMTP_4_DEV_URL|$ICONS_URL" - -if [ -n "$BOXEL_ENVIRONMENT" ]; then - # Environment mode: ensure Postgres, database, migrations, Synapse, and user - # registration all complete BEFORE starting the realm server, so users exist - # when it becomes ready. - REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - export PGPORT="${PGPORT:-5435}" - export PGDATABASE="${PGDATABASE:-boxel_${ENV_SLUG}}" - - ./scripts/start-pg.sh - echo "Waiting for Postgres to accept connections..." - until docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1; do sleep 1; done - "$REPO_ROOT/scripts/ensure-branch-db.sh" - echo "Running database migrations..." - pnpm migrate - ./scripts/start-matrix.sh -fi - -WAIT_ON_TIMEOUT=2400000 NODE_NO_WARNINGS=1 start-server-and-test \ - 'run-p -ln start:pg start:matrix start:smtp start:prerender-dev start:prerender-manager-dev start:worker-development start:development' \ - "$WAIT_URLS" \ - 'run-p -ln start:worker-test start:test-realms' \ - "$NODE_TEST_REALM_READY" \ - 'wait' diff --git a/packages/realm-server/scripts/start-base.sh b/packages/realm-server/scripts/start-base.sh deleted file mode 100755 index 034133a85ea..00000000000 --- a/packages/realm-server/scripts/start-base.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" - -wait_for_postgres - -if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then - MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly "$SCRIPTS_DIR/matrix-registration-secret.ts") - export MATRIX_REGISTRATION_SHARED_SECRET -fi - -PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" - -NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - PGPORT=5435 \ - PGDATABASE=boxel_base \ - REALM_SERVER_SECRET_SEED="mum's the word" \ - REALM_SECRET_SEED="shhh! it's a secret" \ - GRAFANA_SECRET="shhh! it's a secret" \ - MATRIX_URL=http://localhost:8008 \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - ts-node \ - --transpileOnly main \ - --port=4201 \ - --matrixURL='http://localhost:8008' \ - --realmsRootPath='./realms/localhost_4201_base' \ - --prerendererUrl="${PRERENDER_URL}" \ - --migrateDB \ - $1 \ - \ - --path='../base' \ - --username='base_realm' \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl='http://localhost:4201/base/' diff --git a/packages/realm-server/scripts/start-development.sh b/packages/realm-server/scripts/start-development.sh deleted file mode 100755 index 90904ca21e7..00000000000 --- a/packages/realm-server/scripts/start-development.sh +++ /dev/null @@ -1,146 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -ensure_traefik - -sh "$SCRIPTS_DIR/start-icons.sh" & -ICONS_PID=$! -cleanup_icons_server() { - if [ -n "$ICONS_PID" ]; then - kill "$ICONS_PID" >/dev/null 2>&1 || true - fi -} -trap cleanup_icons_server EXIT INT TERM - -wait_for_postgres - -pnpm --dir=../skills-realm skills:setup - -if [ -z "${SKIP_BOXEL_HOMEPAGE:-}" ]; then - pnpm --dir=../boxel-homepage-realm boxel-homepage:setup -fi - -if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then - MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly "$SCRIPTS_DIR/matrix-registration-secret.ts") - export MATRIX_REGISTRATION_SHARED_SECRET -fi - -START_EXPERIMENTS=$(if [ -z "$SKIP_EXPERIMENTS" ]; then echo "true"; else echo ""; fi) -START_CATALOG=true -if [ "${SKIP_CATALOG:-}" = "true" ]; then - START_CATALOG="" -fi -START_BOXEL_HOMEPAGE=$(if [ -z "$SKIP_BOXEL_HOMEPAGE" ]; then echo "true"; else echo ""; fi) -START_SUBMISSION=$(if [ -z "$SKIP_SUBMISSION" ]; then echo "true"; else echo ""; fi) - -# Environment-mode configuration: when BOXEL_ENVIRONMENT is set, use dynamic ports and -# Traefik routing instead of hardcoded ports. -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" - REALM_PORT=0 - REALMS_ROOT="./realms/${ENV_SLUG}" - PGDATABASE_VAL="boxel_${ENV_SLUG}" - MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" - # Ensure per-environment database exists - sh "$SCRIPTS_DIR/../../../scripts/ensure-branch-db.sh" "$ENV_SLUG" -else - REALM_BASE_URL="http://localhost:4201" - REALM_PORT=4201 - REALMS_ROOT="./realms/localhost_4201" - PGDATABASE_VAL="boxel" - MATRIX_URL_VAL="http://localhost:8008" -fi - -DEFAULT_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog/" -CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}" -DEFAULT_NEW_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog-new/" -NEW_CATALOG_REALM_URL="${RESOLVED_NEW_CATALOG_REALM_URL:-$DEFAULT_NEW_CATALOG_REALM_URL}" -DEFAULT_BOXEL_HOMEPAGE_REALM_URL="${REALM_BASE_URL}/boxel-homepage/" -BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HOMEPAGE_REALM_URL}" -DEFAULT_SUBMISSION_REALM_URL="${REALM_BASE_URL}/submissions/" -SUBMISSION_REALM_URL="${RESOLVED_SUBMISSION_REALM_URL:-$DEFAULT_SUBMISSION_REALM_URL}" - -# This can be overridden from the environment to point to a different catalog -# and is used in start-services-for-host-tests.sh to point to a trimmed down -# version of the catalog-realm for faster startup. -CATALOG_REALM_PATH="${CATALOG_REALM_PATH:-../catalog-realm}" -SUBMISSION_REALM_PATH="${SUBMISSION_REALM_PATH:-${REALMS_ROOT}/submissions}" - -if [ -n "$USE_EXTERNAL_CATALOG" ]; then - pnpm --dir=../catalog catalog:setup - pnpm --dir=../catalog catalog:update - CATALOG_REALM_PATH='../catalog/contents' -fi - -if [ -n "$START_SUBMISSION" ]; then - sh "$SCRIPTS_DIR/setup-submission-realm.sh" "$SUBMISSION_REALM_PATH" -fi - - -# In environment mode, override prerender URL and worker manager arg to use Traefik hostnames -if [ -n "$BOXEL_ENVIRONMENT" ]; then - PRERENDER_URL="${PRERENDER_URL:-http://prerender.${ENV_SLUG}.localhost}" - WORKER_MANAGER_ARG="--workerManagerUrl=http://worker.${ENV_SLUG}.localhost" -else - PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" - WORKER_MANAGER_ARG="$1" -fi - -LOW_CREDIT_THRESHOLD="${LOW_CREDIT_THRESHOLD:-2000}" \ - NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - PGPORT=5435 \ - PGDATABASE="${PGDATABASE_VAL}" \ - LOG_LEVELS='*=info' \ - REALM_SERVER_SECRET_SEED="mum's the word" \ - REALM_SECRET_SEED="shhh! it's a secret" \ - GRAFANA_SECRET="shhh! it's a secret" \ - MATRIX_URL="${MATRIX_URL_VAL}" \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - ENABLE_FILE_WATCHER=true \ - ts-node \ - --transpileOnly main \ - --port="${REALM_PORT}" \ - --matrixURL="${MATRIX_URL_VAL}" \ - --realmsRootPath="${REALMS_ROOT}" \ - --prerendererUrl="${PRERENDER_URL}" \ - --migrateDB \ - $WORKER_MANAGER_ARG \ - \ - --path='../base' \ - --username='base_realm' \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl="${REALM_BASE_URL}/base/" \ - \ - ${START_CATALOG:+--path="${CATALOG_REALM_PATH}"} \ - ${START_CATALOG:+--username='catalog_realm'} \ - ${START_CATALOG:+--fromUrl='@cardstack/catalog/'} \ - ${START_CATALOG:+--toUrl="${CATALOG_REALM_URL}"} \ - \ - --path='../skills-realm/contents' \ - --username='skills_realm' \ - --fromUrl="${REALM_BASE_URL}/skills/" \ - --toUrl="${REALM_BASE_URL}/skills/" \ - \ - ${START_SUBMISSION:+--path="${SUBMISSION_REALM_PATH}"} \ - ${START_SUBMISSION:+--username='submission_realm'} \ - ${START_SUBMISSION:+--fromUrl="${SUBMISSION_REALM_URL}"} \ - ${START_SUBMISSION:+--toUrl="${SUBMISSION_REALM_URL}"} \ - \ - ${START_BOXEL_HOMEPAGE:+--path='../boxel-homepage-realm/contents'} \ - ${START_BOXEL_HOMEPAGE:+--username='boxel_homepage_realm'} \ - ${START_BOXEL_HOMEPAGE:+--fromUrl="${BOXEL_HOMEPAGE_REALM_URL}"} \ - ${START_BOXEL_HOMEPAGE:+--toUrl="${BOXEL_HOMEPAGE_REALM_URL}"} \ - \ - ${START_EXPERIMENTS:+--path='../experiments-realm'} \ - ${START_EXPERIMENTS:+--username='experiments_realm'} \ - ${START_EXPERIMENTS:+--fromUrl="${REALM_BASE_URL}/experiments/"} \ - ${START_EXPERIMENTS:+--toUrl="${REALM_BASE_URL}/experiments/"} \ - \ - ${START_CATALOG:+--path='../catalog-new/contents'} \ - ${START_CATALOG:+--username='catalog_new_realm'} \ - ${START_CATALOG:+--fromUrl="${NEW_CATALOG_REALM_URL}"} \ - ${START_CATALOG:+--toUrl="${NEW_CATALOG_REALM_URL}"} diff --git a/packages/realm-server/scripts/start-prerender-dev.sh b/packages/realm-server/scripts/start-prerender-dev.sh deleted file mode 100755 index 7eb95e10e65..00000000000 --- a/packages/realm-server/scripts/start-prerender-dev.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -ensure_traefik - -# Environment-mode configuration -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - PRERENDER_PORT=0 - DEFAULT_HOST_URL="http://host.${ENV_SLUG}.localhost" -else - PRERENDER_PORT=4221 - DEFAULT_HOST_URL="http://localhost:4200" -fi - -# Environment for development prerender server -NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - BOXEL_HOST_URL="${HOST_URL:-$DEFAULT_HOST_URL}" \ - ts-node \ - --transpileOnly prerender/prerender-server \ - --port="${PRERENDER_PORT}" diff --git a/packages/realm-server/scripts/start-prerender-manager-dev.sh b/packages/realm-server/scripts/start-prerender-manager-dev.sh deleted file mode 100755 index 8a2e3f24ea1..00000000000 --- a/packages/realm-server/scripts/start-prerender-manager-dev.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -ensure_traefik - -# Start the prerender manager in development -# Ports default to 4222 unless PRERENDER_MANAGER_PORT is provided - -# Environment-mode configuration -if [ -n "$BOXEL_ENVIRONMENT" ]; then - DEFAULT_PRERENDER_MGR_PORT=0 -else - DEFAULT_PRERENDER_MGR_PORT=4222 -fi - -NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - PRERENDER_MANAGER_VERBOSE_LOGS=false \ - ts-node \ - --transpileOnly prerender/manager-server \ - --port=${PRERENDER_MANAGER_PORT:-$DEFAULT_PRERENDER_MGR_PORT} \ - --exit-on-signal diff --git a/packages/realm-server/scripts/start-services-for-host-tests.sh b/packages/realm-server/scripts/start-services-for-host-tests.sh deleted file mode 100755 index 6b734859b7a..00000000000 --- a/packages/realm-server/scripts/start-services-for-host-tests.sh +++ /dev/null @@ -1,69 +0,0 @@ -#! /bin/sh - -# Use POSIX flags; pipefail is not portable in /bin/sh -set -eu - -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -CATALOG_SRC_PATH="$(cd "$SCRIPTS_DIR/../../catalog-realm" && pwd)" -CATALOG_TEMP_PATH="$(mktemp -d "${TMPDIR:-/tmp}/catalog-realm.hosttests.XXXXXX")" - -cleanup() { - rm -rf "$CATALOG_TEMP_PATH" -} -trap cleanup EXIT INT TERM - -# Always include key config files so the realm loads correctly. -for f in .realm.json package.json tsconfig.json .gitignore; do - if [ -e "$CATALOG_SRC_PATH/$f" ]; then - cp -a "$CATALOG_SRC_PATH/$f" "$CATALOG_TEMP_PATH/" - fi -done - -# We need the Spec folder for one of the tests -mkdir -p "$CATALOG_TEMP_PATH/Spec" -cp -a "$CATALOG_SRC_PATH/Spec/grid.json" "$CATALOG_TEMP_PATH/Spec/" - -# Explicitly keep only the tested parts of the catalog -KEEP_FOLDERS="fields catalog-app components commands" -for item in $KEEP_FOLDERS; do - if [ -d "$CATALOG_SRC_PATH/$item" ]; then - cp -a "$CATALOG_SRC_PATH/$item" "$CATALOG_TEMP_PATH/" - else - echo "ERROR: required catalog item not found: $item" >&2 - exit 1 - fi -done -# Explicitly keep some files needed for the tests -KEEP_FILES="cloudflare-image.gts index.json Spec/f869024a-cdec-4a73-afca-d8d32f258ead.json" -for item in $KEEP_FILES; do - if [ -f "$CATALOG_SRC_PATH/$item" ]; then - cp -a "$CATALOG_SRC_PATH/$item" "$CATALOG_TEMP_PATH/$item" - else - echo "ERROR: required catalog item not found: $item" >&2 - exit 1 - fi -done - -export CATALOG_REALM_PATH="$CATALOG_TEMP_PATH" - -# Make host-test startup logs focus on indexing progress rather than per-request noise. -HOST_TEST_LOG_LEVELS="${HOST_TEST_LOG_LEVELS:-*=info,realm:requests=warn,realm-index-updater=debug,index-runner=debug,index-perf=debug,index-writer=debug,worker=debug,worker-manager=debug}" -SKIP_CATALOG="${SKIP_CATALOG:-}" -# There is a race condition starting up the servers that setting up the -# submission realm triggers which triggers the start-development.sh script to -# SIGTERM. currently we don't need the submission realm for host tests to -# skipping that. but this issue needs to be fixed. -WAIT_ON_TIMEOUT=900000 \ - SKIP_EXPERIMENTS=true \ - SKIP_CATALOG="$SKIP_CATALOG" \ - SKIP_BOXEL_HOMEPAGE=true \ - SKIP_SUBMISSION=true \ - CATALOG_REALM_PATH="$CATALOG_TEMP_PATH" \ - LOG_LEVELS="$HOST_TEST_LOG_LEVELS" \ - NODE_NO_WARNINGS=1 \ - start-server-and-test \ - 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:matrix start:smtp start:worker-development start:development' \ - 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001|http://localhost:4206' \ - 'run-p -ln start:worker-test start:test-realms' \ - 'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \ - 'wait' diff --git a/packages/realm-server/scripts/start-services-for-matrix-tests.sh b/packages/realm-server/scripts/start-services-for-matrix-tests.sh deleted file mode 100755 index 9989c64245a..00000000000 --- a/packages/realm-server/scripts/start-services-for-matrix-tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /bin/sh -pnpm --dir=../skills-realm skills:setup -# There is a race condition starting up the servers that setting up the -# submission realm triggers which triggers the start-development.sh script to -# SIGTERM. currently we don't need the submission realm for host tests to -# skipping that. but this issue needs to be fixed. -# ALSO i really think if we want a submission realm in our matrix tests then -# we probably need to add that via the isolated realm server--not here... -WAIT_ON_TIMEOUT=600000 NODE_NO_WARNINGS=1 SKIP_SUBMISSION=true \ - start-server-and-test \ - 'run-p -ln start:pg start:icons start:prerender-dev start:prerender-manager-dev start:worker-base start:base' \ - 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:4206' \ - 'wait' diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh deleted file mode 100755 index 7f3e0806dce..00000000000 --- a/packages/realm-server/scripts/start-test-realms.sh +++ /dev/null @@ -1,86 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -# In environment mode, share the dev icons server; otherwise start our own -if [ -z "$BOXEL_ENVIRONMENT" ]; then - sh "$SCRIPTS_DIR/start-icons.sh" & - ICONS_PID=$! - cleanup_icons_server() { - if [ -n "$ICONS_PID" ]; then - kill "$ICONS_PID" >/dev/null 2>&1 || true - fi - } - trap cleanup_icons_server EXIT INT TERM -else - ensure_traefik -fi - -wait_for_postgres - -# Environment-mode configuration -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - REALM_TEST_URL="http://realm-test.${ENV_SLUG}.localhost" - REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" - TEST_PORT=0 - PGDATABASE_VAL="boxel_test_${ENV_SLUG}" - REALMS_ROOT="./realms/${ENV_SLUG}_test" - PRERENDER_URL="${PRERENDER_URL:-http://prerender.${ENV_SLUG}.localhost}" - WORKER_MANAGER_ARG="--workerManagerUrl=http://worker-test.${ENV_SLUG}.localhost" - SERVICE_NAME_ARG="--serviceName=realm-test" - MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" - - # Ensure per-environment test database exists - sh "$SCRIPTS_DIR/../../../scripts/ensure-branch-db.sh" "test_${ENV_SLUG}" -else - REALM_TEST_URL="http://localhost:4202" - REALM_BASE_URL="http://localhost:4201" - TEST_PORT=4202 - PGDATABASE_VAL="boxel_test" - REALMS_ROOT="./realms/localhost_4202" - PRERENDER_URL="${PRERENDER_URL:-http://localhost:4221}" - WORKER_MANAGER_ARG="$1" - SERVICE_NAME_ARG="" - MATRIX_URL_VAL="http://localhost:8008" -fi - -pnpm --dir=../skills-realm skills:setup - -if [ -z "$MATRIX_REGISTRATION_SHARED_SECRET" ]; then - MATRIX_REGISTRATION_SHARED_SECRET=$(ts-node --transpileOnly "$SCRIPTS_DIR/matrix-registration-secret.ts") - export MATRIX_REGISTRATION_SHARED_SECRET -fi - -NODE_ENV=test \ - PGPORT=5435 \ - PGDATABASE="${PGDATABASE_VAL}" \ - NODE_NO_WARNINGS=1 \ - REALM_SERVER_SECRET_SEED="mum's the word" \ - REALM_SECRET_SEED="shhh! it's a secret" \ - GRAFANA_SECRET="shhh! it's a secret" \ - MATRIX_URL="${MATRIX_URL_VAL}" \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - ts-node \ - --transpileOnly main \ - --port="${TEST_PORT}" \ - --matrixURL="${MATRIX_URL_VAL}" \ - --realmsRootPath="${REALMS_ROOT}" \ - --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \ - --migrateDB \ - --prerendererUrl="${PRERENDER_URL}" \ - $WORKER_MANAGER_ARG \ - $SERVICE_NAME_ARG \ - \ - --path='./tests/cards' \ - --username='node-test_realm' \ - --fromUrl="${REALM_TEST_URL}/node-test/" \ - --toUrl="${REALM_TEST_URL}/node-test/" \ - \ - --path='../host/tests/cards' \ - --username='test_realm' \ - --fromUrl="${REALM_TEST_URL}/test/" \ - --toUrl="${REALM_TEST_URL}/test/" \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl="${REALM_BASE_URL}/base/" diff --git a/packages/realm-server/scripts/start-worker-base.sh b/packages/realm-server/scripts/start-worker-base.sh deleted file mode 100755 index abe5b3a27ef..00000000000 --- a/packages/realm-server/scripts/start-worker-base.sh +++ /dev/null @@ -1,25 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" -. "$SCRIPTS_DIR/wait-for-prerender.sh" - -wait_for_postgres -PRERENDER_URL="${PRERENDER_URL:-http://localhost:4222}" -wait_for_prerender "$PRERENDER_URL" - -NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ - PGPORT=5435 \ - PGDATABASE=boxel_base \ - REALM_SECRET_SEED="shhh! it's a secret" \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - LOW_CREDIT_THRESHOLD=2000 \ - ts-node \ - --transpileOnly worker-manager \ - --port=4213 \ - --matrixURL='http://localhost:8008' \ - --prerendererUrl="${PRERENDER_URL}" \ - \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl='http://localhost:4201/base/' diff --git a/packages/realm-server/scripts/start-worker-development.sh b/packages/realm-server/scripts/start-worker-development.sh deleted file mode 100755 index d7be73fe22f..00000000000 --- a/packages/realm-server/scripts/start-worker-development.sh +++ /dev/null @@ -1,66 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" -. "$SCRIPTS_DIR/wait-for-prerender.sh" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -ensure_traefik -wait_for_postgres - -# Environment-mode configuration -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" - WORKER_PORT=0 - PGDATABASE_VAL="boxel_${ENV_SLUG}" - PRERENDER_URL="${PRERENDER_URL:-http://prerender-mgr.${ENV_SLUG}.localhost}" - MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" -else - REALM_BASE_URL="http://localhost:4201" - WORKER_PORT=4210 - PGDATABASE_VAL="boxel" - PRERENDER_URL="${PRERENDER_URL:-http://localhost:4222}" - MATRIX_URL_VAL="http://localhost:8008" -fi - -wait_for_prerender "$PRERENDER_URL" - -DEFAULT_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog/" -CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}" -DEFAULT_NEW_CATALOG_REALM_URL="${REALM_BASE_URL}/catalog-new/" -NEW_CATALOG_REALM_URL="${RESOLVED_NEW_CATALOG_REALM_URL:-$DEFAULT_NEW_CATALOG_REALM_URL}" - -START_EXPERIMENTS=$(if [ -z "${SKIP_EXPERIMENTS:-}" ]; then echo "true"; else echo ""; fi) -START_CATALOG=$(if [ -z "${SKIP_CATALOG:-}" ]; then echo "true"; else echo ""; fi) - -NODE_ENV=development \ - NODE_NO_WARNINGS=1 \ - NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ - PGPORT=5435 \ - PGDATABASE="${PGDATABASE_VAL}" \ - LOG_LEVELS='*=info' \ - REALM_SECRET_SEED="shhh! it's a secret" \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - LOW_CREDIT_THRESHOLD=2000 \ - ts-node \ - --transpileOnly worker-manager \ - --allPriorityCount="${WORKER_ALL_PRIORITY_COUNT:-1}" \ - --highPriorityCount="${WORKER_HIGH_PRIORITY_COUNT:-0}" \ - --port="${WORKER_PORT}" \ - --matrixURL="${MATRIX_URL_VAL}" \ - --prerendererUrl="${PRERENDER_URL}" \ - \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl="${REALM_BASE_URL}/base/" \ - \ - ${START_EXPERIMENTS:+--fromUrl="${REALM_BASE_URL}/experiments/"} \ - ${START_EXPERIMENTS:+--toUrl="${REALM_BASE_URL}/experiments/"} \ - \ - ${START_CATALOG:+--fromUrl='@cardstack/catalog/'} \ - ${START_CATALOG:+--toUrl="${CATALOG_REALM_URL}"} \ - \ - --fromUrl="${REALM_BASE_URL}/skills/" \ - --toUrl="${REALM_BASE_URL}/skills/" \ - \ - ${START_CATALOG:+--fromUrl="${NEW_CATALOG_REALM_URL}"} \ - ${START_CATALOG:+--toUrl="${NEW_CATALOG_REALM_URL}"} diff --git a/packages/realm-server/scripts/start-worker-test.sh b/packages/realm-server/scripts/start-worker-test.sh deleted file mode 100755 index 2bbdf8abf13..00000000000 --- a/packages/realm-server/scripts/start-worker-test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#! /bin/sh -SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" -. "$SCRIPTS_DIR/wait-for-pg.sh" -. "$SCRIPTS_DIR/wait-for-prerender.sh" -. "$SCRIPTS_DIR/ensure-traefik.sh" - -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ensure_traefik -fi - -wait_for_postgres - -# Environment-mode configuration -if [ -n "$BOXEL_ENVIRONMENT" ]; then - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') - REALM_TEST_URL="http://realm-test.${ENV_SLUG}.localhost" - REALM_BASE_URL="http://realm-server.${ENV_SLUG}.localhost" - WORKER_PORT=0 - PGDATABASE_VAL="boxel_test_${ENV_SLUG}" - PRERENDER_URL="${PRERENDER_URL:-http://prerender-mgr.${ENV_SLUG}.localhost}" - SERVICE_NAME_ARG="--serviceName=worker-test" - MIGRATE_ARG="--migrateDB" - MATRIX_URL_VAL="http://matrix.${ENV_SLUG}.localhost" - - # Ensure per-environment test database exists - sh "$SCRIPTS_DIR/../../../scripts/ensure-branch-db.sh" "test_${ENV_SLUG}" -else - REALM_TEST_URL="http://localhost:4202" - REALM_BASE_URL="http://localhost:4201" - WORKER_PORT=4211 - PGDATABASE_VAL="boxel_test" - PRERENDER_URL="${PRERENDER_URL:-http://localhost:4222}" - SERVICE_NAME_ARG="" - MIGRATE_ARG="" - MATRIX_URL_VAL="http://localhost:8008" -fi - -wait_for_prerender "$PRERENDER_URL" - -NODE_ENV=test \ - PGPORT=5435 \ - PGDATABASE="${PGDATABASE_VAL}" \ - NODE_NO_WARNINGS=1 \ - NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \ - REALM_SECRET_SEED="shhh! it's a secret" \ - REALM_SERVER_MATRIX_USERNAME=realm_server \ - LOW_CREDIT_THRESHOLD=2000 \ - ts-node \ - --transpileOnly worker-manager \ - --port="${WORKER_PORT}" \ - --matrixURL="${MATRIX_URL_VAL}" \ - --prerendererUrl="${PRERENDER_URL}" \ - $SERVICE_NAME_ARG \ - $MIGRATE_ARG \ - \ - --fromUrl="${REALM_TEST_URL}/node-test/" \ - --toUrl="${REALM_TEST_URL}/node-test/" \ - \ - --fromUrl="${REALM_TEST_URL}/test/" \ - --toUrl="${REALM_TEST_URL}/test/" \ - --fromUrl='https://cardstack.com/base/' \ - --toUrl="${REALM_BASE_URL}/base/" \ - --fromUrl="${REALM_BASE_URL}/skills/" \ - --toUrl="${REALM_BASE_URL}/skills/" diff --git a/packages/realm-server/scripts/wait-for-pg.sh b/packages/realm-server/scripts/wait-for-pg.sh deleted file mode 100755 index 234d10e5684..00000000000 --- a/packages/realm-server/scripts/wait-for-pg.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/sh - -wait_for_postgres() { - COUNT=0 - MAX_ATTEMPTS=10 - - check_postgres_ready() { - docker exec boxel-pg pg_isready -U postgres >/dev/null 2>&1 - } - while ! check_postgres_ready; do - if [ $COUNT -eq 0 ]; then - echo "Waiting for postgres" - fi - if [ $COUNT -eq $MAX_ATTEMPTS ]; then - echo "Failed to detect postgres after $MAX_ATTEMPTS attempts." - exit 1 - fi - COUNT=$((COUNT + 1)) - printf '.' - sleep 5 - done -} diff --git a/packages/realm-server/scripts/wait-for-prerender.sh b/packages/realm-server/scripts/wait-for-prerender.sh deleted file mode 100755 index 5137a6da05a..00000000000 --- a/packages/realm-server/scripts/wait-for-prerender.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/sh - -wait_for_prerender() { - local url="${1:-${PRERENDER_HEALTH_URL:-http://localhost:4221/}}" - local trimmed_url="${url%/}/" - TIMEOUT_SECONDS=1500 - START_TIME=$(date +%s) - - echo "Waiting for prerender server at ${trimmed_url}" - - while ! curl -sf "$trimmed_url" >/dev/null 2>&1; do - CURRENT_TIME=$(date +%s) - ELAPSED=$((CURRENT_TIME - START_TIME)) - if [ "$ELAPSED" -ge "$TIMEOUT_SECONDS" ]; then - echo "Timed out waiting for prerender server after ${TIMEOUT_SECONDS}s" - exit 1 - fi - sleep 1 - done - - echo "Prerender server is ready" -} From 5bc7244841032353f8cd80b694c73a1f6a1b9f09 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 12 Mar 2026 10:18:23 -0700 Subject: [PATCH 12/22] Fix environment mode icons server --- packages/host/config/environment.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/host/config/environment.js b/packages/host/config/environment.js index 21a6a01fa4c..271338e1cb0 100644 --- a/packages/host/config/environment.js +++ b/packages/host/config/environment.js @@ -87,7 +87,11 @@ module.exports = function (environment) { fileSizeLimitBytes: Number( process.env.FILE_SIZE_LIMIT_BYTES ?? DEFAULT_FILE_SIZE_LIMIT_BYTES, ), - iconsURL: process.env.ICONS_URL || defaults.iconsURL, + // In environment mode, use computed Traefik hostname (not env var, which + // may be stale from mise's shell-activation cache in standard mode). + iconsURL: process.env.BOXEL_ENVIRONMENT + ? defaults.iconsURL + : (process.env.ICONS_URL || defaults.iconsURL), publishedRealmBoxelSpaceDomain: process.env.PUBLISHED_REALM_BOXEL_SPACE_DOMAIN || defaults.realmHost, publishedRealmBoxelSiteDomain: From e43a9aa4043882166737c910ca943a560a392bc8 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Fri, 13 Mar 2026 08:29:34 -0700 Subject: [PATCH 13/22] Fix all script --- mise-tasks/dev-all | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/mise-tasks/dev-all b/mise-tasks/dev-all index 80ce28c93d5..3450cdac878 100755 --- a/mise-tasks/dev-all +++ b/mise-tasks/dev-all @@ -2,12 +2,13 @@ #MISE description="Start host app + full dev stack (host must be ready before realm server)" #MISE dir="packages/realm-server" -READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" +# Add node_modules/.bin to PATH (mise run bypasses pnpm which normally does this) +# MISE dir is packages/realm-server, so ../.. is repo root +PATH="$(pwd)/../../node_modules/.bin:$(pwd)/node_modules/.bin:$PATH" -# Host readiness URL -HOST_READY="${HOST_URL}" +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" -# Phase 2 readiness URLs (realm server + services) +# Phase 1 readiness URLs (realm server + services) BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" SKILLS_READY="http-get://${REALM_BASE_URL#http://}/skills/${READY_PATH}" REALM_URLS="${BASE_REALM_READY}|${SKILLS_READY}" @@ -25,7 +26,7 @@ REALM_URLS="${REALM_URLS}|http-get://${REALM_BASE_URL#http://}/software-factory/ REALM_URLS="${REALM_URLS}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}" -# Phase 3 readiness URL (test realms) +# Phase 2 readiness URL (test realms) NODE_TEST_REALM_READY="http-get://${REALM_TEST_URL#http://}/node-test/${READY_PATH}" # In environment mode, bootstrap infra before starting services @@ -40,9 +41,20 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then ./scripts/start-matrix.sh fi +# Start host app in background and wait for it before launching the rest. +# start-server-and-test v1.x only supports 2 server phases (5 args), so we +# start the host manually and use wait-on to gate readiness. +pnpm --filter @cardstack/host start & +HOST_PID=$! +cleanup_host() { + kill "$HOST_PID" >/dev/null 2>&1 || true +} +trap cleanup_host EXIT INT TERM + +echo "Waiting for host app at ${HOST_URL}..." +wait-on "$HOST_URL" --timeout 120000 + WAIT_ON_TIMEOUT=2400000 NODE_NO_WARNINGS=1 start-server-and-test \ - 'pnpm --filter @cardstack/host start' \ - "$HOST_READY" \ 'run-p -ln start:pg start:matrix start:smtp start:prerender-dev start:prerender-manager-dev start:worker-development start:development' \ "$REALM_URLS" \ 'run-p -ln start:worker-test start:test-realms' \ From 980a16f3dd176ffa7c73f6f22ea451c85ab64c99 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 06:37:02 -0700 Subject: [PATCH 14/22] Fix waiting for host --- mise-tasks/dev-all | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mise-tasks/dev-all b/mise-tasks/dev-all index 3450cdac878..a591c4ceac5 100755 --- a/mise-tasks/dev-all +++ b/mise-tasks/dev-all @@ -52,7 +52,8 @@ cleanup_host() { trap cleanup_host EXIT INT TERM echo "Waiting for host app at ${HOST_URL}..." -wait-on "$HOST_URL" --timeout 120000 +until [ "$(curl -s -o /dev/null -w '%{http_code}' "$HOST_URL" 2>/dev/null)" = "200" ]; do sleep 2; done +echo "Host app is ready." WAIT_ON_TIMEOUT=2400000 NODE_NO_WARNINGS=1 start-server-and-test \ 'run-p -ln start:pg start:matrix start:smtp start:prerender-dev start:prerender-manager-dev start:worker-development start:development' \ From 18fc71162f9d822b72c72ed6c8a9bea6b4a930f8 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 09:51:00 -0700 Subject: [PATCH 15/22] Add icons to Matrix test startup --- mise-tasks/test-services-matrix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise-tasks/test-services-matrix b/mise-tasks/test-services-matrix index 8154c99b168..3f8945eaa97 100755 --- a/mise-tasks/test-services-matrix +++ b/mise-tasks/test-services-matrix @@ -9,6 +9,6 @@ BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" WAIT_ON_TIMEOUT=600000 NODE_NO_WARNINGS=1 SKIP_SUBMISSION=true \ start-server-and-test \ - 'run-p -ln start:pg start:prerender-dev start:prerender-manager-dev start:worker-base start:base' \ + 'run-p -ln start:pg start:icons start:prerender-dev start:prerender-manager-dev start:worker-base start:base' \ "${BASE_REALM_READY}|${ICONS_URL}" \ 'wait' From 64ff1cb44896b559198dbeaaa3970d4beb30df60 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 10:20:19 -0700 Subject: [PATCH 16/22] Fix lint error --- packages/host/config/environment.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/host/config/environment.js b/packages/host/config/environment.js index 79dba638157..f34edd6327c 100644 --- a/packages/host/config/environment.js +++ b/packages/host/config/environment.js @@ -93,7 +93,7 @@ module.exports = function (environment) { // may be stale from mise's shell-activation cache in standard mode). iconsURL: process.env.BOXEL_ENVIRONMENT ? defaults.iconsURL - : (process.env.ICONS_URL || defaults.iconsURL), + : process.env.ICONS_URL || defaults.iconsURL, publishedRealmBoxelSpaceDomain: process.env.PUBLISHED_REALM_BOXEL_SPACE_DOMAIN || defaults.realmHost, publishedRealmBoxelSiteDomain: From ee4157c2a49307a447b93d682e96420941df51ea Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 10:48:44 -0700 Subject: [PATCH 17/22] Migrate more services and tasks --- mise-tasks/dev-without-matrix | 15 +++++++++++++++ mise-tasks/services/ai-bot | 9 +++++++++ mise-tasks/services/bot-runner | 9 +++++++++ packages/ai-bot/package.json | 2 +- packages/bot-runner/package.json | 2 +- packages/realm-server/package.json | 2 +- 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100755 mise-tasks/dev-without-matrix create mode 100755 mise-tasks/services/ai-bot create mode 100755 mise-tasks/services/bot-runner diff --git a/mise-tasks/dev-without-matrix b/mise-tasks/dev-without-matrix new file mode 100755 index 00000000000..93962a3e86f --- /dev/null +++ b/mise-tasks/dev-without-matrix @@ -0,0 +1,15 @@ +#!/bin/sh +#MISE description="Start dev stack without starting Matrix/SMTP (expects them already running)" +#MISE dir="packages/realm-server" + +READY_PATH="_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson" +BASE_REALM_READY="http-get://${REALM_BASE_URL#http://}/base/${READY_PATH}" +EXPERIMENTS_READY="http-get://${REALM_BASE_URL#http://}/experiments/${READY_PATH}" +NODE_TEST_REALM_READY="http-get://${REALM_TEST_URL#http://}/node-test/${READY_PATH}" + +WAIT_ON_TIMEOUT=900000 SKIP_BOXEL_HOMEPAGE=true NODE_NO_WARNINGS=1 start-server-and-test \ + 'run-p start:pg start:prerender-dev start:prerender-manager-dev start:worker-development start:development' \ + "${BASE_REALM_READY}|${EXPERIMENTS_READY}|${MATRIX_URL_VAL}|http://localhost:5001|${ICONS_URL}" \ + 'run-p start:worker-test start:test-realms' \ + "${NODE_TEST_REALM_READY}" \ + 'wait' diff --git a/mise-tasks/services/ai-bot b/mise-tasks/services/ai-bot new file mode 100755 index 00000000000..545a1167875 --- /dev/null +++ b/mise-tasks/services/ai-bot @@ -0,0 +1,9 @@ +#!/bin/sh +#MISE description="Start AI bot for development" +#MISE dir="packages/ai-bot" + +MATRIX_URL="${MATRIX_URL_VAL}" \ + NODE_NO_WARNINGS=1 \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE}" \ + exec ts-node --transpileOnly main diff --git a/mise-tasks/services/bot-runner b/mise-tasks/services/bot-runner new file mode 100755 index 00000000000..8a715f01223 --- /dev/null +++ b/mise-tasks/services/bot-runner @@ -0,0 +1,9 @@ +#!/bin/sh +#MISE description="Start bot runner for development" +#MISE dir="packages/bot-runner" + +MATRIX_URL="${MATRIX_URL_VAL}" \ + NODE_NO_WARNINGS=1 \ + PGPORT="${PGPORT}" \ + PGDATABASE="${PGDATABASE}" \ + exec ts-node --transpileOnly main diff --git a/packages/ai-bot/package.json b/packages/ai-bot/package.json index 0d3ba4839f6..e1260570fee 100644 --- a/packages/ai-bot/package.json +++ b/packages/ai-bot/package.json @@ -39,7 +39,7 @@ "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:glint": "glint", "start": "NODE_NO_WARNINGS=1 ts-node --transpileOnly main", - "start:development": "sh scripts/start-development.sh", + "start:development": "mise run services:ai-bot", "test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts", "get-chat": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/get_chat.ts" } diff --git a/packages/bot-runner/package.json b/packages/bot-runner/package.json index d3bc87c19d8..a26b053af63 100644 --- a/packages/bot-runner/package.json +++ b/packages/bot-runner/package.json @@ -16,7 +16,7 @@ }, "scripts": { "start": "NODE_NO_WARNINGS=1 ts-node --transpileOnly main", - "start:development": "NODE_NO_WARNINGS=1 PGDATABASE=boxel PGPORT=5435 ts-node --transpileOnly main", + "start:development": "mise run services:bot-runner", "test": "NODE_NO_WARNINGS=1 qunit --require ts-node/register/transpile-only tests/index.ts", "lint": "glint" } diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json index 4af5d006854..167784a412f 100644 --- a/packages/realm-server/package.json +++ b/packages/realm-server/package.json @@ -112,7 +112,7 @@ "start:all": "mise run dev", "start:skip-optional-realms": "mise run dev-minimal", "start:services-for-host-tests": "mise run test-services-host", - "start:without-matrix": "./scripts/start-without-matrix.sh", + "start:without-matrix": "mise run dev-without-matrix", "start:staging": "./scripts/start-staging.sh", "start:development": "mise run services:realm-server -- --workerManagerPort=4210", "start:production": "./scripts/start-production.sh", From 52363cb8c07d8dc32fc6815f6c7bd85f9e587c35 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 11:04:11 -0700 Subject: [PATCH 18/22] Extract some setup for host --- packages/host/config/environment.js | 8 ++-- packages/host/scripts/ember-serve.js | 55 ++-------------------- packages/host/scripts/serve-dist.js | 56 ++-------------------- packages/host/scripts/traefik-helpers.js | 60 ++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 106 deletions(-) create mode 100644 packages/host/scripts/traefik-helpers.js diff --git a/packages/host/config/environment.js b/packages/host/config/environment.js index f34edd6327c..673ecfd217a 100644 --- a/packages/host/config/environment.js +++ b/packages/host/config/environment.js @@ -8,8 +8,10 @@ const DEFAULT_FILE_SIZE_LIMIT_BYTES = 5 * 1024 * 1024; // 5MB let sqlSchema = fs.readFileSync(getLatestSchemaFile(), 'utf8'); -// Environment-mode: when BOXEL_ENVIRONMENT is set, derive default URLs from Traefik hostnames -function environmentSlug() { +// Environment-mode: when BOXEL_ENVIRONMENT is set, derive default URLs from Traefik hostnames. +// ENV_SLUG is set by mise's env-vars.sh; fall back to computing it for non-mise contexts. +function getEnvSlug() { + if (process.env.ENV_SLUG) return process.env.ENV_SLUG; let raw = process.env.BOXEL_ENVIRONMENT || ''; return raw .toLowerCase() @@ -32,7 +34,7 @@ function environmentDefaults() { openRouterRealmURL: 'http://localhost:4201/openrouter/', }; } - let slug = environmentSlug(); + let slug = getEnvSlug(); let realmHost = `realm-server.${slug}.localhost`; return { realmServerURL: `http://${realmHost}/`, diff --git a/packages/host/scripts/ember-serve.js b/packages/host/scripts/ember-serve.js index 2c5ac561af7..fc5a430766d 100644 --- a/packages/host/scripts/ember-serve.js +++ b/packages/host/scripts/ember-serve.js @@ -8,58 +8,9 @@ const { spawn } = require('child_process'); const path = require('path'); const net = require('net'); -const fs = require('fs'); const BOXEL_ENVIRONMENT = process.env.BOXEL_ENVIRONMENT; -function sanitizeSlug(raw) { - return raw - .toLowerCase() - .replace(/\//g, '-') - .replace(/[^a-z0-9-]/g, '') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); -} - -function getTraefikDynamicDir() { - try { - const { execSync } = require('child_process'); - const mounted = execSync( - `docker inspect boxel-traefik --format '{{range .Mounts}}{{if eq .Destination "/etc/traefik/dynamic"}}{{.Source}}{{end}}{{end}}'`, - { encoding: 'utf-8' }, - ).trim(); - if (mounted) return mounted; - } catch { - // fall through - } - return path.resolve(__dirname, '..', '..', '..', 'traefik', 'dynamic'); -} - -function registerWithTraefik(slug, hostname, port) { - const dynamicDir = getTraefikDynamicDir(); - const configPath = path.join(dynamicDir, `${slug}-host.yml`); - const routerKey = `host-${slug}`; - - const entry = [ - 'http:', - ' routers:', - ` ${routerKey}:`, - ' rule: "Host(`' + hostname + '`)"', - ` service: ${routerKey}`, - ' entryPoints:', - ' - web', - ' services:', - ` ${routerKey}:`, - ' loadBalancer:', - ' servers:', - ` - url: "http://host.docker.internal:${port}"`, - '', - ].join('\n'); - const tmpPath = configPath + '.tmp'; - fs.writeFileSync(tmpPath, entry, 'utf-8'); - fs.renameSync(tmpPath, configPath); -} - function startEmber(port) { const args = ['serve', '--port', String(port)]; const child = spawn('ember', args, { @@ -76,13 +27,15 @@ function startEmber(port) { } if (!BOXEL_ENVIRONMENT) { - // Legacy mode: default ember serve on port 4200 + // Standard mode: default ember serve on port 4200 startEmber(4200); } else { const { ensureTraefik } = require('./ensure-traefik'); + const { getEnvSlug, registerWithTraefik } = require('./traefik-helpers'); + ensureTraefik(); - const slug = sanitizeSlug(BOXEL_ENVIRONMENT); + const slug = getEnvSlug(); const hostname = `host.${slug}.localhost`; // Point the client at the per-environment Synapse via Traefik diff --git a/packages/host/scripts/serve-dist.js b/packages/host/scripts/serve-dist.js index b756f1d80ac..e6924781c92 100644 --- a/packages/host/scripts/serve-dist.js +++ b/packages/host/scripts/serve-dist.js @@ -31,66 +31,18 @@ function runServe(port) { return child; } -function sanitizeSlug(raw) { - return raw - .toLowerCase() - .replace(/\//g, '-') - .replace(/[^a-z0-9-]/g, '') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); -} - -function getTraefikDynamicDir() { - try { - const { execSync } = require('child_process'); - const mounted = execSync( - `docker inspect boxel-traefik --format '{{range .Mounts}}{{if eq .Destination "/etc/traefik/dynamic"}}{{.Source}}{{end}}{{end}}'`, - { encoding: 'utf-8' }, - ).trim(); - if (mounted) return mounted; - } catch { - // fall through - } - return path.resolve(__dirname, '..', '..', '..', 'traefik', 'dynamic'); -} - -function registerWithTraefik(slug, hostname, port) { - const fs = require('fs'); - const dynamicDir = getTraefikDynamicDir(); - const configPath = path.join(dynamicDir, `${slug}-host.yml`); - const routerKey = `host-${slug}`; - - const entry = [ - 'http:', - ' routers:', - ` ${routerKey}:`, - ' rule: "Host(`' + hostname + '`)"', - ` service: ${routerKey}`, - ' entryPoints:', - ' - web', - ' services:', - ` ${routerKey}:`, - ' loadBalancer:', - ' servers:', - ` - url: "http://host.docker.internal:${port}"`, - '', - ].join('\n'); - const tmpPath = configPath + '.tmp'; - fs.writeFileSync(tmpPath, entry, 'utf-8'); - fs.renameSync(tmpPath, configPath); -} - if (!BOXEL_ENVIRONMENT) { - // Legacy mode: hardcoded port 4200 + // Standard mode: hardcoded port 4200 runServe(4200); } else { - // Environment mode: dynamic port + Traefik registration const { ensureTraefik } = require('./ensure-traefik'); + const { getEnvSlug, registerWithTraefik } = require('./traefik-helpers'); + ensureTraefik(); const net = require('net'); - const slug = sanitizeSlug(BOXEL_ENVIRONMENT); + const slug = getEnvSlug(); const hostname = `host.${slug}.localhost`; // Find a free port diff --git a/packages/host/scripts/traefik-helpers.js b/packages/host/scripts/traefik-helpers.js new file mode 100644 index 00000000000..726455eb247 --- /dev/null +++ b/packages/host/scripts/traefik-helpers.js @@ -0,0 +1,60 @@ +/** + * Shared helpers for Traefik registration in environment mode. + * Both ember-serve.js and serve-dist.js use these. + */ + +const path = require('path'); +const fs = require('fs'); + +function getEnvSlug() { + // Prefer ENV_SLUG from mise's env-vars.sh; fall back to computing it + if (process.env.ENV_SLUG) return process.env.ENV_SLUG; + const raw = process.env.BOXEL_ENVIRONMENT || ''; + return raw + .toLowerCase() + .replace(/\//g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); +} + +function getTraefikDynamicDir() { + try { + const { execSync } = require('child_process'); + const mounted = execSync( + `docker inspect boxel-traefik --format '{{range .Mounts}}{{if eq .Destination "/etc/traefik/dynamic"}}{{.Source}}{{end}}{{end}}'`, + { encoding: 'utf-8' }, + ).trim(); + if (mounted) return mounted; + } catch { + // fall through + } + return path.resolve(__dirname, '..', '..', '..', 'traefik', 'dynamic'); +} + +function registerWithTraefik(slug, hostname, port) { + const dynamicDir = getTraefikDynamicDir(); + const configPath = path.join(dynamicDir, `${slug}-host.yml`); + const routerKey = `host-${slug}`; + + const entry = [ + 'http:', + ' routers:', + ` ${routerKey}:`, + ' rule: "Host(`' + hostname + '`)"', + ` service: ${routerKey}`, + ' entryPoints:', + ' - web', + ' services:', + ` ${routerKey}:`, + ' loadBalancer:', + ' servers:', + ` - url: "http://host.docker.internal:${port}"`, + '', + ].join('\n'); + const tmpPath = configPath + '.tmp'; + fs.writeFileSync(tmpPath, entry, 'utf-8'); + fs.renameSync(tmpPath, configPath); +} + +module.exports = { getEnvSlug, getTraefikDynamicDir, registerWithTraefik }; From 31ffeed03a7f8773800c474e424c334c0f9bf2a5 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 11:24:10 -0700 Subject: [PATCH 19/22] Extract slug calculation --- mise-tasks/lib/env-vars.sh | 16 +++++++++++--- packages/ai-bot/scripts/start-development.sh | 5 ++++- packages/host/scripts/traefik-helpers.js | 3 ++- .../matrix/scripts/assert-synapse-running.sh | 5 ++++- .../matrix/scripts/register-realm-users.sh | 5 ++++- packages/matrix/scripts/start-synapse.sh | 5 ++++- packages/realm-server/scripts/start-icons.sh | 5 ++++- packages/realm-server/scripts/start-matrix.sh | 5 ++++- scripts/ensure-branch-db.sh | 9 ++++++-- scripts/env-slug.sh | 21 +++++++++++++++++++ scripts/stop-branch.sh | 3 ++- 11 files changed, 69 insertions(+), 13 deletions(-) create mode 100755 scripts/env-slug.sh diff --git a/mise-tasks/lib/env-vars.sh b/mise-tasks/lib/env-vars.sh index 1bb85b05712..65640d8cc72 100755 --- a/mise-tasks/lib/env-vars.sh +++ b/mise-tasks/lib/env-vars.sh @@ -6,9 +6,19 @@ # variables are correctly set when BOXEL_ENVIRONMENT is present. The standard-mode # branch uses ${VAR:-default} to avoid clobbering production/staging env vars in CI. -compute_env_slug() { - echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g' -} +# Resolve repo root from this file's location (works whether sourced or executed). +# When sourced via mise's _.source, $0 may be the parent shell, so we also +# try BASH_SOURCE and fall back to the path relative to .mise.toml (repo root). +if [ -n "${BASH_SOURCE:-}" ]; then + _ENV_VARS_DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd)" + _REPO_ROOT="$(cd "$_ENV_VARS_DIR/../.." && pwd)" +elif [ -f "./scripts/env-slug.sh" ]; then + _REPO_ROOT="$(pwd)" +else + _REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +fi +. "$_REPO_ROOT/scripts/env-slug.sh" +unset _ENV_VARS_DIR _REPO_ROOT export PGPORT="${PGPORT:-5435}" diff --git a/packages/ai-bot/scripts/start-development.sh b/packages/ai-bot/scripts/start-development.sh index 19054e34a02..72491ed071f 100644 --- a/packages/ai-bot/scripts/start-development.sh +++ b/packages/ai-bot/scripts/start-development.sh @@ -4,11 +4,14 @@ set -e +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + export NODE_NO_WARNINGS=1 export PGPORT="${PGPORT:-5435}" if [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(resolve_env_slug) CONTAINER_NAME="boxel-synapse-${SLUG}" PORT_OUTPUT=$(docker port "$CONTAINER_NAME" 8008/tcp 2>/dev/null | head -1) || true diff --git a/packages/host/scripts/traefik-helpers.js b/packages/host/scripts/traefik-helpers.js index 726455eb247..07245937f56 100644 --- a/packages/host/scripts/traefik-helpers.js +++ b/packages/host/scripts/traefik-helpers.js @@ -7,7 +7,8 @@ const path = require('path'); const fs = require('fs'); function getEnvSlug() { - // Prefer ENV_SLUG from mise's env-vars.sh; fall back to computing it + // Prefer ENV_SLUG from mise's env-vars.sh (canonical: scripts/env-slug.sh); + // fall back to computing it for non-mise contexts. if (process.env.ENV_SLUG) return process.env.ENV_SLUG; const raw = process.env.BOXEL_ENVIRONMENT || ''; return raw diff --git a/packages/matrix/scripts/assert-synapse-running.sh b/packages/matrix/scripts/assert-synapse-running.sh index 7e6cd0b7eb4..5d6010b12b7 100755 --- a/packages/matrix/scripts/assert-synapse-running.sh +++ b/packages/matrix/scripts/assert-synapse-running.sh @@ -2,8 +2,11 @@ # Check if the correct Synapse container is running (environment-aware). # If not running, start it. +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + if [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(resolve_env_slug) CONTAINER_NAME="boxel-synapse-${SLUG}" else CONTAINER_NAME="boxel-synapse" diff --git a/packages/matrix/scripts/register-realm-users.sh b/packages/matrix/scripts/register-realm-users.sh index 34717c5d019..1cc5545016c 100755 --- a/packages/matrix/scripts/register-realm-users.sh +++ b/packages/matrix/scripts/register-realm-users.sh @@ -1,8 +1,11 @@ #! /bin/sh +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + # Determine the Synapse health check URL (environment-aware) if [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(resolve_env_slug) CONTAINER_NAME="boxel-synapse-${SLUG}" # Read the dynamic host port from the running container SYNAPSE_HOST_PORT=$(docker port "$CONTAINER_NAME" 8008/tcp 2>/dev/null | head -1 | awk -F: '{print $NF}') diff --git a/packages/matrix/scripts/start-synapse.sh b/packages/matrix/scripts/start-synapse.sh index 93f34635958..aeadba1367a 100755 --- a/packages/matrix/scripts/start-synapse.sh +++ b/packages/matrix/scripts/start-synapse.sh @@ -1,8 +1,11 @@ #!/bin/sh # Start Synapse with per-environment data directory when BOXEL_ENVIRONMENT is set. +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + if [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(resolve_env_slug) SYNAPSE_DATA_DIR="./synapse-data-${SLUG}" else SYNAPSE_DATA_DIR="./synapse-data" diff --git a/packages/realm-server/scripts/start-icons.sh b/packages/realm-server/scripts/start-icons.sh index 8b2b58aed47..ab770a42b55 100644 --- a/packages/realm-server/scripts/start-icons.sh +++ b/packages/realm-server/scripts/start-icons.sh @@ -1,5 +1,8 @@ #! /bin/sh +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + if [ -n "$BOXEL_ENVIRONMENT" ]; then # In environment mode, use port 0 (dynamic) and register with Traefik. # http-server doesn't support port 0, so we pick a free port ourselves. @@ -9,7 +12,7 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then ICONS_PID=$! # Register icons service with Traefik via a small node script - ENV_SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + ENV_SLUG=$(resolve_env_slug) node -e " const fs = require('fs'); const path = require('path'); diff --git a/packages/realm-server/scripts/start-matrix.sh b/packages/realm-server/scripts/start-matrix.sh index 468ea2264b1..12d07ac6047 100755 --- a/packages/realm-server/scripts/start-matrix.sh +++ b/packages/realm-server/scripts/start-matrix.sh @@ -2,13 +2,16 @@ # Ensure Synapse is running. In environment mode, also auto-register users # since each environment gets a fresh Synapse data dir. +SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPTS_DIR/../../../scripts/env-slug.sh" + cd ../matrix pnpm assert-synapse-running # In environment mode, register users once per fresh Synapse data dir. if [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(resolve_env_slug) export PGDATABASE="${PGDATABASE:-boxel_${SLUG}}" export PGPORT="${PGPORT:-5435}" diff --git a/scripts/ensure-branch-db.sh b/scripts/ensure-branch-db.sh index da775dfbbed..52c6f307d59 100755 --- a/scripts/ensure-branch-db.sh +++ b/scripts/ensure-branch-db.sh @@ -5,12 +5,17 @@ set -e +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/env-slug.sh" + if [ -n "$1" ]; then SLUG="$1" +elif [ -n "$ENV_SLUG" ]; then + SLUG="$ENV_SLUG" elif [ -n "$BOXEL_ENVIRONMENT" ]; then - SLUG=$(echo "$BOXEL_ENVIRONMENT" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(compute_env_slug "$BOXEL_ENVIRONMENT") else - SLUG=$(git branch --show-current | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') + SLUG=$(compute_env_slug "$(git branch --show-current)") fi DB_NAME="boxel_${SLUG}" diff --git a/scripts/env-slug.sh b/scripts/env-slug.sh new file mode 100755 index 00000000000..145f7d38eaa --- /dev/null +++ b/scripts/env-slug.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Shared slug computation for environment names. +# Source this file and call compute_env_slug, or use resolve_env_slug +# which checks ENV_SLUG first (set by mise's env-vars.sh). +# +# Usage: +# . "$(dirname "$0")/../../scripts/env-slug.sh" # adjust path as needed +# SLUG=$(resolve_env_slug) # uses ENV_SLUG or BOXEL_ENVIRONMENT +# SLUG=$(compute_env_slug "my/Branch-Name") # explicit input + +compute_env_slug() { + echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g' +} + +resolve_env_slug() { + if [ -n "$ENV_SLUG" ]; then + echo "$ENV_SLUG" + elif [ -n "$BOXEL_ENVIRONMENT" ]; then + compute_env_slug "$BOXEL_ENVIRONMENT" + fi +} diff --git a/scripts/stop-branch.sh b/scripts/stop-branch.sh index bc8d73d42c1..3d80bb6256d 100755 --- a/scripts/stop-branch.sh +++ b/scripts/stop-branch.sh @@ -11,6 +11,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/env-slug.sh" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" TRAEFIK_DIR="$REPO_ROOT/traefik/dynamic" @@ -37,7 +38,7 @@ if [ -z "$BRANCH" ]; then exit 1 fi -SLUG=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's|/|-|g; s|[^a-z0-9-]||g; s|-\+|-|g; s|^-\|-$||g') +SLUG=$(compute_env_slug "$BRANCH") echo "Stopping all services for environment: $BRANCH (slug: $SLUG)" From aa09cf7ba4778bf5501d705e01f09d408870acad Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 11:33:39 -0700 Subject: [PATCH 20/22] Update README --- README.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 6bb91322224..f1dd6496c24 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,51 @@ Both catalogs are controlled by the same `SKIP_CATALOG` flag — setting `SKIP_C To learn more about Boxel and Cards, see our [documentation](./docs/README.md) +## mise Tasks + +Development services are managed as [mise](https://mise.jdx.dev/) file-based tasks. The `pnpm start:*` commands in `packages/realm-server/package.json` are shims that call `mise run` under the hood, so either form works: + +```bash +# These are equivalent: +pnpm start:all # shim in package.json +mise run dev # direct mise task + +pnpm start:development # shim +mise run services:realm-server # direct +``` + +### Task layout + +``` +mise-tasks/ + lib/env-vars.sh # Shared env computation (sourced by .mise.toml) + infra/ + ensure-traefik # Ensure Traefik is running (env mode only) + ensure-pg # Wait for PostgreSQL to be ready + ensure-db # Ensure per-environment database exists + ensure-synapse # Ensure Synapse + user registration + wait-for-prerender # Wait for prerender server + services/ + realm-server # Full development realm server + realm-server-base # Base realm only (for matrix tests) + worker, worker-base, # Worker managers + worker-test + prerender, prerender-mgr # Prerender server + manager + icons # Boxel icons HTTP server + ai-bot # AI bot + bot-runner # Bot runner + dev # Full dev stack (realm server + workers + test realms) + dev-all # Host app + full dev stack (single command) + dev-minimal # Dev stack without optional realms + dev-without-matrix # Dev stack (expects Matrix already running) + test-services-host # Services for host test suite + test-services-matrix # Services for matrix test suite +``` + +All tasks automatically receive environment variables from `mise-tasks/lib/env-vars.sh` (service URLs, ports, database names, paths). In environment mode (`BOXEL_ENVIRONMENT` set), these point to Traefik hostnames with dynamic ports. In standard mode, they use fixed localhost ports. + +To list all available tasks: `mise tasks ls` + ## Running the Host App There exists a "dev" mode in which we can use ember-cli to host the card runtime host application which includes live reloads. Additionally, you can also use the realm server to host the app, which is how it will be served in production, or have staging or production as the backing infrastructure. @@ -73,6 +118,14 @@ In order to run the ember-cli hosted app: 2. `pnpm start` in the host/ workspace to serve the ember app. 3. `pnpm start:all` in the realm-server/ to serve the base and experiments realms -- this will also allow you to switch between the app and the tests without having to restart servers). This expects the Ember application to be running at `http://localhost:4200`, if you’re running it elsewhere you can specify it with `HOST_URL=http://localhost:5200 pnpm start:all`. +Alternatively, you can run everything with a single command from the repo root: + +```bash +mise run dev-all +``` + +This starts the host app first, waits for it to be ready, then starts the realm server and all supporting services. + The app is available at http://localhost:4200. You will be prompted to register an account. To make it easier, you can execute `pnpm register-test-user` in `packages/matrix/`. Now you can sign in with the test user using the credentials `username: user`, `password: password`. When you are done running the app you can stop the synapse server by running the following from the `packages/matrix` workspace: @@ -94,7 +147,7 @@ Live reloads are not available in this mode, however, if you use start the serve #### Using `start:all` -Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. Use the environment variable `WORKER_HIGH_PRIORITY_COUNT` to add additional workers that service only user initiated requests and `WORKER_ALL_PRIORITY_COUNT` to add workers that service all jobs (system or user initiated). By default there is 1 all priority worker for each realm server. Here's what is spun up with `start:all`: +Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` (or `mise run dev`) which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. For faster startup, `pnpm start:skip-optional-realms` (or `mise run dev-minimal`) skips experiments, catalog, homepage, and submission realms. Use the environment variable `WORKER_HIGH_PRIORITY_COUNT` to add additional workers that service only user initiated requests and `WORKER_ALL_PRIORITY_COUNT` to add workers that service all jobs (system or user initiated). By default there is 1 all priority worker for each realm server. Here's what is spun up with `start:all`: | Port | Description | Running `start:all` | Running `start:base` | | ----- | --------------------------------------------------------------------------------------------- | ------------------- | -------------------- | @@ -108,15 +161,15 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` | :4211 | Test Worker Manager (spins up 1 worker by default) | ✅ | 🚫 | | :4212 | Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅ | 🚫 | | :4213 | Worker Manager for matrix client tests - base realm server (playwright controlled - 1 worker) | ✅ | 🚫 | -| :4221 | Prerender server | 🚫 | 🚫 | -| :4222 | Prerender manager | 🚫 | 🚫 | +| :4221 | Prerender server | ✅ | 🚫 | +| :4222 | Prerender manager | ✅ | 🚫 | | :5001 | Mail user interface for viewing emails sent to local SMTP | ✅ | 🚫 | | :5435 | Postgres DB | ✅ | 🚫 | | :8008 | Matrix synapse server | ✅ | 🚫 | #### Using `start:development` -You can also use `start:development` if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. In order to use `start:development` you must also make sure to run `start:worker-development` in order to start the workers which are normally started in `start:all`. +You can also use `start:development` (or `mise run services:realm-server`) if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. In order to use `start:development` you must also make sure to run `start:worker-development` (or `mise run services:worker`) in order to start the workers which are normally started in `start:all`. ### Card Pre-rendering @@ -138,7 +191,7 @@ From `packages/realm-server`: - `pnpm start:prerender-dev` (defaults to port 4221) -First start the pre-rendering manager. Then start the prerender server. (TBD: incorporate into start:all) +First start the pre-rendering manager. Then start the prerender server. Both are automatically started as part of `pnpm start:all` (`mise run dev`). #### Pre-rendering Environment variables @@ -467,42 +520,56 @@ If the isolated realm server fails to get stopped you may see an error about por ## Environment mode: parallel environments -“Environment mode” lets us run multiple Boxel environments simultaneously. This is experimental and opt-in via a `BOXEL_ENVIRONMENT` environment variable. +“Environment mode” lets multiple Boxel environments run simultaneously on the same machine. It is opt-in via the `BOXEL_ENVIRONMENT` environment variable. When set, all services use dynamic ports and register with a Traefik reverse proxy so each environment gets its own `*.localhost` hostnames. + +All environment configuration is centralized in `mise-tasks/lib/env-vars.sh`, which is automatically sourced by every mise task. The slug computation used for hostnames and database names lives in `scripts/env-slug.sh`. Here’s an example using Git worktrees and a `parallel` environment name: ```bash git worktree add ../parallel -ln -s "$(pwd)/packages/boxel-icons/dist" ../parallel/packages/boxel-icons/dist # skip freshly building boxel-icons, which is quite slow +ln -s “$(pwd)/packages/boxel-icons/dist” ../parallel/packages/boxel-icons/dist # skip freshly building boxel-icons, which is quite slow cd ../parallel pnpm install ``` -In a tab for the UI: +Start everything with a single command: ```bash -BOXEL_ENVIRONMENT=parallel pnpm --filter @cardstack/host start +BOXEL_ENVIRONMENT=parallel mise run dev-all ``` -In a tab for the realm server, skipping building the catalog since it takes a long time to index: +Or start services in separate tabs: ```bash -BOXEL_ENVIRONMENT=parallel SKIP_CATALOG=true pnpm --filter @cardstack/realm-server start:all -``` +# Tab 1: host app +BOXEL_ENVIRONMENT=parallel pnpm --filter @cardstack/host start -Optionally, for the AI bot: +# Tab 2: realm server (skip catalog for faster startup) +BOXEL_ENVIRONMENT=parallel SKIP_CATALOG=true pnpm --filter @cardstack/realm-server start:all -```bash +# Tab 3 (optional): AI bot BOXEL_ENVIRONMENT=parallel OPENROUTER_API_KEY=your_api_key pnpm --filter @cardstack/ai-bot start:development ``` +In environment mode, services are available at: + +| Service | Hostname | +| ------------- | ----------------------------------------- | +| Host app | `http://host..localhost` | +| Realm server | `http://realm-server..localhost` | +| Matrix | `http://matrix..localhost` | +| Icons | `http://icons..localhost` | + +Where `` is the lowercased, sanitized form of `BOXEL_ENVIRONMENT` (e.g., `feature/my-branch` becomes `feature-my-branch`). + ### Utility script -Run this script to terminate any dangling processes: +Run this script to terminate any dangling processes for an environment: ``` -./scripts/stop-branch.sh +./scripts/stop-branch.sh [environment-name] ``` It also has a `--drop-db` flag if you need to start over, and `--dry-run` to see what processes would be terminated. From 9480166c20cd513a68578ec7f7681bdaba2161f9 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 12:05:54 -0700 Subject: [PATCH 21/22] Remove documentation of legacy approaches --- README.md | 92 +++++++++++++++++++++++-------------------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index f1dd6496c24..f7e83a1da99 100644 --- a/README.md +++ b/README.md @@ -47,16 +47,7 @@ To learn more about Boxel and Cards, see our [documentation](./docs/README.md) ## mise Tasks -Development services are managed as [mise](https://mise.jdx.dev/) file-based tasks. The `pnpm start:*` commands in `packages/realm-server/package.json` are shims that call `mise run` under the hood, so either form works: - -```bash -# These are equivalent: -pnpm start:all # shim in package.json -mise run dev # direct mise task - -pnpm start:development # shim -mise run services:realm-server # direct -``` +Development services are managed as [mise](https://mise.jdx.dev/) file-based tasks in `mise-tasks/`. ### Task layout @@ -116,7 +107,7 @@ In order to run the ember-cli hosted app: 1. `pnpm build` in the boxel-ui/addon workspace to build the boxel-ui addon. 2. `pnpm start` in the host/ workspace to serve the ember app. -3. `pnpm start:all` in the realm-server/ to serve the base and experiments realms -- this will also allow you to switch between the app and the tests without having to restart servers). This expects the Ember application to be running at `http://localhost:4200`, if you’re running it elsewhere you can specify it with `HOST_URL=http://localhost:5200 pnpm start:all`. +3. `mise run dev` from the repo root to serve the base and experiments realms -- this will also allow you to switch between the app and the tests without having to restart servers). This expects the Ember application to be running at `http://localhost:4200`, if you’re running it elsewhere you can specify it with `HOST_URL=http://localhost:5200 mise run dev`. Alternatively, you can run everything with a single command from the repo root: @@ -139,37 +130,37 @@ pnpm stop:synapse In order to run the realm server hosted app: 1. `pnpm start:build` in the host/ workspace to re-build the host app (this step can be omitted if you do not want host app re-builds) -2. `pnpm start:all` in the realm-server/ to serve the base and experiments realms +2. `mise run dev` from the repo root to serve the base and experiments realms You can visit the URL of each realm server to view that realm's app. So for instance, the base realm's app is available at `http://localhost:4201/base` and the experiments realm's app is at `http://localhost:4201/experiments`. Live reloads are not available in this mode, however, if you use start the server with the environment variable `DISABLE_MODULE_CACHING=true` you can just refresh the page to grab the latest code changes if you are running rebuilds (step #1 and #2 above). -#### Using `start:all` +#### Using `mise run dev` -Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` (or `mise run dev`) which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. For faster startup, `pnpm start:skip-optional-realms` (or `mise run dev-minimal`) skips experiments, catalog, homepage, and submission realms. Use the environment variable `WORKER_HIGH_PRIORITY_COUNT` to add additional workers that service only user initiated requests and `WORKER_ALL_PRIORITY_COUNT` to add workers that service all jobs (system or user initiated). By default there is 1 all priority worker for each realm server. Here's what is spun up with `start:all`: +Instead of running `mise run services:realm-server-base`, you can alternatively use `mise run dev` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. For faster startup, `mise run dev-minimal` skips experiments, catalog, homepage, and submission realms. Use the environment variable `WORKER_HIGH_PRIORITY_COUNT` to add additional workers that service only user initiated requests and `WORKER_ALL_PRIORITY_COUNT` to add workers that service all jobs (system or user initiated). By default there is 1 all priority worker for each realm server. Here's what is spun up with `mise run dev`: -| Port | Description | Running `start:all` | Running `start:base` | -| ----- | --------------------------------------------------------------------------------------------- | ------------------- | -------------------- | -| :4201 | `/base` base realm | ✅ | ✅ | -| :4201 | `/skills` skills realm | ✅ | 🚫 | -| :4201 | `/experiments` experiments realm | ✅ | 🚫 | -| :4202 | `/test` host test realm, `/node-test` node test realm | ✅ | 🚫 | -| :4205 | `/test` realm for matrix client tests (playwright controlled) | 🚫 | 🚫 | -| :4206 | Boxel icons server | ✅ | 🚫 | -| :4210 | Development Worker Manager (spins up 1 worker by default) | ✅ | 🚫 | -| :4211 | Test Worker Manager (spins up 1 worker by default) | ✅ | 🚫 | -| :4212 | Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅ | 🚫 | -| :4213 | Worker Manager for matrix client tests - base realm server (playwright controlled - 1 worker) | ✅ | 🚫 | -| :4221 | Prerender server | ✅ | 🚫 | -| :4222 | Prerender manager | ✅ | 🚫 | -| :5001 | Mail user interface for viewing emails sent to local SMTP | ✅ | 🚫 | -| :5435 | Postgres DB | ✅ | 🚫 | -| :8008 | Matrix synapse server | ✅ | 🚫 | +| Port | Description | `mise run dev` | `mise run services:realm-server-base` | +| ----- | --------------------------------------------------------------------------------------------- | -------------- | ------------------------------------- | +| :4201 | `/base` base realm | ✅ | ✅ | +| :4201 | `/skills` skills realm | ✅ | 🚫 | +| :4201 | `/experiments` experiments realm | ✅ | 🚫 | +| :4202 | `/test` host test realm, `/node-test` node test realm | ✅ | 🚫 | +| :4205 | `/test` realm for matrix client tests (playwright controlled) | 🚫 | 🚫 | +| :4206 | Boxel icons server | ✅ | 🚫 | +| :4210 | Development Worker Manager (spins up 1 worker by default) | ✅ | 🚫 | +| :4211 | Test Worker Manager (spins up 1 worker by default) | ✅ | 🚫 | +| :4212 | Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅ | 🚫 | +| :4213 | Worker Manager for matrix client tests - base realm server (playwright controlled - 1 worker) | ✅ | 🚫 | +| :4221 | Prerender server | ✅ | 🚫 | +| :4222 | Prerender manager | ✅ | 🚫 | +| :5001 | Mail user interface for viewing emails sent to local SMTP | ✅ | 🚫 | +| :5435 | Postgres DB | ✅ | 🚫 | +| :8008 | Matrix synapse server | ✅ | 🚫 | -#### Using `start:development` +#### Using `mise run services:realm-server` -You can also use `start:development` (or `mise run services:realm-server`) if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. In order to use `start:development` you must also make sure to run `start:worker-development` (or `mise run services:worker`) in order to start the workers which are normally started in `start:all`. +You can also use `mise run services:realm-server` if you want the functionality of `mise run dev`, but without running the test realms. This will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. You must also make sure to run `mise run services:worker` in order to start the workers which are normally started in `mise run dev`. ### Card Pre-rendering @@ -181,17 +172,12 @@ Boxel supports server-side rendering of cards via a lightweight prerender servic #### Pre-rendering start scripts -From `packages/realm-server`: - -1. Prerender manager - -- `pnpm start:prerender-manager-dev` (defaults to port 4222) - -2. Prerender server +To run individually from the repo root: -- `pnpm start:prerender-dev` (defaults to port 4221) +1. Prerender manager: `mise run services:prerender-mgr` (defaults to port 4222) +2. Prerender server: `mise run services:prerender` (defaults to port 4221) -First start the pre-rendering manager. Then start the prerender server. Both are automatically started as part of `pnpm start:all` (`mise run dev`). +Start the manager first, then the server. Both are automatically started as part of `mise run dev`. #### Pre-rendering Environment variables @@ -239,7 +225,7 @@ The realm server uses the request accept header to determine the type of request ### Database -Boxel uses a Postgres database. In development, the Postgres database runs within a docker container, `boxel-pg`, that is started as part of `pnpm start:all`. You can manually start and stop the `boxel-pg` docker container using `pnpm start:pg` and `pnpm stop:pg`. The postgres database runs on port 5435 so that it doesn't conflict with a natively installed postgres that may be running on your system. +Boxel uses a Postgres database. In development, the Postgres database runs within a docker container, `boxel-pg`, that is started as part of `mise run dev`. You can manually start and stop the `boxel-pg` docker container using `mise run infra:ensure-pg` and `pnpm stop:pg` (from `packages/realm-server`). The postgres database runs on port 5435 so that it doesn't conflict with a natively installed postgres that may be running on your system. When running tests for matrix tests we isolate the database between each test run by actually creating a new database for each test with a random database name (e.g. `test_db_1234567`). The test databases can be dropped by running `pnpm test:cleanup` from the `packages/matrix` workspace, this will drop all databases that match the pattern `test_db_%`. @@ -249,7 +235,7 @@ If you wish to drop the development databases you can execute: pnpm full-reset ``` -You can then run `pnpm migrate up` (with `PGDATABASE` set accordingly if you want to migrate a database other than `boxel`) or just start the realm server (`pnpm start:all`) to create the database again. +You can then run `pnpm migrate up` (with `PGDATABASE` set accordingly if you want to migrate a database other than `boxel`) or just start the realm server (`mise run dev`) to create the database again. To interact with your local database directly you can use psql: @@ -293,7 +279,7 @@ This will create a new SQLite schema based on the current postgres DB (the schem The boxel platform leverages a Matrix server called Synapse in order to support identity, workflow, and chat behaviors. This project uses a dockerized Matrix server. We have multiple matrix server configurations (currently one for development that uses a persistent DB, and one for testing that uses an in-memory DB). You can find and configure these matrix servers at `packages/matrix/docker/synapse/*`. -This server is automatically started as part of the `pnpm start:all` script, but if you wish to control it separately, from `packages/matrix`, execute: +This server is automatically started as part of `mise run dev`, but if you wish to control it separately, from `packages/matrix`, execute: ``` pnpm start:synapse @@ -338,7 +324,7 @@ pnpm stop:admin #### SMTP Server -Matrix requires an SMTP server in order to send emails. In order to facilitate this we leverage [smtp4dev](https://github.com/rnwood/smtp4dev) in dev and test (CI) environments . This is a docker container that includes both a local SMTP server and hosts a web app for viewing all emails send from the SMTP server (the emails never leave the docker container). smtp4dev runs in the same docker network as synapse, so the SMTP port is never projected to the docker host. smtp4dev also runs the web app used to view emails sent from the SMTP server at `http://localhost:5001`. You can open a browser tab with this URL to view any emails sent from the matrix server. As well as, our matrix tests leverage the mail web app in order to perform email assertions. smtp4dev is automatically started as part of running `pnpm start:all` in the `packages/realm-server` workspace. +Matrix requires an SMTP server in order to send emails. In order to facilitate this we leverage [smtp4dev](https://github.com/rnwood/smtp4dev) in dev and test (CI) environments . This is a docker container that includes both a local SMTP server and hosts a web app for viewing all emails send from the SMTP server (the emails never leave the docker container). smtp4dev runs in the same docker network as synapse, so the SMTP port is never projected to the docker host. smtp4dev also runs the web app used to view emails sent from the SMTP server at `http://localhost:5001`. You can open a browser tab with this URL to view any emails sent from the matrix server. As well as, our matrix tests leverage the mail web app in order to perform email assertions. smtp4dev is automatically started as part of `mise run dev`. ## Boxel UI Component Explorer @@ -373,7 +359,7 @@ STRIPE_API_KEY=... pnpm sync-stripe-products 4. Go to `packages/realm-server`, pass STRIPE_WEBHOOK_SECRET & STRIPE_API_KEY environment value and start the realm server. STRIPE_WEBHOOK_SECRET is the value you got from stripe cli in Step 2. ``` -STRIPE_WEBHOOK_SECRET=... STRIPE_API_KEY=... pnpm start:all +STRIPE_WEBHOOK_SECRET=... STRIPE_API_KEY=... mise run dev ``` 5. Perform "Setup up Secure Payment Method" flow. Subscribe with valid test card [here](https://docs.stripe.com/testing#cards) @@ -451,7 +437,7 @@ There are currently 5 test suites: To run the `packages/host/` workspace tests start the following servers: -1. `pnpm start:all` in the `packages/realm-server/` to serve _both_ the base realm and the realm that serves the test cards +1. `mise run dev` from the repo root to serve _both_ the base realm and the realm that serves the test cards 2. `pnpm start` in the `packages/host/` workspace to serve ember The tests are available at `http://localhost:4200/tests` @@ -462,7 +448,7 @@ First make sure to generate the host app's `dist/` output in order to support ca To run the `packages/realm-server/` workspace tests start: -1. `pnpm start:all` in the `packages/realm-server/` to serve _both_ the base realm and the realm that serves the test cards for node. +1. `mise run dev` from the repo root to serve _both_ the base realm and the realm that serves the test cards for node. 2. Run `pnpm test` in the `packages/realm-server/` workspace to run the realm node tests. `TEST_MODULE=realm-endpoints-test.ts pnpm test-module` is an example of how to run a single test module. ### Boxel UI @@ -484,10 +470,10 @@ The matrix client relies upon the host app and the realm servers. Start the host pnpm start ``` -Then start the realm server for matrix tests (does not start the matrix server). From the `packages/realm-server` folder: +Then start the realm server for matrix tests (does not start the matrix server). From the repo root: ``` -MATRIX_REGISTRATION_SHARED_SECRET='xxxx' pnpm start:services-for-matrix-tests +MATRIX_REGISTRATION_SHARED_SECRET='xxxx' mise run test-services-matrix ``` Then to run the tests from the CLI execute the following from `packages/matrix`: @@ -547,10 +533,10 @@ Or start services in separate tabs: BOXEL_ENVIRONMENT=parallel pnpm --filter @cardstack/host start # Tab 2: realm server (skip catalog for faster startup) -BOXEL_ENVIRONMENT=parallel SKIP_CATALOG=true pnpm --filter @cardstack/realm-server start:all +BOXEL_ENVIRONMENT=parallel SKIP_CATALOG=true mise run dev # Tab 3 (optional): AI bot -BOXEL_ENVIRONMENT=parallel OPENROUTER_API_KEY=your_api_key pnpm --filter @cardstack/ai-bot start:development +BOXEL_ENVIRONMENT=parallel mise run services:ai-bot ``` In environment mode, services are available at: From 2880c0859b8657949c533e9da448be8abae29244 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 16 Mar 2026 12:42:14 -0700 Subject: [PATCH 22/22] Move more tasks and services --- README.md | 37 ++++++++++++++++++---------------- mise-tasks/infra/start-admin | 5 +++++ mise-tasks/infra/start-synapse | 5 +++++ mise-tasks/infra/stop-admin | 4 ++++ mise-tasks/infra/stop-pg | 4 ++++ mise-tasks/infra/stop-synapse | 5 +++++ mise-tasks/services/host-build | 5 +++++ 7 files changed, 48 insertions(+), 17 deletions(-) create mode 100755 mise-tasks/infra/start-admin create mode 100755 mise-tasks/infra/start-synapse create mode 100755 mise-tasks/infra/stop-admin create mode 100755 mise-tasks/infra/stop-pg create mode 100755 mise-tasks/infra/stop-synapse create mode 100755 mise-tasks/services/host-build diff --git a/README.md b/README.md index f7e83a1da99..a3d352d792f 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,11 @@ mise-tasks/ lib/env-vars.sh # Shared env computation (sourced by .mise.toml) infra/ ensure-traefik # Ensure Traefik is running (env mode only) - ensure-pg # Wait for PostgreSQL to be ready + ensure-pg, stop-pg # Start/stop PostgreSQL ensure-db # Ensure per-environment database exists ensure-synapse # Ensure Synapse + user registration + start-synapse, stop-synapse # Start/stop Synapse manually + start-admin, stop-admin # Matrix admin console (port 8080) wait-for-prerender # Wait for prerender server services/ realm-server # Full development realm server @@ -67,6 +69,7 @@ mise-tasks/ worker-test prerender, prerender-mgr # Prerender server + manager icons # Boxel icons HTTP server + host-build # Build and watch the host app ai-bot # AI bot bot-runner # Bot runner dev # Full dev stack (realm server + workers + test realms) @@ -119,18 +122,18 @@ This starts the host app first, waits for it to be ready, then starts the realm The app is available at http://localhost:4200. You will be prompted to register an account. To make it easier, you can execute `pnpm register-test-user` in `packages/matrix/`. Now you can sign in with the test user using the credentials `username: user`, `password: password`. -When you are done running the app you can stop the synapse server by running the following from the `packages/matrix` workspace: +When you are done running the app you can stop the synapse server: ``` -pnpm stop:synapse +mise run infra:stop-synapse ``` ### Realm server Hosted App In order to run the realm server hosted app: -1. `pnpm start:build` in the host/ workspace to re-build the host app (this step can be omitted if you do not want host app re-builds) -2. `mise run dev` from the repo root to serve the base and experiments realms +1. `mise run services:host-build` to re-build the host app (this step can be omitted if you do not want host app re-builds) +2. `mise run dev` to serve the base and experiments realms You can visit the URL of each realm server to view that realm's app. So for instance, the base realm's app is available at `http://localhost:4201/base` and the experiments realm's app is at `http://localhost:4201/experiments`. @@ -225,7 +228,7 @@ The realm server uses the request accept header to determine the type of request ### Database -Boxel uses a Postgres database. In development, the Postgres database runs within a docker container, `boxel-pg`, that is started as part of `mise run dev`. You can manually start and stop the `boxel-pg` docker container using `mise run infra:ensure-pg` and `pnpm stop:pg` (from `packages/realm-server`). The postgres database runs on port 5435 so that it doesn't conflict with a natively installed postgres that may be running on your system. +Boxel uses a Postgres database. In development, the Postgres database runs within a docker container, `boxel-pg`, that is started as part of `mise run dev`. You can manually start and stop the `boxel-pg` docker container using `mise run infra:ensure-pg` and `mise run infra:stop-pg`. The postgres database runs on port 5435 so that it doesn't conflict with a natively installed postgres that may be running on your system. When running tests for matrix tests we isolate the database between each test run by actually creating a new database for each test with a random database name (e.g. `test_db_1234567`). The test databases can be dropped by running `pnpm test:cleanup` from the `packages/matrix` workspace, this will drop all databases that match the pattern `test_db_%`. @@ -279,18 +282,18 @@ This will create a new SQLite schema based on the current postgres DB (the schem The boxel platform leverages a Matrix server called Synapse in order to support identity, workflow, and chat behaviors. This project uses a dockerized Matrix server. We have multiple matrix server configurations (currently one for development that uses a persistent DB, and one for testing that uses an in-memory DB). You can find and configure these matrix servers at `packages/matrix/docker/synapse/*`. -This server is automatically started as part of `mise run dev`, but if you wish to control it separately, from `packages/matrix`, execute: +This server is automatically started as part of `mise run dev`, but if you wish to control it separately: ``` -pnpm start:synapse +mise run infra:start-synapse ``` The local Matrix server will be running at `http://localhost:8008`. -To stop the matrix server, from `packages/matrix`, execute: +To stop the matrix server: ``` -pnpm stop:synapse +mise run infra:stop-synapse ``` #### Matrix Administration @@ -299,27 +302,27 @@ Matrix administration requires an administrative user and a special client in or First you must create an administrative user: -1. start the matrix server `pnpm start:synapse` +1. start the matrix server `mise run infra:start-synapse` 2. run a script to create an administrative user: ``` docker exec -it boxel-synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml -u admin -p your_admin_password --admin ``` Alternatively, you can execute `pnpm register-test-admin` and utilize the following credentials: `user: admin` and `password: password`. -After you have created an administrative user and can start the admin console by executing the following in the packages/matrix workspace: +After you have created an administrative user and can start the admin console: ``` -pnpm start:admin +mise run infra:start-admin ``` Then visit `http://localhost:8080`, and enter the admin user's username (`admin`) and the password, also enter in your matrix server url `http://localhost:8008` in the homeserver URL field, and click "Signin". Note you can use this same administrative interface to login to the staging and production matrix server. The credentials are available in AWS SSM Parameter Store. -To stop the admin console run the following in the packages/matrix workspace: +To stop the admin console: ``` -pnpm stop:admin +mise run infra:stop-admin ``` #### SMTP Server @@ -458,10 +461,10 @@ To run the `packages/realm-server/` workspace tests start: ### Matrix tests -This test suite contains tests that exercise matrix functionality. These tests are located at `packages/matrix/tests`, and are executed using the [Playwright](https://playwright.dev/) test runner. To run the tests from the command line, first make sure that the matrix server is not already running. You can stop the matrix server by executing the following from `packages/matrix` +This test suite contains tests that exercise matrix functionality. These tests are located at `packages/matrix/tests`, and are executed using the [Playwright](https://playwright.dev/) test runner. To run the tests from the command line, first make sure that the matrix server is not already running: ``` -pnpm stop:synapse +mise run infra:stop-synapse ``` The matrix client relies upon the host app and the realm servers. Start the host app from the `packages/host` folder: diff --git a/mise-tasks/infra/start-admin b/mise-tasks/infra/start-admin new file mode 100755 index 00000000000..09ed70ce8c4 --- /dev/null +++ b/mise-tasks/infra/start-admin @@ -0,0 +1,5 @@ +#!/bin/sh +#MISE description="Start the Matrix admin console (port 8080)" +#MISE dir="packages/matrix" + +exec ts-node --transpileOnly ./scripts/admin-console diff --git a/mise-tasks/infra/start-synapse b/mise-tasks/infra/start-synapse new file mode 100755 index 00000000000..2e0d393dd1b --- /dev/null +++ b/mise-tasks/infra/start-synapse @@ -0,0 +1,5 @@ +#!/bin/sh +#MISE description="Start the Matrix Synapse server" +#MISE dir="packages/matrix" + +exec sh scripts/start-synapse.sh diff --git a/mise-tasks/infra/stop-admin b/mise-tasks/infra/stop-admin new file mode 100755 index 00000000000..fc50dc12c18 --- /dev/null +++ b/mise-tasks/infra/stop-admin @@ -0,0 +1,4 @@ +#!/bin/sh +#MISE description="Stop the Matrix admin console" + +docker stop synapse-admin && docker rm synapse-admin diff --git a/mise-tasks/infra/stop-pg b/mise-tasks/infra/stop-pg new file mode 100755 index 00000000000..86c6c47b4c9 --- /dev/null +++ b/mise-tasks/infra/stop-pg @@ -0,0 +1,4 @@ +#!/bin/sh +#MISE description="Stop the PostgreSQL docker container" + +docker stop boxel-pg >/dev/null diff --git a/mise-tasks/infra/stop-synapse b/mise-tasks/infra/stop-synapse new file mode 100755 index 00000000000..d8cf9a5b54a --- /dev/null +++ b/mise-tasks/infra/stop-synapse @@ -0,0 +1,5 @@ +#!/bin/sh +#MISE description="Stop the Matrix Synapse server" +#MISE dir="packages/matrix" + +exec ts-node --transpileOnly ./scripts/synapse.ts stop diff --git a/mise-tasks/services/host-build b/mise-tasks/services/host-build new file mode 100755 index 00000000000..30fc2424466 --- /dev/null +++ b/mise-tasks/services/host-build @@ -0,0 +1,5 @@ +#!/bin/sh +#MISE description="Build and watch the host app (no live reload)" +#MISE dir="packages/host" + +exec node --max-old-space-size=8192 ./node_modules/.bin/ember build --watch