From 784d4a3afd0c9d495bc5b34667c07bfe523d00ca Mon Sep 17 00:00:00 2001 From: Benjamin Shafii Date: Wed, 10 Jun 2026 09:37:15 -0700 Subject: [PATCH 1/2] ci(den-db): automated Drizzle migrations for PlanetScale - den-db-check.yml: PR gate that fails when the schema changes without a committed migration (db:generate must be a no-op) - den-db-migrate.yml: applies db:migrate to production PlanetScale when migration files land on dev; manual dispatch supports a one-time baseline (with dry-run) for the push-managed prod database - db:baseline script: records existing migrations as applied in __drizzle_migrations without executing them - drizzle.config.ts: enable TLS for the PlanetScale host/user/password credential path (required for MySQL-protocol connections from CI) - .env.example documents DATABASE_HOST/DATABASE_USERNAME/DATABASE_PASSWORD (same names as the GitHub Actions secrets) - README: CI workflow docs, baseline guide, expand/contract policy --- .github/workflows/den-db-check.yml | 45 ++++++ .github/workflows/den-db-migrate.yml | 104 +++++++++++++ ee/packages/den-db/.env.example | 22 +-- ee/packages/den-db/README.md | 46 ++++++ ee/packages/den-db/drizzle.config.ts | 2 + ee/packages/den-db/package.json | 1 + .../den-db/scripts/baseline-migrations.ts | 143 ++++++++++++++++++ 7 files changed, 353 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/den-db-check.yml create mode 100644 .github/workflows/den-db-migrate.yml create mode 100644 ee/packages/den-db/scripts/baseline-migrations.ts diff --git a/.github/workflows/den-db-check.yml b/.github/workflows/den-db-check.yml new file mode 100644 index 0000000000..b6e4a76428 --- /dev/null +++ b/.github/workflows/den-db-check.yml @@ -0,0 +1,45 @@ +name: Den DB Check + +on: + pull_request: + paths: + - "ee/packages/den-db/**" + +permissions: + contents: read + +jobs: + migrations-in-sync: + name: Schema and migrations in sync + runs-on: blacksmith-4vcpu-ubuntu-2204 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 11.4.0 + + - name: Install dependencies + run: pnpm install --filter "@openwork-ee/den-db..." + + - name: Generate migrations from schema + run: pnpm --filter @openwork-ee/den-db db:generate + + - name: Fail if schema changed without a committed migration + run: | + set -euo pipefail + changes="$(git status --porcelain -- ee/packages/den-db/drizzle)" + if [ -n "$changes" ]; then + echo "$changes" + echo "::error::Schema changed without a committed migration. Run 'pnpm --filter @openwork-ee/den-db db:generate' locally and commit the generated files in ee/packages/den-db/drizzle." + exit 1 + fi + echo "Migrations are in sync with the schema." diff --git a/.github/workflows/den-db-migrate.yml b/.github/workflows/den-db-migrate.yml new file mode 100644 index 0000000000..e9959c6855 --- /dev/null +++ b/.github/workflows/den-db-migrate.yml @@ -0,0 +1,104 @@ +name: Den DB Migrate + +# Applies Drizzle migrations to the production PlanetScale database. +# +# Required GitHub Actions secrets (Settings -> Secrets and variables -> Actions): +# DATABASE_HOST PlanetScale host (e.g. aws.connect.psdb.cloud) +# DATABASE_USERNAME PlanetScale branch password username +# DATABASE_PASSWORD PlanetScale branch password +# +# Runs automatically when migration files land on dev, and manually via +# workflow_dispatch. For a database previously managed with db:push (no +# __drizzle_migrations table yet), run once manually with baseline=true +# to record existing migrations as applied without executing them. + +on: + push: + branches: + - dev + paths: + - "ee/packages/den-db/drizzle/**" + workflow_dispatch: + inputs: + baseline: + description: "One-time: mark existing migrations as applied without executing them (for a db previously managed via db:push)" + type: boolean + default: false + baseline_through: + description: "Baseline through this migration tag (default: latest, e.g. 0020_breezy_siren)" + type: string + required: false + dry_run: + description: "Print what would happen without applying migrations" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: den-db-migrate + cancel-in-progress: false + +jobs: + migrate: + name: Apply migrations + if: github.repository == 'different-ai/openwork' + runs-on: blacksmith-4vcpu-ubuntu-2204 + + env: + DATABASE_HOST: ${{ secrets.DATABASE_HOST }} + DATABASE_USERNAME: ${{ secrets.DATABASE_USERNAME }} + DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} + + steps: + - name: Verify database secrets are configured + run: | + set -euo pipefail + missing="" + [ -z "$DATABASE_HOST" ] && missing="$missing DATABASE_HOST" + [ -z "$DATABASE_USERNAME" ] && missing="$missing DATABASE_USERNAME" + [ -z "$DATABASE_PASSWORD" ] && missing="$missing DATABASE_PASSWORD" + if [ -n "$missing" ]; then + echo "::error::Missing GitHub Actions secrets:$missing. Add them under Settings -> Secrets and variables -> Actions." + exit 1 + fi + echo "Database secrets present." + + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 11.4.0 + + - name: Install dependencies + run: pnpm install --filter "@openwork-ee/den-db..." + + - name: Baseline existing migrations (one-time) + if: github.event_name == 'workflow_dispatch' && inputs.baseline + run: | + set -euo pipefail + extra="" + if [ -n "${{ inputs.baseline_through }}" ]; then + extra="--through ${{ inputs.baseline_through }}" + fi + if [ "${{ inputs.dry_run }}" = "true" ]; then + pnpm --filter @openwork-ee/den-db db:baseline -- $extra + else + pnpm --filter @openwork-ee/den-db db:baseline -- --yes $extra + fi + + - name: Apply migrations + if: github.event_name != 'workflow_dispatch' || !inputs.dry_run + run: pnpm --filter @openwork-ee/den-db db:migrate + + - name: Dry run note + if: github.event_name == 'workflow_dispatch' && inputs.dry_run + run: echo "Dry run requested -- skipped 'db:migrate'. Baseline output above (if enabled) shows the plan." diff --git a/ee/packages/den-db/.env.example b/ee/packages/den-db/.env.example index 2d9c437180..4c81abc3cb 100644 --- a/ee/packages/den-db/.env.example +++ b/ee/packages/den-db/.env.example @@ -1,14 +1,16 @@ -# MySQL mode: if DATABASE_URL is set, den-db uses mysql/mysql2. -DATABASE_URL= - -# Required dedicated DB encryption key for encrypted columns. Minimum 32 chars. -# Generate one with: openssl rand -base64 128 -DEN_DB_ENCRYPTION_KEY= - -# PlanetScale mode: used when DATABASE_URL is not set. +# Den DB credentials. +# +# Local use: copy this file to .env.local (gitignored) and fill in the values. +# CI use: create GitHub Actions secrets with the SAME names +# (Settings -> Secrets and variables -> Actions) for the +# .github/workflows/den-db-migrate.yml workflow. +# +# PlanetScale connection (host/username/password from a PlanetScale +# password for the production branch): DATABASE_HOST= DATABASE_USERNAME= DATABASE_PASSWORD= -# Optional explicit env file path for Drizzle commands. -# OPENWORK_DEN_DB_ENV_PATH=/absolute/path/to/.env.production +# Alternative: a single MySQL connection URL (takes precedence over the +# trio above; used by the local docker dev stack): +# DATABASE_URL=mysql://root:password@127.0.0.1:3306/openwork_den diff --git a/ee/packages/den-db/README.md b/ee/packages/den-db/README.md index 9b2b6c7e0a..161fd842a3 100644 --- a/ee/packages/den-db/README.md +++ b/ee/packages/den-db/README.md @@ -29,8 +29,54 @@ Run Drizzle migrations against a configured database: pnpm --dir ee/packages/den-db db:migrate ``` +## Automated migrations (CI) + +Two GitHub Actions workflows keep schema and database in sync: + +- `.github/workflows/den-db-check.yml` — on every PR touching this package, + runs `db:generate` and fails if the schema changed without a committed + migration. +- `.github/workflows/den-db-migrate.yml` — applies migrations to the + production PlanetScale database when migration files land on `dev` + (and via manual `workflow_dispatch`). + +The migrate workflow reads these repository secrets (same names as the +local env vars — see `.env.example`): + +| Secret | Value | +| --- | --- | +| `DATABASE_HOST` | PlanetScale host (e.g. `aws.connect.psdb.cloud`) | +| `DATABASE_USERNAME` | PlanetScale branch password username | +| `DATABASE_PASSWORD` | PlanetScale branch password | + +### One-time baseline + +A database previously managed with `db:push` has no `__drizzle_migrations` +table, so the first `db:migrate` would try to replay every migration. +Record the existing history once (marks migrations as applied without +executing them): + +```bash +pnpm --dir ee/packages/den-db db:baseline # dry run +pnpm --dir ee/packages/den-db db:baseline -- --yes # record +``` + +Or run the `Den DB Migrate` workflow manually with `baseline: true` +(use `dry_run: true` first to see the plan). + +### Migration policy + +Migrations run **before** new code deploys, so they must be +expand/contract safe: additive columns are nullable or defaulted, no +renames or drops while old code still reads the schema, contract steps +ship as a later migration once no deployed code references the old shape. + ## Notes +- The migration chain has no `0000` baseline (history starts at `0001`, + which alters pre-existing tables), so an empty database cannot be built + by replaying migrations. Create fresh databases with `db:push` (dev) and + use `db:baseline` + `db:migrate` for databases that already have the schema. - `db:generate` is the default path for new migration files. - `drizzle/meta/` must stay in sync with the SQL migration history so future generation stays incremental. - Only repair `drizzle/meta/` manually when recovering broken Drizzle history. diff --git a/ee/packages/den-db/drizzle.config.ts b/ee/packages/den-db/drizzle.config.ts index ea72da42ec..71b0d8e892 100644 --- a/ee/packages/den-db/drizzle.config.ts +++ b/ee/packages/den-db/drizzle.config.ts @@ -33,6 +33,8 @@ function resolveDrizzleDbCredentials() { host, user, password, + // PlanetScale requires TLS for MySQL-protocol connections. + ssl: { rejectUnauthorized: true }, } } diff --git a/ee/packages/den-db/package.json b/ee/packages/den-db/package.json index 3adda1063c..56a4436e5c 100644 --- a/ee/packages/den-db/package.json +++ b/ee/packages/den-db/package.json @@ -75,6 +75,7 @@ "scripts": { "build": "pnpm run build:utils && tsup", "build:utils": "pnpm --dir ../utils run build", + "db:baseline": "node --import tsx scripts/baseline-migrations.ts", "db:generate": "pnpm run build && node --import tsx ./node_modules/drizzle-kit/bin.cjs generate --config drizzle.config.ts", "db:migrate": "pnpm run build && node --import tsx ./node_modules/drizzle-kit/bin.cjs migrate --config drizzle.config.ts", "db:push": "pnpm run build && node --import tsx ./node_modules/drizzle-kit/bin.cjs push --config drizzle.config.ts" diff --git a/ee/packages/den-db/scripts/baseline-migrations.ts b/ee/packages/den-db/scripts/baseline-migrations.ts new file mode 100644 index 0000000000..4b27806d2f --- /dev/null +++ b/ee/packages/den-db/scripts/baseline-migrations.ts @@ -0,0 +1,143 @@ +/** + * One-time baseline for databases that were previously managed with + * `db:push` (state-based) and have no `__drizzle_migrations` table. + * + * Marks existing migrations as applied WITHOUT executing them, so a + * subsequent `db:migrate` only runs migrations newer than the baseline. + * + * Usage: + * pnpm --filter @openwork-ee/den-db db:baseline # dry run + * pnpm --filter @openwork-ee/den-db db:baseline -- --yes # apply + * pnpm --filter @openwork-ee/den-db db:baseline -- --yes --through 0020_breezy_siren + * + * Connects with DATABASE_URL (mysql2) or DATABASE_HOST/DATABASE_USERNAME/ + * DATABASE_PASSWORD (PlanetScale HTTP driver) -- same as the rest of den-db. + */ +import "../src/load-env.ts" +import crypto from "node:crypto" +import { readFileSync } from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" + +const MIGRATIONS_TABLE = "__drizzle_migrations" + +const packageDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") +const drizzleDir = path.join(packageDir, "drizzle") + +type JournalEntry = { + idx: number + when: number + tag: string +} + +type Executor = { + query: (sql: string, args?: (string | number)[]) => Promise[]> + close: () => Promise +} + +function parseArgs() { + const args = process.argv.slice(2) + const apply = args.includes("--yes") + const throughIndex = args.indexOf("--through") + const through = throughIndex >= 0 ? args[throughIndex + 1] : undefined + return { apply, through } +} + +async function createExecutor(): Promise { + const databaseUrl = process.env.DATABASE_URL?.trim() + + if (databaseUrl) { + const mysql = await import("mysql2/promise") + const connection = await mysql.createConnection(databaseUrl) + return { + query: async (sql, args = []) => { + const [rows] = await connection.query(sql, args) + return Array.isArray(rows) ? (rows as Record[]) : [] + }, + close: () => connection.end(), + } + } + + const host = process.env.DATABASE_HOST?.trim() + const username = process.env.DATABASE_USERNAME?.trim() + const password = process.env.DATABASE_PASSWORD ?? "" + + if (!host || !username) { + throw new Error("Provide DATABASE_URL, or DATABASE_HOST/DATABASE_USERNAME/DATABASE_PASSWORD (see .env.example)") + } + + const { Client } = await import("@planetscale/database") + const client = new Client({ host, username, password }) + return { + query: async (sql, args = []) => { + const result = await client.execute(sql, args) + return result.rows as Record[] + }, + close: async () => {}, + } +} + +async function main() { + const { apply, through } = parseArgs() + + const journal = JSON.parse(readFileSync(path.join(drizzleDir, "meta", "_journal.json"), "utf8")) as { + entries: JournalEntry[] + } + const entries = [...journal.entries].sort((a, b) => a.when - b.when) + + if (entries.length === 0) { + console.log("No migrations in journal; nothing to baseline.") + return + } + + const throughEntry = through ? entries.find((e) => e.tag === through) : entries[entries.length - 1] + if (!throughEntry) { + throw new Error(`--through tag "${through}" not found in drizzle/meta/_journal.json`) + } + + const executor = await createExecutor() + try { + await executor.query( + `create table if not exists \`${MIGRATIONS_TABLE}\` (id serial primary key, hash text not null, created_at bigint)`, + ) + + const rows = await executor.query(`select max(created_at) as latest from \`${MIGRATIONS_TABLE}\``) + const latestRaw = rows[0]?.latest + const latest = latestRaw == null ? 0 : Number(latestRaw) + + const pending = entries.filter((e) => e.when > latest && e.when <= throughEntry.when) + + console.log(`Baseline target: ${throughEntry.tag} (when=${throughEntry.when})`) + console.log(`Already recorded through: created_at=${latest || "none"}`) + console.log(`Entries to mark as applied (without executing): ${pending.length}`) + for (const entry of pending) { + console.log(` - ${entry.tag}`) + } + + if (pending.length === 0) { + console.log("Nothing to do.") + return + } + + if (!apply) { + console.log("\nDry run. Re-run with --yes to record the baseline.") + return + } + + for (const entry of pending) { + const sqlContents = readFileSync(path.join(drizzleDir, `${entry.tag}.sql`), "utf8") + const hash = crypto.createHash("sha256").update(sqlContents).digest("hex") + await executor.query(`insert into \`${MIGRATIONS_TABLE}\` (hash, created_at) values (?, ?)`, [hash, entry.when]) + console.log(`Recorded ${entry.tag}`) + } + + console.log(`\nBaseline complete. 'db:migrate' will now only apply migrations newer than ${throughEntry.tag}.`) + } finally { + await executor.close() + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exit(1) +}) From fa72a8264c7da76c4800e05ba29034103f8a7fd2 Mon Sep 17 00:00:00 2001 From: Benjamin Shafii Date: Wed, 10 Jun 2026 09:37:46 -0700 Subject: [PATCH 2/2] chore(den-db): keep existing .env.example entries alongside CI notes --- ee/packages/den-db/.env.example | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ee/packages/den-db/.env.example b/ee/packages/den-db/.env.example index 4c81abc3cb..8cc6325726 100644 --- a/ee/packages/den-db/.env.example +++ b/ee/packages/den-db/.env.example @@ -1,16 +1,22 @@ # Den DB credentials. # # Local use: copy this file to .env.local (gitignored) and fill in the values. -# CI use: create GitHub Actions secrets with the SAME names -# (Settings -> Secrets and variables -> Actions) for the -# .github/workflows/den-db-migrate.yml workflow. -# -# PlanetScale connection (host/username/password from a PlanetScale -# password for the production branch): +# CI use: the .github/workflows/den-db-migrate.yml workflow reads GitHub +# Actions secrets with the SAME names (Settings -> Secrets and variables -> +# Actions): DATABASE_HOST, DATABASE_USERNAME, DATABASE_PASSWORD. + +# MySQL mode: if DATABASE_URL is set, den-db uses mysql/mysql2. +DATABASE_URL= + +# Required dedicated DB encryption key for encrypted columns. Minimum 32 chars. +# Generate one with: openssl rand -base64 128 +DEN_DB_ENCRYPTION_KEY= + +# PlanetScale mode: used when DATABASE_URL is not set. +# Values come from a PlanetScale password for the production branch. DATABASE_HOST= DATABASE_USERNAME= DATABASE_PASSWORD= -# Alternative: a single MySQL connection URL (takes precedence over the -# trio above; used by the local docker dev stack): -# DATABASE_URL=mysql://root:password@127.0.0.1:3306/openwork_den +# Optional explicit env file path for Drizzle commands. +# OPENWORK_DEN_DB_ENV_PATH=/absolute/path/to/.env.production