From 07e1f0ac9080f523013f173aa0eaca6e87212ef1 Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 3 Jul 2026 10:44:28 -0700 Subject: [PATCH 1/5] Regenerate Changelog on New Release --- web/DEPLOYMENT.md | 16 ++++ web/apps/docs/.gitignore | 4 + .../docs/content/docs/reference/changelog.mdx | 51 ----------- web/apps/docs/package.json | 5 +- web/apps/docs/scripts/gen-changelog.mjs | 87 +++++++++++++++++++ web/turbo.json | 14 +++ 6 files changed, 124 insertions(+), 53 deletions(-) delete mode 100644 web/apps/docs/content/docs/reference/changelog.mdx create mode 100644 web/apps/docs/scripts/gen-changelog.mjs diff --git a/web/DEPLOYMENT.md b/web/DEPLOYMENT.md index 42eff2926..c01ed1337 100644 --- a/web/DEPLOYMENT.md +++ b/web/DEPLOYMENT.md @@ -96,6 +96,22 @@ docs project's main-branch production. If you want a preview to proxy to the docs project's branch preview, you'd need a build-time hook to rewrite `DOCS_URL`; we haven't wired that yet. +## Generated content + +The docs changelog page (`/reference/changelog`) is not checked in. It is +generated from the repository-root `CHANGELOG.md` — the canonical changelog +git-cliff maintains at release time — by +`apps/docs/scripts/gen-changelog.mjs`, which runs at the start of the docs +`dev` and `build` scripts. Every merge to main therefore republishes the +page from the current changelog, and the web CI build on pull requests +fails if a changelog edit stops compiling as MDX. + +Because of this, don't configure an **Ignored Build Step** on the docs +Vercel project that skips builds when `web/` is unchanged: a release merge +touches only root files (`CHANGELOG.md`, version bumps), and skipping that +deploy would leave the published changelog stale until the next `web/` +change. + ## Operating - **Cache invalidation**: Vercel handles automatic edge invalidation on diff --git a/web/apps/docs/.gitignore b/web/apps/docs/.gitignore index 0abf60296..7317f8473 100644 --- a/web/apps/docs/.gitignore +++ b/web/apps/docs/.gitignore @@ -4,3 +4,7 @@ out/ .source/ next-env.d.ts *.tsbuildinfo + +# Generated from the repository-root CHANGELOG.md by scripts/gen-changelog.mjs +# (runs as part of `dev` and `build`). +content/docs/reference/changelog.mdx diff --git a/web/apps/docs/content/docs/reference/changelog.mdx b/web/apps/docs/content/docs/reference/changelog.mdx deleted file mode 100644 index 18a832d51..000000000 --- a/web/apps/docs/content/docs/reference/changelog.mdx +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Changelog -description: Notable changes per release. Latest first. ---- - -The complete commit-level changelog lives in -[`CHANGELOG.md`](https://github.com/TraceMachina/nativelink/blob/main/CHANGELOG.md) -on the repository. This page summarises what's worth knowing per -release. - -## v1.3 — May 2026 - -- **Persistent workers.** Long-lived worker processes for hot - toolchains (JVM, Bazel-style). 5-20× speedup on JVM-heavy builds. - See [Persistent workers](/deployment/persistent-workers). -- **3× faster scheduler dispatch.** Lock-free path on the hot - routing function; `p99` dispatch latency drops below 1 ms. -- **First-class GPU support.** Workers can advertise CUDA / ROCm - capabilities; the scheduler matches actions on them. -- **`fast_slow` store layering** is now stable. - -## v1.2 — March 2026 - -- **LRE shipped to GA.** Hermetic builds on `localhost`, no Docker. -- **Multi-region CAS replication.** Built-in cross-region sync for - the durable store layer. -- **OpenTelemetry tracing** wired through every RPC. Configurable - sample rate per operation. - -## v1.1 — January 2026 - -- **Worker autoscaling improvements.** New `queue_depth` Prometheus - metric, drives HPA more accurately than CPU. -- **S3 store retry hardening.** Exponential backoff with jitter, - per-request circuit breakers. -- **Configurable Action Cache TTL.** Per-instance overrides. - -## v1.0 — November 2025 - -- **GA release.** Production-stable APIs, semver from this point on. -- **FSL-Apache licensing.** The repository moved to - `FSL-1.1-Apache-2.0` for most modules, with feature-specific - Business Source License modules called out in source headers. See - the [license page](https://nativelink.com/license). -- **Helm chart** for Kubernetes deployment promoted to stable. - -## v0.x — pre-GA - -The 0.x line was iterated heavily; see the -[GitHub release notes](https://github.com/TraceMachina/nativelink/releases) -for the full history. diff --git a/web/apps/docs/package.json b/web/apps/docs/package.json index d61a9b2bd..ceaede8d4 100644 --- a/web/apps/docs/package.json +++ b/web/apps/docs/package.json @@ -25,9 +25,10 @@ }, "private": true, "scripts": { - "build": "next build", + "build": "node scripts/gen-changelog.mjs && next build", "clean": "rm -rf .next .turbo .source", - "dev": "next dev --port 3001", + "dev": "node scripts/gen-changelog.mjs && next dev --port 3001", + "gen:changelog": "node scripts/gen-changelog.mjs", "gen:config-reference": "node scripts/gen-config-reference.mjs", "lint": "biome check .", "start": "next start --port 3001", diff --git a/web/apps/docs/scripts/gen-changelog.mjs b/web/apps/docs/scripts/gen-changelog.mjs new file mode 100644 index 000000000..bead9eaf5 --- /dev/null +++ b/web/apps/docs/scripts/gen-changelog.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +// Generate the changelog reference page from the repository's canonical +// CHANGELOG.md (maintained with git-cliff at release time; see /cliff.toml and +// the "Creating a Release" section of CONTRIBUTING.md). +// +// The output at content/docs/reference/changelog.mdx is gitignored and +// regenerated as part of the `dev` and `build` scripts, so the published page +// always mirrors the CHANGELOG.md of the commit being deployed: every merge to +// main produces a fresh page via the production build, and the web CI build on +// pull requests fails if a changelog edit stops compiling as MDX. +// +// The transform is deliberately small: +// 1. drop HTML comments (`` etc.) — they are invalid MDX, +// 2. drop the static git-cliff header (everything before the first `## `), +// replacing it with frontmatter and a short intro, +// 3. backslash-escape `<`, `{` and `}` outside inline code spans and fenced +// code blocks so MDX doesn't parse commit subjects like `Result` +// or `clippy::{...}` as JSX. +// +// Usage from web/: +// bun --filter @nativelink/docs gen:changelog + +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const sourceFile = join(here, "../../../../CHANGELOG.md"); +const targetFile = join(here, "../content/docs/reference/changelog.mdx"); + +/** Escape MDX/JSX-significant characters in prose (text outside code). */ +const escapeProse = (text) => text.replace(/[<{}]/g, (c) => `\\${c}`); + +/** + * Escape one markdown line, leaving single-backtick inline code spans + * untouched. Odd split indices are the captured code spans. Double-backtick + * spans would be over-escaped, but git-cliff commit subjects don't use them + * and the failure mode is a stray backslash in rendered text, not a build + * break. + */ +const escapeLine = (line) => + line + .split(/(`[^`]*`)/) + .map((segment, i) => (i % 2 === 1 ? segment : escapeProse(segment))) + .join(""); + +const raw = readFileSync(sourceFile, "utf8").replace(//g, ""); + +// Keep everything from the first release heading on; the git-cliff header +// (title + "All notable changes..." line) is replaced by the intro below. +const firstHeading = raw.search(/^## /m); +if (firstHeading === -1) { + throw new Error(`no "## " release heading found in ${sourceFile}`); +} + +let inFence = false; +const body = raw + .slice(firstHeading) + .trimEnd() + .split("\n") + .map((line) => { + if (/^\s{0,3}(```|~~~)/.test(line)) { + inFence = !inFence; + return line; + } + return inFence ? line : escapeLine(line); + }) + .join("\n"); + +const page = `--- +title: Changelog +description: Notable changes per release. Latest first. +--- + +{/* Generated by scripts/gen-changelog.mjs — do not edit by hand. */} +{/* The source of truth is CHANGELOG.md at the repository root. */} + +This page mirrors +[\`CHANGELOG.md\`](https://github.com/TraceMachina/nativelink/blob/main/CHANGELOG.md), +the canonical changelog maintained with [git-cliff](https://git-cliff.org) +as part of each release. + +${body} +`; + +writeFileSync(targetFile, page); +console.log(`gen-changelog: wrote ${targetFile}`); diff --git a/web/turbo.json b/web/turbo.json index bb964c964..50a0fa595 100644 --- a/web/turbo.json +++ b/web/turbo.json @@ -1,6 +1,20 @@ { "$schema": "https://turbo.build/schema.json", "tasks": { + "@nativelink/docs#build": { + "dependsOn": [ + "^build" + ], + "inputs": [ + "$TURBO_DEFAULT$", + "$TURBO_ROOT$/../CHANGELOG.md" + ], + "outputs": [ + ".next/**", + "!.next/cache/**", + "dist/**" + ] + }, "build": { "dependsOn": [ "^build" From 00bf9a5d6575fe0ab9f1fe497f2c70866d95401d Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 3 Jul 2026 12:19:06 -0700 Subject: [PATCH 2/5] Fix incomplete multi-character sanitization in gen-changelog CodeQL flagged the single-pass strip of spans: crafted input like - y --> reconstructs ` etc.) — they are invalid MDX, -// 2. drop the static git-cliff header (everything before the first `## `), +// 1. drop the static git-cliff header (everything before the first `## `), // replacing it with frontmatter and a short intro, +// 2. drop whole-line HTML comment markers (`` etc.) — HTML +// comments are invalid MDX, // 3. backslash-escape `<`, `{` and `}` outside inline code spans and fenced // code blocks so MDX doesn't parse commit subjects like `Result` // or `clippy::{...}` as JSX. // +// Step 3 is also what makes step 2 safe to keep simple: comments are only +// ever dropped as whole lines, never spliced out of a line, so removal can't +// concatenate text into a new `/g, ""); +/** Whole-line HTML comment, like the git-cliff footer markers. */ +const isCommentLine = (line) => /^$/.test(line.trim()); + +const raw = readFileSync(sourceFile, "utf8"); // Keep everything from the first release heading on; the git-cliff header -// (title + "All notable changes..." line) is replaced by the intro below. +// (comment markers, title, "All notable changes..." line) is replaced by the +// intro below. const firstHeading = raw.search(/^## /m); if (firstHeading === -1) { throw new Error(`no "## " release heading found in ${sourceFile}`); @@ -56,16 +67,20 @@ if (firstHeading === -1) { let inFence = false; const body = raw .slice(firstHeading) - .trimEnd() .split("\n") .map((line) => { if (/^\s{0,3}(```|~~~)/.test(line)) { inFence = !inFence; return line; } - return inFence ? line : escapeLine(line); + if (inFence) { + return line; + } + return isCommentLine(line) ? null : escapeLine(line); }) - .join("\n"); + .filter((line) => line !== null) + .join("\n") + .trimEnd(); const page = `--- title: Changelog From 4b36817d925b4cf3704783d530534c093e1ff8af Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 3 Jul 2026 12:28:43 -0700 Subject: [PATCH 3/5] fix hallucination --- web/apps/docs/scripts/gen-changelog.mjs | 28 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/web/apps/docs/scripts/gen-changelog.mjs b/web/apps/docs/scripts/gen-changelog.mjs index 900442bc7..0a9d5430c 100644 --- a/web/apps/docs/scripts/gen-changelog.mjs +++ b/web/apps/docs/scripts/gen-changelog.mjs @@ -12,17 +12,16 @@ // The transform is deliberately small: // 1. drop the static git-cliff header (everything before the first `## `), // replacing it with frontmatter and a short intro, -// 2. drop whole-line HTML comment markers (`` etc.) — HTML -// comments are invalid MDX, +// 2. drop git-cliff's whole-line comment marker lines (HTML comments are +// invalid MDX) by exact string comparison against the known markers, // 3. backslash-escape `<`, `{` and `}` outside inline code spans and fenced // code blocks so MDX doesn't parse commit subjects like `Result` // or `clippy::{...}` as JSX. // -// Step 3 is also what makes step 2 safe to keep simple: comments are only -// ever dropped as whole lines, never spliced out of a line, so removal can't -// concatenate text into a new `$/.test(line.trim()); +/** + * The exact whole-line comment markers git-cliff wraps the changelog in + * (from the header/footer templates in /cliff.toml). Matching is by string + * equality, not a pattern, so there is nothing to circumvent; any other + * comment-like text is neutralized into escaped prose by escapeLine. + */ +const GIT_CLIFF_MARKER_LINES = new Set([ + "", + "", + "", +]); const raw = readFileSync(sourceFile, "utf8"); @@ -76,7 +84,7 @@ const body = raw if (inFence) { return line; } - return isCommentLine(line) ? null : escapeLine(line); + return GIT_CLIFF_MARKER_LINES.has(line.trim()) ? null : escapeLine(line); }) .filter((line) => line !== null) .join("\n") From 3c4b99f899a89d2b58fd08da855a85a7d43dce2b Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 3 Jul 2026 12:43:59 -0700 Subject: [PATCH 4/5] fix Vercel issues --- web/DEPLOYMENT.md | 8 +++ web/apps/docs/scripts/gen-changelog.mjs | 72 +++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/web/DEPLOYMENT.md b/web/DEPLOYMENT.md index c01ed1337..7edefc2e5 100644 --- a/web/DEPLOYMENT.md +++ b/web/DEPLOYMENT.md @@ -106,6 +106,14 @@ git-cliff maintains at release time — by page from the current changelog, and the web CI build on pull requests fails if a changelog edit stops compiling as MDX. +The script reads `CHANGELOG.md` from disk when the repository root is +available (local checkouts, GitHub CI, Vercel with **Include source files +outside of the Root Directory** enabled). Vercel's default build container +only mounts the workspace, so there the script instead fetches the file +from GitHub raw pinned to `VERCEL_GIT_COMMIT_SHA` — the exact commit being +deployed, so the bytes match a full checkout. If neither works the build +fails with instructions rather than publishing a stale page. + Because of this, don't configure an **Ignored Build Step** on the docs Vercel project that skips builds when `web/` is unchanged: a release merge touches only root files (`CHANGELOG.md`, version bumps), and skipping that diff --git a/web/apps/docs/scripts/gen-changelog.mjs b/web/apps/docs/scripts/gen-changelog.mjs index 0a9d5430c..5fe7a1d78 100644 --- a/web/apps/docs/scripts/gen-changelog.mjs +++ b/web/apps/docs/scripts/gen-changelog.mjs @@ -23,17 +23,78 @@ // outside code reaches the page as escaped `\<` prose, so no HTML comment or // tag can survive into the compiled MDX, however it is split across lines. // +// Source resolution: CHANGELOG.md sits at the nativelink repository root, +// which is ABOVE the web/ workspace. Local checkouts and GitHub CI have it on +// disk, so we walk up from this script until we find it (next to cliff.toml, +// so an unrelated CHANGELOG.md can't be picked up). Vercel only mounts the +// workspace into the build container, so there we fetch the file from GitHub +// raw, pinned to the exact commit being deployed via Vercel's built-in +// VERCEL_GIT_* variables — same bytes, no drift. +// // Usage from web/: // bun --filter @nativelink/docs gen:changelog -import { readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const here = dirname(fileURLToPath(import.meta.url)); -const sourceFile = join(here, "../../../../CHANGELOG.md"); const targetFile = join(here, "../content/docs/reference/changelog.mdx"); +/** + * Find CHANGELOG.md by walking up from this script towards the filesystem + * root. Requiring cliff.toml as a sibling pins the match to the nativelink + * repository root rather than any stray changelog on the way up. + */ +function findLocalChangelog() { + for (let dir = here; ; ) { + if (existsSync(join(dir, "cliff.toml")) && existsSync(join(dir, "CHANGELOG.md"))) { + return join(dir, "CHANGELOG.md"); + } + const parent = resolve(dir, ".."); + if (parent === dir) { + return null; + } + dir = parent; + } +} + +/** Fetch CHANGELOG.md at the exact deployed commit from GitHub raw. */ +async function fetchChangelogForDeployedCommit() { + const owner = process.env.VERCEL_GIT_REPO_OWNER; + const repo = process.env.VERCEL_GIT_REPO_SLUG; + const sha = process.env.VERCEL_GIT_COMMIT_SHA; + if (!owner || !repo || !sha) { + return null; + } + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${sha}/CHANGELOG.md`; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`gen-changelog: fetching ${url} failed: HTTP ${response.status}`); + } + console.log(`gen-changelog: read CHANGELOG.md from ${url}`); + return response.text(); +} + +async function readChangelog() { + const localFile = findLocalChangelog(); + if (localFile) { + return { text: readFileSync(localFile, "utf8"), origin: localFile }; + } + const fetched = await fetchChangelogForDeployedCommit(); + if (fetched !== null) { + return { text: fetched, origin: "GitHub raw (deployed commit)" }; + } + throw new Error( + [ + `gen-changelog: CHANGELOG.md not found next to a cliff.toml in any directory above ${here},`, + "and the VERCEL_GIT_* variables needed to fetch it for the deployed commit are unset.", + "Run from a nativelink checkout, or on Vercel enable 'Include source", + "files outside of the Root Directory' so the repository root is available to the build.", + ].join(" "), + ); +} + /** Escape MDX/JSX-significant characters in prose (text outside code). */ const escapeProse = (text) => text.replace(/[<{}]/g, (c) => `\\${c}`); @@ -62,14 +123,14 @@ const GIT_CLIFF_MARKER_LINES = new Set([ "", ]); -const raw = readFileSync(sourceFile, "utf8"); +const { text: raw, origin } = await readChangelog(); // Keep everything from the first release heading on; the git-cliff header // (comment markers, title, "All notable changes..." line) is replaced by the // intro below. const firstHeading = raw.search(/^## /m); if (firstHeading === -1) { - throw new Error(`no "## " release heading found in ${sourceFile}`); + throw new Error(`no "## " release heading found in ${origin}`); } let inFence = false; @@ -106,5 +167,6 @@ as part of each release. ${body} `; +mkdirSync(dirname(targetFile), { recursive: true }); writeFileSync(targetFile, page); console.log(`gen-changelog: wrote ${targetFile}`); From 5e4336d07a65dfab57ec4bc9da44c1e014c49488 Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 3 Jul 2026 16:09:28 -0700 Subject: [PATCH 5/5] Address Bex/clanker comments --- web/DEPLOYMENT.md | 4 ++- web/apps/docs/.gitignore | 2 +- web/apps/docs/scripts/gen-changelog.mjs | 39 ++++++++++++++++--------- web/turbo.json | 3 ++ 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/web/DEPLOYMENT.md b/web/DEPLOYMENT.md index 7edefc2e5..512df6f04 100644 --- a/web/DEPLOYMENT.md +++ b/web/DEPLOYMENT.md @@ -104,7 +104,9 @@ git-cliff maintains at release time — by `apps/docs/scripts/gen-changelog.mjs`, which runs at the start of the docs `dev` and `build` scripts. Every merge to main therefore republishes the page from the current changelog, and the web CI build on pull requests -fails if a changelog edit stops compiling as MDX. +fails if a changelog edit stops compiling. The page is emitted as plain +markdown (`changelog.md`, not `.mdx`) so JSX- and import-like text in +commit subjects is inert rather than syntax. The script reads `CHANGELOG.md` from disk when the repository root is available (local checkouts, GitHub CI, Vercel with **Include source files diff --git a/web/apps/docs/.gitignore b/web/apps/docs/.gitignore index 7317f8473..02bf83b1b 100644 --- a/web/apps/docs/.gitignore +++ b/web/apps/docs/.gitignore @@ -7,4 +7,4 @@ next-env.d.ts # Generated from the repository-root CHANGELOG.md by scripts/gen-changelog.mjs # (runs as part of `dev` and `build`). -content/docs/reference/changelog.mdx +content/docs/reference/changelog.md diff --git a/web/apps/docs/scripts/gen-changelog.mjs b/web/apps/docs/scripts/gen-changelog.mjs index 5fe7a1d78..ee373e68b 100644 --- a/web/apps/docs/scripts/gen-changelog.mjs +++ b/web/apps/docs/scripts/gen-changelog.mjs @@ -3,25 +3,30 @@ // CHANGELOG.md (maintained with git-cliff at release time; see /cliff.toml and // the "Creating a Release" section of CONTRIBUTING.md). // -// The output at content/docs/reference/changelog.mdx is gitignored and +// The output at content/docs/reference/changelog.md is gitignored and // regenerated as part of the `dev` and `build` scripts, so the published page // always mirrors the CHANGELOG.md of the commit being deployed: every merge to // main produces a fresh page via the production build, and the web CI build on -// pull requests fails if a changelog edit stops compiling as MDX. +// pull requests fails if a changelog edit stops compiling. // -// The transform is deliberately small: +// The page is emitted as plain markdown (.md), which fumadocs compiles +// without JSX or ESM support: `{`, `}`, and statement-like lines such as +// `import x from "y"` in commit subjects are inert text there, while in .mdx +// each of them is syntax that can break the compile. The transform: // 1. drop the static git-cliff header (everything before the first `## `), // replacing it with frontmatter and a short intro, -// 2. drop git-cliff's whole-line comment marker lines (HTML comments are -// invalid MDX) by exact string comparison against the known markers, -// 3. backslash-escape `<`, `{` and `}` outside inline code spans and fenced -// code blocks so MDX doesn't parse commit subjects like `Result` -// or `clippy::{...}` as JSX. +// 2. drop git-cliff's whole-line comment marker lines by exact string +// comparison against the known markers (step 3 would otherwise escape +// them into visible text), +// 3. backslash-escape `\` and `<` outside inline code spans and fenced +// code blocks, so commit subjects can't smuggle raw HTML like +// `