diff --git a/.githooks/pre-push b/.githooks/pre-push index 2f3125b72..32ac80330 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -19,6 +19,21 @@ # set -uo pipefail +# ── Always-run, toolchain-free guard (BEFORE the TS-skip below — unrelated gate). +# Reactor-excluded Java/Kotlin integration-test poms must track the reactor +# . They are outside the reactor, so a release `versions:set` skips them; +# the drift only surfaces as a red `release-gate (java|kotlin)` ("Non-resolvable +# parent POM") on the next tag. Catch it here — it's just grep, no JVM/Maven. +_HOOK_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" +if [ -x "$_HOOK_ROOT/scripts/check-pom-versions.sh" ]; then + if ! "$_HOOK_ROOT/scripts/check-pom-versions.sh" >/tmp/metaobjects-prepush-pom.log 2>&1; then + echo "" >&2 + echo " ✖ pre-push BLOCKED: Java pom version drift —" >&2 + sed 's/^/ /' /tmp/metaobjects-prepush-pom.log >&2 + exit 1 + fi +fi + if [ "${SKIP_TS_TYPECHECK:-}" = "1" ]; then echo "pre-push: SKIP_TS_TYPECHECK=1 set — skipping the TS typecheck gate (you asked for it)." >&2 exit 0 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f1282e6b5..57a3ae95f 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,5 +1,17 @@ name: conformance -on: [push, pull_request] + +# COST: the cross-port conformance matrix + full Java reactor + drift/mutation gates +# are heavy, so they NO LONGER run on every push/PR — release gate (tag `v*`) + +# on-demand (workflow_dispatch) only. Run them LOCALLY before opening/merging a PR: +# scripts/ci-local.sh # full parity (this + integration suite + drift) +# scripts/ci-local.sh --quick # everything except the docker integration suite +# The .githooks/pre-push hook also runs the TS build+typecheck gate locally, and the +# cheap public-repo SECURITY gate (hygiene / leak-scan) still runs on every PR. +on: + push: + tags: + - 'v*' + workflow_dispatch: jobs: fixture-lint: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8e34f995a..2b645d203 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -5,23 +5,19 @@ name: integration-tests # `bun test` path — against a real Postgres for all five ports. Per-language jobs # run in parallel; any port red blocks the gate. # -# Triggers: -# - pull_request (any target branch) — gates every PR so persistence/api-contract -# drift can't merge green (these corpora are not covered by conformance.yml). -# - push to `main` — guards the default branch directly (e.g. forward-merges that -# land on main without going through a PR). -# - tag push matching v* (release gate, per docs/RELEASING.md). +# COST: this 5-port Testcontainers matrix is EXPENSIVE, so it does NOT run on every +# push/PR. Triggers: +# - tag push matching v* (the release gate, per docs/RELEASING.md). # - manual dispatch (workflow_dispatch). -# NOTE: this runs a 5-port Testcontainers matrix on every PR; if CI cost is too -# high, narrow `pull_request.branches` or re-introduce an opt-in label. +# Day-to-day, run these corpora LOCALLY before opening/merging a PR: +# scripts/ci-local.sh # full local CI (this suite + conformance + drift) +# scripts/integration-test.sh +# The cheap public-repo SECURITY gate (hygiene / leak-scan) still runs on every PR. on: push: - branches: - - main tags: - 'v*' - pull_request: workflow_dispatch: jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e31833d0e..45042e5ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,9 +24,15 @@ conformance corpora. This guide covers how to propose changes. 1. **Fork** the repo and create a branch from `main`. 2. Make your change following the discipline below — **tests first (TDD)**. -3. Run the relevant test suite locally; make sure it's green. +3. Run the gates locally with **`scripts/ci-local.sh`** (or `--quick` to skip the + docker integration suite) and make sure it's green. **This matters:** to keep CI + costs down, the heavy correctness gates (the cross-port conformance matrix, the + full Java reactor, the drift/mutation gates, and the integration-tests + testcontainers matrix) **no longer run automatically on PRs** — they are release + gate (`v*`) + on-demand only. Local CI is now the primary correctness gate. 4. Open a **pull request** against `main` and fill out the PR template. -5. CI must pass and a maintainer must approve before merge. +5. The public-repo `leak-scan` gate still runs on every PR; a maintainer runs the + full gates (locally or via the on-demand workflow) and approves before merge. ## The cardinal rule: the metamodel is the spine @@ -67,7 +73,24 @@ cd server/python && pytest # Python (in its .venv) ``` Cross-language persistence / api-contract corpora (Docker + Testcontainers) run via -`scripts/integration-test.sh` and in CI. +`scripts/integration-test.sh`. + +### Local CI (run this before opening/merging a PR) + +Because the heavy gates no longer auto-run on PRs (cost), reproduce the full CI +locally with one command: + +```bash +scripts/ci-local.sh # full parity: leak-scan, all-port conformance, + # java reactor, drift/mutation gates, + docker + # integration suite +scripts/ci-local.sh --quick # everything except the docker integration suite +``` + +It SKIPS (loudly, not silently) any port whose toolchain (bun/dotnet/uv/mvn) isn't +installed. The `.githooks/pre-push` hook additionally runs the TS build+typecheck +gate and a Java pom-version drift guard on every push (activate hooks once per +clone: `git config core.hooksPath .githooks`). ## Releasing diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 9ba887342..97b1b332b 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -316,7 +316,18 @@ maintainer's GPG key. ## Procedure -1. **Bump** the version in all poms (parent + modules): `grep -rl 7.4.0 --include=pom.xml server/java | xargs sed -i 's/7\.4\.0/7.4.1/g'` (verify every `7.4.0` is the project version, not a third-party dep). +1. **Bump** the version in **all** poms — parent + reactor modules **and the two + reactor-EXCLUDED integration-test modules** (`server/java/integration-tests/pom.xml`, + `server/java/integration-tests-kotlin/pom.xml`). Use the tree-wide `grep`, NOT + `mvn versions:set`: `versions:set` only walks the reactor and silently leaves the + excluded modules behind, so their `` lags and the next tag fails + `release-gate (java|kotlin)` with "Non-resolvable parent POM". + ```bash + grep -rl 7.4.0 --include=pom.xml server/java | xargs sed -i 's/7\.4\.0/7.4.1/g' + ``` + (Verify every `7.4.0` is the project version, not a third-party dep.) + Then assert the excluded modules are in sync: `scripts/check-pom-versions.sh` + (also enforced on every push by `.githooks/pre-push` and by `scripts/ci-local.sh`). 2. **Validate locally:** `cd server/java && mvn -q clean install -DskipTests` (or with tests / `scripts/integration-test.sh java` if runtime changed). 3. **Deploy:** `mvn -Prelease deploy` from `server/java`. The `central-publishing-maven-plugin` (`central`, `true`) uploads diff --git a/scripts/check-pom-versions.sh b/scripts/check-pom-versions.sh new file mode 100755 index 000000000..1afe095b7 --- /dev/null +++ b/scripts/check-pom-versions.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Guard: the reactor-EXCLUDED Java/Kotlin integration-test modules must carry the +# SAME parent as the reactor root (server/java/pom.xml). +# +# Why this exists: integration-tests + integration-tests-kotlin are intentionally +# NOT listed in the parent reactor's (they need docker; `mvn test` stays +# docker-free — see scripts/integration-test.sh). Because they are outside the +# reactor, `mvn versions:set` during a release bumps every reactor module but +# SILENTLY SKIPS these two. Their then lags (e.g. reactor at +# 7.5.1-SNAPSHOT, these stuck at 7.4.4-SNAPSHOT), so `../pom.xml` no longer matches +# the declared parent → Maven rejects relativePath, falls back to repos, the +# SNAPSHOT isn't there, and `release-gate (java|kotlin)` fails with +# "Non-resolvable parent POM". This guard turns that latent drift into a loud, +# instant, toolchain-free failure (just grep — no JVM/Maven needed). +# +# Usage: scripts/check-pom-versions.sh # exit 1 + diagnostic on drift +# +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +REACTOR_POM="$ROOT/server/java/pom.xml" + +# The reactor root's own is the first element in its pom (it has +# no , so this is unambiguous). +reactor_version="$(grep -m1 -oE '[^<]+' "$REACTOR_POM" | sed -E 's###g')" +if [ -z "$reactor_version" ]; then + echo "check-pom-versions: could not read reactor version from $REACTOR_POM" >&2 + exit 2 +fi + +EXCLUDED_POMS=( + "server/java/integration-tests/pom.xml" + "server/java/integration-tests-kotlin/pom.xml" +) + +fail=0 +for rel in "${EXCLUDED_POMS[@]}"; do + pom="$ROOT/$rel" + [ -f "$pom" ] || { echo "check-pom-versions: missing $rel" >&2; fail=1; continue; } + # Parent version is the first inside the block. + parent_version="$(awk '//,/<\/parent>/' "$pom" | grep -m1 -oE '[^<]+' | sed -E 's###g')" + if [ "$parent_version" != "$reactor_version" ]; then + echo " ✖ $rel parent is '$parent_version' but the reactor is '$reactor_version'." >&2 + fail=1 + fi +done + +if [ "$fail" -ne 0 ]; then + echo "" >&2 + echo "check-pom-versions: reactor-excluded integration-test module(s) drifted from the" >&2 + echo "reactor version. A version bump must update these too (they are outside the" >&2 + echo "reactor, so 'mvn versions:set' skips them). Fix — set their parent to" >&2 + echo "the reactor version ('$reactor_version') in:" >&2 + echo " server/java/integration-tests/pom.xml" >&2 + echo " server/java/integration-tests-kotlin/pom.xml" >&2 + exit 1 +fi + +echo "check-pom-versions: ✓ excluded integration-test modules match reactor $reactor_version" +exit 0 diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh new file mode 100755 index 000000000..38732ab6b --- /dev/null +++ b/scripts/ci-local.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# +# Local CI — run the same gates as .github/workflows/ on your machine. +# +# As of the cost-reduction change, the heavy correctness gates (the 5-port +# conformance matrix, the full Java reactor, the drift/mutation gates, and the +# whole integration-tests testcontainers matrix) NO LONGER run automatically on +# every push/PR — they are `workflow_dispatch` + release-tag (`v*`) only. This +# script is how you run them locally before opening/merging a PR, so nothing red +# leaves your machine. The cheap public-repo SECURITY backstop (leak-scan) still +# runs automatically in CI on every PR; it is included here too. +# +# Usage: +# scripts/ci-local.sh # FULL parity: all-port conformance + full Java +# # reactor + drift/mutation gates + docker +# # integration suite. Run before a PR or tag. +# scripts/ci-local.sh --quick # FAST inner-loop tier: the cheap TS-centric +# # gates only (leak-scan, pom parity, fixture- +# # lint, ts build+typecheck, ts conformance, +# # completeness, drift). Skips the other- +# # language ports, the reactor, and docker. +# scripts/ci-local.sh --help +# +# Ports whose toolchain (bun / dotnet / uv / mvn) is not installed are SKIPPED +# with a loud warning (not silently passed). A full pre-release run is expected on +# a machine with every toolchain — otherwise rely on the release-tag CI run. +# +# Mirrors: hygiene.yml (leak-scan) · conformance.yml (fixture-lint, typecheck, +# 5-port conformance, kotlin, java-reactor, completeness-gate, doc-template-drift, +# embedded-library-drift) · integration-tests.yml (5-port suite + migrate-ts-pg). +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +QUICK=0 +for arg in "$@"; do + case "$arg" in + --quick) QUICK=1 ;; + -h|--help) sed -n '2,27p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown arg: $arg (see --help)" >&2; exit 2 ;; + esac +done + +PASS=(); FAIL=(); SKIP=() +have() { command -v "$1" >/dev/null 2>&1; } + +step() { # step "" + local name="$1"; shift + echo "" + echo "── ▶ $name ──────────────────────────────────────────────" + if "$@"; then PASS+=("$name"); else FAIL+=("$name"); echo " ✖ $name FAILED" >&2; fi +} +step_if() { # step_if "" — SKIP (not fail) when tool absent + local tool="$1"; local name="$2"; shift 2 + if have "$tool"; then step "$name" "$@"; else + echo ""; echo "── ⊘ $name — SKIPPED ('$tool' not installed) ──" >&2 + SKIP+=("$name (no $tool)") + fi +} + +# ── hygiene.yml: public-repo leak scan (the SECURITY gate kept in CI) ────────── +gate_leak_scan() { + local base="origin/main" + git rev-parse --verify -q "$base" >/dev/null 2>&1 || base="HEAD~1" + bash .githooks/leak-scan.sh "$base" +} + +# ── release hygiene: reactor-excluded pom version parity (drift guard) ───────── +gate_pom_versions() { scripts/check-pom-versions.sh; } + +# ── conformance.yml — fixture lint + workspace typecheck ────────────────────── +gate_fixture_lint() { + bun install && ( cd server/typescript/packages/conformance && bun bin/conformance.ts lint ../../../../fixtures/conformance ) +} +gate_ts_build_typecheck() { bun run --filter '*' build && bun run --filter '*' typecheck; } + +# ── conformance.yml — per-port conformance corpora (exact CI commands) ──────── +gate_conf_ts() { + bun install || return 1 + ( cd server/typescript/packages/metadata && bun test test/conformance.test.ts test/yaml-conformance.test.ts test/object-model-conformance.test.ts \ + && bun test test/registry-conformance.test.ts test/registry-coverage.test.ts ) || return 1 + ( cd server/typescript/packages/render && bun test test/render-conformance.test.ts test/verify-conformance.test.ts test/extract/extract-conformance.test.ts test/output-prompt-conformance.test.ts ) || return 1 + ( cd server/typescript/packages/integration-tests && bun test test/validation-conformance.test.ts ) || return 1 + ( cd server/typescript/packages/migrate-ts && bun test ) || return 1 + ( cd server/typescript/packages/codegen-ts && bun test ) || return 1 + ( cd server/typescript/packages/cli && bun test ) || return 1 +} +gate_conf_csharp() { + ( cd server/csharp \ + && dotnet test MetaObjects.Conformance.Tests/MetaObjects.Conformance.Tests.csproj --nologo --verbosity quiet \ + && dotnet test MetaObjects.Render.Tests/MetaObjects.Render.Tests.csproj --nologo --verbosity quiet \ + && dotnet test MetaObjects.Codegen.Tests/MetaObjects.Codegen.Tests.csproj --filter "FullyQualifiedName~ValidationConformance" --nologo --verbosity quiet \ + && dotnet test MetaObjects.Codegen.Tests/MetaObjects.Codegen.Tests.csproj --filter "FullyQualifiedName~GeneratorRegistryConformance" --nologo --verbosity quiet \ + && dotnet test MetaObjects.Cli.Tests/MetaObjects.Cli.Tests.csproj --nologo --verbosity quiet ) +} +gate_conf_java() { + ( cd server/java \ + && mvn -pl metadata,render,codegen-spring -am install -DskipTests -q \ + && mvn -pl metadata test -Dtest='ConformanceTest,YamlConformanceTest,ObjectModelConformanceTest,RegistryManifestConformanceTest' -q \ + && mvn -pl render test -Dtest='RenderCrossPortReportTest,VerifyConformanceTest,ExtractConformanceTest,OutputPromptConformanceTest' -q \ + && mvn -pl codegen-spring test -Dtest='ValidationConformanceTest,GeneratorRegistryConformanceTest' -q ) +} +gate_conf_python() { + ( cd server/python \ + && uv run pytest tests/conformance -q \ + && uv run pytest tests/render -q \ + && uv run pytest tests/codegen/test_validation_conformance.py -q \ + && uv run pytest tests/codegen/test_cli_registry.py -q ) +} +gate_conf_kotlin() { + ( cd server/java \ + && mvn -pl codegen-kotlin -am install -DskipTests -q \ + && mvn -pl codegen-kotlin test -Dtest='ObjectModelConformanceTest,OutputPromptConformanceTest,ValidationConformanceTest,GeneratorRegistryConformanceTest,RegistryManifestConformanceTest' -q ) +} + +# ── conformance.yml — full reactor + drift/mutation gates ───────────────────── +gate_java_reactor() { ( cd server/java && mvn -q clean install ); } +gate_completeness() { ( cd server/typescript/packages/metadata && bun run conformance:mutation ); } +gate_doc_template_drift() { + bash scripts/sync-doc-templates.sh \ + && git diff --exit-code -- \ + server/typescript/packages/codegen-ts/templates \ + server/typescript/packages/codegen-ts/src/render-engine/embedded-templates.generated.ts +} +gate_embedded_library_drift() { + bun run scripts/generate-embedded-library.ts \ + && git diff --exit-code -- server/typescript/packages/metadata/src/library/embedded-library.generated.ts +} + +# ── integration-tests.yml (docker) ──────────────────────────────────────────── +gate_integration() { scripts/integration-test.sh all; } + +echo "metaobjects local CI (mode: $([ "$QUICK" = 1 ] && echo 'quick — TS + drift gates only' || echo 'full — all ports + reactor + docker'))" + +# Fast tier — cheap, TS-centric, single-toolchain (bun). Runs in both modes; this is +# the whole of --quick. (The pre-push hook covers ts build+typecheck on its own.) +step "leak-scan (security)" gate_leak_scan +step "pom-version parity" gate_pom_versions +step_if bun "fixture-lint" gate_fixture_lint +step_if bun "ts build + typecheck" gate_ts_build_typecheck +step_if bun "conformance: typescript" gate_conf_ts +step_if bun "completeness-gate (mutation)" gate_completeness +step_if bun "doc-template drift" gate_doc_template_drift +step_if bun "embedded-library drift" gate_embedded_library_drift + +if [ "$QUICK" -eq 1 ]; then + echo "" + echo "── ⊘ --quick: SKIPPING the other-language ports (csharp/java/python/kotlin)," + echo " the full Java reactor, and the docker integration suite. Run the full" + echo " \`scripts/ci-local.sh\` (no flag) before opening/merging a PR or tagging." + SKIP+=("csharp/java/python/kotlin conformance (--quick)") + SKIP+=("java-reactor (--quick)") + SKIP+=("integration-tests (--quick)") +else + # Heavy tier — other-language ports + full reactor + docker integration suite. + step_if dotnet "conformance: csharp" gate_conf_csharp + step_if mvn "conformance: java" gate_conf_java + step_if uv "conformance: python" gate_conf_python + step_if mvn "conformance: kotlin" gate_conf_kotlin + step_if mvn "java-reactor (clean install)" gate_java_reactor + if docker info >/dev/null 2>&1; then + step "integration-tests (5-port + docker)" gate_integration + else + echo ""; echo " ✖ docker is DOWN — cannot run the integration suite. Start docker, or use --quick." >&2 + FAIL+=("integration-tests (docker down)") + fi +fi + +echo "" +echo "══════════════════════ SUMMARY ══════════════════════" +for p in "${PASS[@]:-}"; do [ -n "${p:-}" ] && echo " ✓ $p"; done +for s in "${SKIP[@]:-}"; do [ -n "${s:-}" ] && echo " ⊘ $s"; done +for f in "${FAIL[@]:-}"; do [ -n "${f:-}" ] && echo " ✖ $f"; done +echo "══════════════════════════════════════════════════════" + +if [ "${#FAIL[@]}" -gt 0 ]; then echo "LOCAL CI FAILED — ${#FAIL[@]} gate(s) red." >&2; exit 1; fi +if [ "${#SKIP[@]}" -gt 0 ]; then echo "LOCAL CI PASSED (with ${#SKIP[@]} skipped — see ⊘ above)."; exit 0; fi +echo "LOCAL CI PASSED"