diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index d6be430af..5c4368937 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -8,9 +8,22 @@ # push to main -> affected ports only (scripts/ci-affected-ports.sh) # nightly + dispatch -> full matrix # -# Measured 2026-07-03 (full tier, all ports in parallel, warm caches): +# Java is split into two parallel jobs — java-fast (java+kotlin conformance, the +# quick correctness signal ~2-3m) and java-slow (full reactor + docker +# integration). The split surfaces the conformance verdict fast and means a +# cancel/OOM mid-integration no longer kills the fast signal. `detect` emits the +# umbrella token `java`; both jobs gate on it. +# +# Integration-running jobs (ts, python, csharp, java-slow) share a Postgres +# SIDECAR instead of each test booting its own Testcontainers Postgres per +# scenario (the java+kotlin lanes alone booted ~30-40 serial containers). Each +# port's PG helper honors METAOBJECTS_TEST_PG_URL and CREATEs a uniquely-named +# database per scenario against the sidecar. The sidecar publishes 5432 to a +# DYNAMIC host port (`ports: - 5432`, read back via job.services.*.ports) so +# concurrent jobs on the shared self-hosted host never collide on a fixed port. +# +# Measured 2026-07-03 (pre-split, full tier, all ports in parallel, warm caches): # detect 0.2m · gates 0.1m · csharp 2m · python 2.6m · ts 5.3m · java 9.8m -# -> full-matrix wall-clock ~10m (slowest port); single-port pushes ~2-6m. name: local-ci on: push: @@ -69,47 +82,126 @@ jobs: if: contains(needs.detect.outputs.ports, 'ts') runs-on: [self-hosted, linux] timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: metaobjects + POSTGRES_PASSWORD: metaobjects + POSTGRES_DB: metaobjects_test + ports: + - 5432 # dynamic host port — no fixed-port collision + options: >- + --health-cmd "pg_isready -U metaobjects" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 with: persist-credentials: false - - run: scripts/ci-local.sh --only ts --strict-toolchains + - env: + METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test + run: scripts/ci-local.sh --only ts --strict-toolchains - java: + # Java FAST lane — java + kotlin conformance only (the quick correctness signal). + # No docker/integration, so no sidecar. + java-fast: needs: detect if: contains(needs.detect.outputs.ports, 'java') runs-on: [self-hosted, linux] - timeout-minutes: 60 + timeout-minutes: 25 steps: - uses: actions/checkout@v4 with: persist-credentials: false - - run: scripts/ci-local.sh --only java --strict-toolchains + - run: scripts/ci-local.sh --only java-fast --strict-toolchains + + # Java SLOW lane — full reactor + docker integration (java + kotlin). Uses the + # shared Postgres sidecar so integration stops booting per-scenario containers. + java-slow: + needs: detect + if: contains(needs.detect.outputs.ports, 'java') + runs-on: [self-hosted, linux] + timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: metaobjects + POSTGRES_PASSWORD: metaobjects + POSTGRES_DB: metaobjects_test + ports: + - 5432 + options: >- + --health-cmd "pg_isready -U metaobjects" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - env: + METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test + run: scripts/ci-local.sh --only java-slow --strict-toolchains python: needs: detect if: contains(needs.detect.outputs.ports, 'python') runs-on: [self-hosted, linux] timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: metaobjects + POSTGRES_PASSWORD: metaobjects + POSTGRES_DB: metaobjects_test + ports: + - 5432 + options: >- + --health-cmd "pg_isready -U metaobjects" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 with: persist-credentials: false - - run: scripts/ci-local.sh --only python --strict-toolchains + - env: + METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test + run: scripts/ci-local.sh --only python --strict-toolchains csharp: needs: detect if: contains(needs.detect.outputs.ports, 'csharp') runs-on: [self-hosted, linux] timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: metaobjects + POSTGRES_PASSWORD: metaobjects + POSTGRES_DB: metaobjects_test + ports: + - 5432 + options: >- + --health-cmd "pg_isready -U metaobjects" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 with: persist-credentials: false - - run: scripts/ci-local.sh --only csharp --strict-toolchains + - env: + METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test + run: scripts/ci-local.sh --only csharp --strict-toolchains notify: - needs: [detect, gates, ts, java, python, csharp] + needs: [detect, gates, ts, java-fast, java-slow, python, csharp] if: always() && contains(needs.*.result, 'failure') runs-on: [self-hosted, linux] steps: diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 1ff6a08ad..e82eed111 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -28,6 +28,11 @@ # # completeness-gate, integration-tests ts # # java → java+kotlin conformance, reactor, # # integration-tests java+kotlin +# # java-fast → java+kotlin conformance only +# # (the quick correctness signal) +# # java-slow → reactor + integration-tests +# # (java+kotlin); CI runs the two +# # lanes as separate parallel jobs # # python → python conformance + integration-tests # # csharp → csharp conformance + integration-tests # scripts/ci-local.sh --strict-toolchains @@ -59,8 +64,8 @@ while [ $# -gt 0 ]; do --quick) QUICK=1 ;; --strict-toolchains) STRICT=1 ;; --only) shift; case "${1:-}" in - gates|ts|java|python|csharp) ONLY="$ONLY ${1}" ;; - *) echo "--only expects gates|ts|java|python|csharp, got '${1:-}'" >&2; exit 2 ;; + gates|ts|java|java-fast|java-slow|python|csharp) ONLY="$ONLY ${1}" ;; + *) echo "--only expects gates|ts|java|java-fast|java-slow|python|csharp, got '${1:-}'" >&2; exit 2 ;; esac ;; -h|--help) awk 'NR==1{next} /^set -uo/{exit} {sub(/^# ?/,""); print}' "$0"; exit 0 ;; *) echo "unknown arg: $1 (see --help)" >&2; exit 2 ;; @@ -73,6 +78,9 @@ want() { # want
— true when the section should run [ -z "$ONLY" ] && return 0 case " $ONLY " in *" $1 "*) return 0 ;; *) return 1 ;; esac } +want_any() { # want_any — true when ANY listed section should run + local s; for s in "$@"; do want "$s" && return 0; done; return 1 +} PASS=(); FAIL=(); SKIP=() have() { command -v "$1" >/dev/null 2>&1; } @@ -158,7 +166,11 @@ gate_conf_kotlin() { } # ── conformance.yml — full reactor + drift/mutation gates ───────────────────── -gate_java_reactor() { ( cd server/java && mvn -q clean install ); } +# JaCoCo is skipped here: its coverage gate is haltOnFailure=false (pom.xml), so it +# never fails the build — it is pure instrumentation overhead for a pass/fail CI +# signal. `clean` stays: self-hosted workspaces persist and actions/checkout wipes +# only git-tracked state, so a stale target/ can otherwise leak between runs. +gate_java_reactor() { ( cd server/java && mvn -q clean install -Djacoco.skip=true ); } gate_completeness() { ( cd server/typescript/packages/metadata && bun run conformance:mutation ); } gate_doc_template_drift() { bash scripts/sync-doc-templates.sh \ @@ -230,13 +242,19 @@ if [ "$QUICK" -eq 1 ]; then SKIP+=("integration-tests (--quick)") else # Heavy tier — other-language ports + full reactor + docker integration suite. - if want csharp; then step_if dotnet "conformance: csharp" gate_conf_csharp; fi - if want java; then step_if mvn "conformance: java" gate_conf_java; fi - if want python; then step_if uv "conformance: python" gate_conf_python; fi - if want java; then - step_if mvn "conformance: kotlin" gate_conf_kotlin - step_if mvn "java-reactor (clean install)" gate_java_reactor - fi + # + # The Java port is split into two lanes so CI can run them as separate jobs + # (see .github/workflows/local-ci.yml): a FAST lane (java+kotlin conformance, + # ~2-3m — the correctness signal) and a SLOW lane (full reactor + docker + # integration). Running them as parallel jobs surfaces the conformance verdict + # quickly and means a cancel/OOM mid-integration no longer kills the fast + # signal. `--only java` (local, unqualified) still runs BOTH lanes, in order, + # so a plain local run is byte-equivalent to before. + if want csharp; then step_if dotnet "conformance: csharp" gate_conf_csharp; fi + if want_any java java-fast; then step_if mvn "conformance: java" gate_conf_java; fi + if want python; then step_if uv "conformance: python" gate_conf_python; fi + if want_any java java-fast; then step_if mvn "conformance: kotlin" gate_conf_kotlin; fi + if want_any java java-slow; then step_if mvn "java-reactor (install)" gate_java_reactor; fi # Docker integration — full suite when no --only, per-port otherwise. if [ -z "$ONLY" ]; then if docker info >/dev/null 2>&1; then @@ -246,10 +264,10 @@ else FAIL+=("integration-tests (docker down)") fi else - want ts && run_integration_for ts ts - want java && run_integration_for java java kotlin - want python && run_integration_for python python - want csharp && run_integration_for csharp csharp + want ts && run_integration_for ts ts + want_any java java-slow && run_integration_for java java kotlin + want python && run_integration_for python python + want csharp && run_integration_for csharp csharp fi fi