Skip to content

ci(java): split java lane into fast/slow + shared PG sidecar (~3× faster, crash-resistant) #162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dmealing merged 1 commit into from
Jul 4, 2026
Merged

ci(java): split java lane into fast/slow + shared PG sidecar (~3× faster, crash-resistant) #162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 101 additions & 9 deletions .github/workflows/local-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,22 @@
# push to main -> affected ports only (scripts/ci-affected-ports.sh)
# nightly + dispatch -> full matrix
#
# Measured 2026-07-03 (full tier, all ports in parallel, warm caches):
# Java is split into two parallel jobs — java-fast (java+kotlin conformance, the
# quick correctness signal ~2-3m) and java-slow (full reactor + docker
# integration). The split surfaces the conformance verdict fast and means a
# cancel/OOM mid-integration no longer kills the fast signal. `detect` emits the
# umbrella token `java`; both jobs gate on it.
#
# Integration-running jobs (ts, python, csharp, java-slow) share a Postgres
# SIDECAR instead of each test booting its own Testcontainers Postgres per
# scenario (the java+kotlin lanes alone booted ~30-40 serial containers). Each
# port's PG helper honors METAOBJECTS_TEST_PG_URL and CREATEs a uniquely-named
# database per scenario against the sidecar. The sidecar publishes 5432 to a
# DYNAMIC host port (`ports: - 5432`, read back via job.services.*.ports) so
# concurrent jobs on the shared self-hosted host never collide on a fixed port.
#
# Measured 2026-07-03 (pre-split, full tier, all ports in parallel, warm caches):
# detect 0.2m · gates 0.1m · csharp 2m · python 2.6m · ts 5.3m · java 9.8m
# -> full-matrix wall-clock ~10m (slowest port); single-port pushes ~2-6m.
name: local-ci
on:
push:
Expand Down Expand Up @@ -69,47 +82,126 @@ jobs:
if: contains(needs.detect.outputs.ports, 'ts')
runs-on: [self-hosted, linux]
timeout-minutes: 45
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: metaobjects
POSTGRES_PASSWORD: metaobjects
POSTGRES_DB: metaobjects_test
ports:
- 5432 # dynamic host port — no fixed-port collision
options: >-
--health-cmd "pg_isready -U metaobjects"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- run: scripts/ci-local.sh --only ts --strict-toolchains
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only ts --strict-toolchains

java:
# Java FAST lane — java + kotlin conformance only (the quick correctness signal).
# No docker/integration, so no sidecar.
java-fast:
needs: detect
if: contains(needs.detect.outputs.ports, 'java')
runs-on: [self-hosted, linux]
timeout-minutes: 60
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- run: scripts/ci-local.sh --only java --strict-toolchains
- run: scripts/ci-local.sh --only java-fast --strict-toolchains

# Java SLOW lane — full reactor + docker integration (java + kotlin). Uses the
# shared Postgres sidecar so integration stops booting per-scenario containers.
java-slow:
needs: detect
if: contains(needs.detect.outputs.ports, 'java')
runs-on: [self-hosted, linux]
timeout-minutes: 45
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: metaobjects
POSTGRES_PASSWORD: metaobjects
POSTGRES_DB: metaobjects_test
ports:
- 5432
options: >-
--health-cmd "pg_isready -U metaobjects"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only java-slow --strict-toolchains

python:
needs: detect
if: contains(needs.detect.outputs.ports, 'python')
runs-on: [self-hosted, linux]
timeout-minutes: 45
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: metaobjects
POSTGRES_PASSWORD: metaobjects
POSTGRES_DB: metaobjects_test
ports:
- 5432
options: >-
--health-cmd "pg_isready -U metaobjects"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- run: scripts/ci-local.sh --only python --strict-toolchains
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only python --strict-toolchains

csharp:
needs: detect
if: contains(needs.detect.outputs.ports, 'csharp')
runs-on: [self-hosted, linux]
timeout-minutes: 45
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: metaobjects
POSTGRES_PASSWORD: metaobjects
POSTGRES_DB: metaobjects_test
ports:
- 5432
options: >-
--health-cmd "pg_isready -U metaobjects"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- run: scripts/ci-local.sh --only csharp --strict-toolchains
- env:
METAOBJECTS_TEST_PG_URL: postgres://metaobjects:metaobjects@localhost:${{ job.services.postgres.ports['5432'] }}/metaobjects_test
run: scripts/ci-local.sh --only csharp --strict-toolchains

notify:
needs: [detect, gates, ts, java, python, csharp]
needs: [detect, gates, ts, java-fast, java-slow, python, csharp]
if: always() && contains(needs.*.result, 'failure')
runs-on: [self-hosted, linux]
steps:
Expand Down
46 changes: 32 additions & 14 deletions scripts/ci-local.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
# # completeness-gate, integration-tests ts
# # java → java+kotlin conformance, reactor,
# # integration-tests java+kotlin
# # java-fast → java+kotlin conformance only
# # (the quick correctness signal)
# # java-slow → reactor + integration-tests
# # (java+kotlin); CI runs the two
# # lanes as separate parallel jobs
# # python → python conformance + integration-tests
# # csharp → csharp conformance + integration-tests
# scripts/ci-local.sh --strict-toolchains
Expand Down Expand Up @@ -59,8 +64,8 @@ while [ $# -gt 0 ]; do
--quick) QUICK=1 ;;
--strict-toolchains) STRICT=1 ;;
--only) shift; case "${1:-}" in
gates|ts|java|python|csharp) ONLY="$ONLY ${1}" ;;
*) echo "--only expects gates|ts|java|python|csharp, got '${1:-}'" >&2; exit 2 ;;
gates|ts|java|java-fast|java-slow|python|csharp) ONLY="$ONLY ${1}" ;;
*) echo "--only expects gates|ts|java|java-fast|java-slow|python|csharp, got '${1:-}'" >&2; exit 2 ;;
esac ;;
-h|--help) awk 'NR==1{next} /^set -uo/{exit} {sub(/^# ?/,""); print}' "$0"; exit 0 ;;
*) echo "unknown arg: $1 (see --help)" >&2; exit 2 ;;
Expand All @@ -73,6 +78,9 @@ want() { # want <section> — true when the section should run
[ -z "$ONLY" ] && return 0
case " $ONLY " in *" $1 "*) return 0 ;; *) return 1 ;; esac
}
want_any() { # want_any <section...> — true when ANY listed section should run
local s; for s in "$@"; do want "$s" && return 0; done; return 1
}

PASS=(); FAIL=(); SKIP=()
have() { command -v "$1" >/dev/null 2>&1; }
Expand Down Expand Up @@ -158,7 +166,11 @@ gate_conf_kotlin() {
}

# ── conformance.yml — full reactor + drift/mutation gates ─────────────────────
gate_java_reactor() { ( cd server/java && mvn -q clean install ); }
# JaCoCo is skipped here: its coverage gate is haltOnFailure=false (pom.xml), so it
# never fails the build — it is pure instrumentation overhead for a pass/fail CI
# signal. `clean` stays: self-hosted workspaces persist and actions/checkout wipes
# only git-tracked state, so a stale target/ can otherwise leak between runs.
gate_java_reactor() { ( cd server/java && mvn -q clean install -Djacoco.skip=true ); }
gate_completeness() { ( cd server/typescript/packages/metadata && bun run conformance:mutation ); }
gate_doc_template_drift() {
bash scripts/sync-doc-templates.sh \
Expand Down Expand Up @@ -230,13 +242,19 @@ if [ "$QUICK" -eq 1 ]; then
SKIP+=("integration-tests (--quick)")
else
# Heavy tier — other-language ports + full reactor + docker integration suite.
if want csharp; then step_if dotnet "conformance: csharp" gate_conf_csharp; fi
if want java; then step_if mvn "conformance: java" gate_conf_java; fi
if want python; then step_if uv "conformance: python" gate_conf_python; fi
if want java; then
step_if mvn "conformance: kotlin" gate_conf_kotlin
step_if mvn "java-reactor (clean install)" gate_java_reactor
fi
#
# The Java port is split into two lanes so CI can run them as separate jobs
# (see .github/workflows/local-ci.yml): a FAST lane (java+kotlin conformance,
# ~2-3m — the correctness signal) and a SLOW lane (full reactor + docker
# integration). Running them as parallel jobs surfaces the conformance verdict
# quickly and means a cancel/OOM mid-integration no longer kills the fast
# signal. `--only java` (local, unqualified) still runs BOTH lanes, in order,
# so a plain local run is byte-equivalent to before.
if want csharp; then step_if dotnet "conformance: csharp" gate_conf_csharp; fi
if want_any java java-fast; then step_if mvn "conformance: java" gate_conf_java; fi
if want python; then step_if uv "conformance: python" gate_conf_python; fi
if want_any java java-fast; then step_if mvn "conformance: kotlin" gate_conf_kotlin; fi
if want_any java java-slow; then step_if mvn "java-reactor (install)" gate_java_reactor; fi
# Docker integration — full suite when no --only, per-port otherwise.
if [ -z "$ONLY" ]; then
if docker info >/dev/null 2>&1; then
Expand All @@ -246,10 +264,10 @@ else
FAIL+=("integration-tests (docker down)")
fi
else
want ts && run_integration_for ts ts
want java && run_integration_for java java kotlin
want python && run_integration_for python python
want csharp && run_integration_for csharp csharp
want ts && run_integration_for ts ts
want_any java java-slow && run_integration_for java java kotlin
want python && run_integration_for python python
want csharp && run_integration_for csharp csharp
fi
fi

Expand Down
Loading