From b038eacd70ba1d4b61dac7d2d5823089071c8e39 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sat, 4 Jul 2026 12:40:23 -0400 Subject: [PATCH] fix(integration-tests): gate shared-sidecar mode on a real connection (kills the ts-integration flake) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ts integration lane intermittently hung to the scenarios' 60s test timeout on the main push path (twice: the #163 and #165 merges), always in the sidecar-backed generated-routes lane (create-201 / jsonb-open-bag-roundtrip). Root cause: shared-sidecar mode (startOnSharedPostgres) did `admin.connect()` immediately with NO readiness gate and NO client-side connect timeout — unlike the per-container path, which deliberately gates on a real `canConnect()` because `pg_isready` (what the CI `services: postgres` health-check uses) can report success during postgres' first-boot init window. Under CI host contention the sidecar is also transiently slow to accept connections. With no connect timeout, pg's first connection to a not-yet-answering sidecar hangs indefinitely → the scenario burns its full 60s timeout instead of retrying. Fix: extend the file's existing readiness pattern to shared mode — `waitForSharedReady()` polls a real short-timeout probe connection until the sidecar answers (or a 45s budget, kept under the 60s test timeout so the retry loop can actually succeed), and every Client now carries connectionTimeoutMillis so a stalled attempt fails fast and is retried rather than hanging. When the sidecar is healthy the gate returns on the first probe (~ms); when it never comes up, it throws a clear error instead of an opaque timeout. Verified: the two previously-flaky lanes (api-contract-generated, api-contract-jsonb) run 23/23 green against a real local sidecar in shared mode. The change can only prevent hangs — no behavior change on the healthy path. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ew1XfYSbEAezxjs9opynAe --- .../src/postgres-container.ts | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/server/typescript/packages/integration-tests/src/postgres-container.ts b/server/typescript/packages/integration-tests/src/postgres-container.ts index 9cd4e344e..abcae6447 100644 --- a/server/typescript/packages/integration-tests/src/postgres-container.ts +++ b/server/typescript/packages/integration-tests/src/postgres-container.ts @@ -31,6 +31,14 @@ import { Client } from "pg"; /** Env var naming the shared CI Postgres sidecar (admin URL). Unset = per-container fallback. */ const SHARED_PG_URL_ENV = "METAOBJECTS_TEST_PG_URL"; +/** Per-attempt connect timeout so a stalled connection fails fast instead of + * hanging (pg's default has no client-side connect timeout). */ +const CONNECT_TIMEOUT_MS = 10_000; +/** Total budget for the shared sidecar to start answering real connections. Kept + * under the scenarios' 60s test timeout so the retry loop has headroom to succeed + * (and to leave time for CREATE DATABASE + server boot) rather than racing it. */ +const SHARED_READY_DEADLINE_MS = 45_000; + export interface RunningPg { readonly connectionUri: string; stop(): Promise; @@ -62,8 +70,17 @@ export async function startPostgres(image = "postgres:16-alpine"): Promise { + // Gate on a REAL connection before doing anything. The CI `services: postgres` + // health-check uses `pg_isready`, which — like the per-container path documents + // (waitForPgReady) — can report success during postgres' first-boot init window; + // and under CI host contention the server can be transiently slow to accept + // connections. Without this gate the first admin.connect() below hangs until the + // scenario's 60s test timeout (the intermittent ts-integration flake). Retrying a + // short-timeout probe waits the sidecar out gracefully instead. + await waitForSharedReady(adminUri); + const dbName = `mo_test_${randomUUID().replace(/-/g, "")}`; - const admin = new Client({ connectionString: adminUri }); + const admin = new Client({ connectionString: adminUri, connectionTimeoutMillis: CONNECT_TIMEOUT_MS }); await admin.connect(); try { // Identifier is a fixed-shape generated name (no user input) — safe to inline. @@ -75,7 +92,7 @@ async function startOnSharedPostgres(adminUri: string): Promise { return { connectionUri, stop: async () => { - const drop = new Client({ connectionString: adminUri }); + const drop = new Client({ connectionString: adminUri, connectionTimeoutMillis: CONNECT_TIMEOUT_MS }); await drop.connect(); try { await drop.query(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`); @@ -86,6 +103,20 @@ async function startOnSharedPostgres(adminUri: string): Promise { }; } +// Poll a real client connection to the shared sidecar until it answers or the +// deadline elapses — the shared-mode analogue of waitForPgReady (mode 2). Cheap +// (~ms) when the sidecar is already healthy; only retries when it is genuinely slow. +async function waitForSharedReady(adminUri: string): Promise { + const deadline = Date.now() + SHARED_READY_DEADLINE_MS; + while (Date.now() < deadline) { + if (await canConnect(adminUri)) return; + await new Promise((res) => setTimeout(res, 250)); + } + throw new Error( + `shared Postgres sidecar (${SHARED_PG_URL_ENV}) did not accept a connection within ${SHARED_READY_DEADLINE_MS / 1000}s`, + ); +} + /** Replace the database (path) component of a postgres:// URL. */ function withDatabase(uri: string, dbName: string): string { const u = new URL(uri); @@ -113,7 +144,7 @@ async function waitForPgReady(name: string, uri: string): Promise { } async function canConnect(uri: string): Promise { - const client = new Client({ connectionString: uri }); + const client = new Client({ connectionString: uri, connectionTimeoutMillis: CONNECT_TIMEOUT_MS }); try { await client.connect(); await client.query("SELECT 1");