From e8e4bbdee26e94950c82ecb788c9ce4266664fc3 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:59:10 -0700 Subject: [PATCH] fix(gate,queue,ai): defer the gate on a renderer outage, re-drive merge-train waiters, debounce force-push storms, and book the real retry budget (#9464, #9483, #9479) --- src/db/repositories.ts | 43 ++++-- src/github/webhook-coalesce.ts | 47 ++++++- src/github/webhook.ts | 7 +- src/queue/processors.ts | 10 +- src/review/visual/capture.ts | 42 +++++- src/review/visual/shot.ts | 37 +++-- src/services/ai-review.ts | 17 ++- test/unit/ai-review.test.ts | 53 ++++++++ test/unit/db-parsers.test.ts | 53 +++++++- test/unit/github-webhook-coalesce.test.ts | 93 ++++++++++++- test/unit/queue-3.test.ts | 156 ++++++++++++++++++++++ test/unit/selfhost-queue-common.test.ts | 3 +- test/unit/selfhost-sqlite-queue.test.ts | 53 ++++++-- test/unit/visual-capture.test.ts | 109 ++++++++++++--- test/unit/visual-shot.test.ts | 21 ++- test/unit/webhook.test.ts | 33 +++++ 16 files changed, 709 insertions(+), 68 deletions(-) diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 4160d2d446..16fcd6f25b 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -3109,24 +3109,43 @@ export async function countRecentAuditEventsForActorInRepoWithTargetSuffix( return row.count; } -// #orb-stale-recheck-priority: the THREE self-resolving "denied" detail strings the executor's live staleness -// rechecks can produce (agent-action-executor.ts's "8) Live ... re-verification" block -- duplicateStaleReason / -// mergeableStaleReason / threadStaleReason, each suffixed with " — action not executed" by that block's shared -// `audit("denied", ...)` call). Deliberately NOT imported from agent-action-executor.ts: that module imports +// #orb-stale-recheck-priority: the self-resolving "denied" detail strings the executor can produce for a PR +// that NOTHING will otherwise re-drive. Three come from its live staleness rechecks (agent-action-executor.ts's +// "8) Live ... re-verification" block -- duplicateStaleReason / mergeableStaleReason / threadStaleReason); the +// fourth is the merge-train wait. All are suffixed with " — action not executed" by the shared +// `audit("denied", ...)` call. Deliberately NOT imported from agent-action-executor.ts: that module imports // FROM db/repositories.ts (installation tokens, PR records, ...), so importing back would create a real // module-load cycle -- same hazard agent-actions.ts's CONCRETE_EVIDENCE_BLOCKER_CODES comment documents for the -// identical reason. A source-text parity test guards these three literals against producer-side drift instead. -// Deliberately EXCLUDES a CI-staleness denial (ciStaleReason): CI flipping already re-triggers a fresh -// evaluation via the check-run/status webhook that changed it, so it doesn't share the other three's "no -// webhook ever reaches this PR" gap. +// identical reason. A source-text parity test guards these literals against producer-side drift instead. +// +// #9483: the merge-train wait was originally excluded here as "durable, externally-actioned", on the theory +// that the blocker merging would wake the waiter via maybeEnqueueSiblingRegateForMergedPr. That classification +// was wrong, and the blocker MERGING is the only case it holds for. A blocker can also leave the train by +// being CLOSED UNMERGED (that wake returns early on `!mergedAt`), by AGING past MERGE_TRAIN_MAX_WAIT_MS, or by +// gaining the manual-review label (#9039 eviction) -- and the latter two are consulted only at the waiter's own +// evaluation time, which nothing schedules. In every one of those three the waiter sat open indefinitely with +// no signal, which under one-shot review reads to the contributor as a silent rejection. It is in fact the +// PUREST instance of the gap this whole mechanism exists for: resolving the blocker fires a webhook about the +// BLOCKER, never about the PR waiting on it. +// +// Bounded by construction, so this cannot become a retry storm: a matching PR inherits +// surfaceRepairPriorityPullNumbers' existing REGATE_REPAIR_MAX_ATTEMPTS_PER_PR budget, and that budget's +// lookback window is ROLLING and the same 24h as MERGE_TRAIN_MAX_WAIT_MS -- so a PR blocked long enough for its +// blocker to age out of the train has also had its earliest attempts age out, and gets fresh looks exactly when +// the age-out makes them useful. +// +// Still deliberately EXCLUDES a CI-staleness denial (ciStaleReason): CI flipping already re-triggers a fresh +// evaluation via the check-run/status webhook that changed it, so it doesn't share the others' "no webhook ever +// reaches this PR" gap. const STALE_RECHECK_DENIAL_DETAIL_PATTERN = - /^(duplicate-cluster winner #\d+ is no longer open|the base-branch conflict that justified this close has since cleared|the review thread\(s\) that justified this close are now all resolved) — action not executed$/; + /^(duplicate-cluster winner #\d+ is no longer open|the base-branch conflict that justified this close has since cleared|the review thread\(s\) that justified this close are now all resolved|merge train: waiting for older mergeable sibling #\d+) — action not executed$/; /** * PR numbers within `repoFullName` whose most recent `agent.action.close`/`agent.action.merge` attempt was - * DENIED by one of the executor's live staleness rechecks (duplicate-cluster winner / base-conflict / review- - * thread) within `sinceIso`, rather than by a durable, externally-actioned reason (a manual-review label, a - * merge-train wait, contributor-cap contention, ...). Those rechecks exist precisely because the fact that + * DENIED by a self-resolving reason (duplicate-cluster winner / base-conflict / review-thread staleness, or a + * merge-train wait -- see the pattern's own comment for why the last one belongs here) within `sinceIso`, + * rather than by a durable, externally-actioned reason (a manual-review label, contributor-cap contention, + * ...). Those rechecks exist precisely because the fact that * justified the close/merge can flip WITHOUT a webhook ever notifying THIS pr -- a duplicate-cluster sibling * merging fires a webhook about the SIBLING, not this PR, so nothing naturally re-triggers a look here. Callers * (surfaceRepairPriorityPullNumbers) fold this into the SAME priority set the outage-repair path already uses, diff --git a/src/github/webhook-coalesce.ts b/src/github/webhook-coalesce.ts index b910d1bbab..de424974b1 100644 --- a/src/github/webhook-coalesce.ts +++ b/src/github/webhook-coalesce.ts @@ -11,6 +11,46 @@ const COALESCABLE_PULL_REQUEST_ACTIONS = new Set([ "ready_for_review", ]); +/** + * #9479: `synchronize` -- a PUSH -- gets its own PR-scoped coalesce key and a trailing quiet window; every other + * action above keeps the head-SHA-scoped key unchanged. + * + * Every dedup layer in the pipeline was keyed on the head SHA (this key, and the AI-review lock's + * `...@${headSha}:${mode}`), which is exactly the wrong key for a force-push storm: each push MINTS a new SHA, so + * five amend-and-repush cycles in a minute looked like five unrelated events and bought five full prologues + * (file list, up to 96k chars of grounding fetch, RAG + impact-map embeddings, an enrichment POST) and five LLM + * calls. Four of those five review heads no longer exist by the time their review lands. `skipStaleReviewOutput` + * suppresses the stale COMMENT, but only after the spend; the `review burst` ops alert reports the storm without + * throttling it. + * + * Dropping the SHA makes consecutive pushes collide, and the queue's coalesce keeps the LATEST payload while + * extending `run_after` (`GREATEST(run_after, ...)` in pg-queue/sqlite-queue's enqueue) -- so a burst converges to + * ONE job, at the last push plus the window, reviewing the head that actually survived. + * + * Deliberately a SEPARATE key from `pr-refresh` rather than simply dropping the SHA there. A shared PR-scoped key + * would let a push swallow a still-pending `opened`/`ready_for_review` in the same window, since coalescing + * overwrites the payload and only the last action survives -- trading a spend bug for a lost-lifecycle-event bug. + */ +const PUSH_COALESCE_ACTION = "synchronize"; + +/** How long to hold a push before reviewing it, so a re-push lands inside the same window and replaces it. Sized + * to match CI_COALESCE_WINDOW_SECONDS (processors.ts), the pipeline's existing burst window, and to comfortably + * cover a human amend-and-force-push cycle without adding meaningful latency to an ordinary single push -- which + * then waits on CI anyway. */ +export const PUSH_COALESCE_QUIET_WINDOW_SECONDS = 45; + +/** + * The delay to enqueue this delivery with. Non-zero ONLY for a push, whose key above is built to coalesce. + * + * On a queue that coalesces (self-host pg/sqlite, which is where the ORB runs) this is what CREATES the window: + * without a delay the first push is claimed immediately and later pushes find no pending row to merge into. On + * Cloudflare Queues, which has no job_key coalescing, it is a plain 45s deferral of push-triggered reviews and + * nothing more -- correct, just not a saving. + */ +export function githubWebhookCoalesceDelaySeconds(eventName: string, payload: GitHubWebhookPayload): number { + return eventName === "pull_request" && payload.action === PUSH_COALESCE_ACTION ? PUSH_COALESCE_QUIET_WINDOW_SECONDS : 0; +} + // #selfhost-backlog-convergence: every "labeled"/"unlabeled" delivery re-syncs the PR row // (upsertPullRequestFromGitHub) regardless of which specific label changed -- shouldProcessPullRequestPublicSurface // (processors.ts) additionally runs the public-surface pipeline itself, but only when the changed label is a @@ -61,10 +101,11 @@ export function githubWebhookCoalesceKey( const pr = normalizedNumber(payload.pull_request?.number) ?? normalizedNumber((payload as { number?: unknown }).number); + if (pr === null) return null; + // #9479: a push is keyed on the PR alone, so the NEXT push collapses into it. See PUSH_COALESCE_ACTION. + if (action === PUSH_COALESCE_ACTION) return `github-webhook:pr-push:${repo}#${pr}`; const headSha = normalizedSha(payload.pull_request?.head?.sha); - return pr !== null - ? `github-webhook:pr-refresh:${repo}#${pr}${headSha ? `@${headSha}` : ""}` - : null; + return `github-webhook:pr-refresh:${repo}#${pr}${headSha ? `@${headSha}` : ""}`; } if (eventName === "pull_request" && COALESCABLE_PULL_REQUEST_LABEL_ACTIONS.has(action)) { const pr = diff --git a/src/github/webhook.ts b/src/github/webhook.ts index 71537b675c..18209075e6 100644 --- a/src/github/webhook.ts +++ b/src/github/webhook.ts @@ -8,6 +8,7 @@ import { isSelfHostedReviewRuntime } from "../selfhost/review-runtime"; import { incr } from "../selfhost/metrics"; import { getSelfHostRequestTraceParent } from "../selfhost/trace-context"; import { isNonActionableWebhookNoise } from "./self-authored"; +import { githubWebhookCoalesceDelaySeconds } from "./webhook-coalesce"; const DEFAULT_MAX_WEBHOOK_BODY_BYTES = 1024 * 1024; // #9054: how long a 'queued'/'superseded' webhook_events row may sit unprocessed before a redelivery of the @@ -225,7 +226,11 @@ export async function enqueueWebhookByEnv(env: Env, deliveryId: string, eventNam try { // Send to the dedicated WEBHOOKS lane (not the shared JOBS queue) so a maintenance burst on JOBS can never // starve real GitHub events into the DLQ. (#audit-webhook-queue) - await env.WEBHOOKS.send(message); + // #9479: a push is deferred by a short quiet window so a force-push storm coalesces into one review of the + // head that survives, instead of buying a full prologue + LLM call per intermediate SHA. Zero for every + // other event, so this is a no-op for everything but `pull_request`/`synchronize` -- see + // githubWebhookCoalesceDelaySeconds. + await env.WEBHOOKS.send(message, { delaySeconds: githubWebhookCoalesceDelaySeconds(eventName, payload) }); } catch (error) { // Enqueue failed: flip the event to "error" so the dedup guard above lets GitHub redeliver / the next pull // re-deliver, instead of treating the webhook as handled (#786). Also covers the deploy-ordering case where diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 1d7d4dbaa4..18b8b6540d 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -12540,7 +12540,7 @@ async function maybePublishPrPublicSurface( // ordinary "nothing found" capture would, with no separate code path to maintain. const capture = reviewVisualConfig.enabled === false - ? { routes: [], interactions: [], previewPending: false } + ? { routes: [], interactions: [], previewPending: false, renderFailed: false } : await buildCapture(env, token, captureTarget, visualFiles, githubRateLimitAdmissionKeyForInstallation(installationId), reviewVisualConfig, changedCssFiles); beforeAfter = capture.routes; interactionPreviews = capture.interactions; @@ -12566,8 +12566,14 @@ async function maybePublishPrPublicSurface( // preview deploy isn't live yet (capture.previewPending). Schedule a delayed re-review to re-capture // the now-ready shot — bounded by `attempt` so a never-resolving preview can't loop (the deployment_status // webhook also refills it; this is the backstop when that event is missed/late). + // + // #9464: `capture.renderFailed` joins previewPending here as a SECOND blip class. captureShot + // swallows a renderer error per shot and returns a null PNG, so buildCapture used to return + // normally -- previewPending false, nothing thrown -- and neither the #9030 nor the #9207 guard + // fired. The maintenance pass then read "no evidence, no retry pending" and CLOSED the PR one-shot. + // A browserless outage now degrades to "we could not capture evidence, holding" instead. const previewPollAttempt = webhook.previewPollAttempt ?? 0; - if (capture.previewPending) { + if (capture.previewPending || capture.renderFailed) { await scheduleVisualCaptureRetry(env, { webhook, repoFullName, diff --git a/src/review/visual/capture.ts b/src/review/visual/capture.ts index d533cfe130..2de23cd413 100644 --- a/src/review/visual/capture.ts +++ b/src/review/visual/capture.ts @@ -89,6 +89,18 @@ export interface CaptureResult { routes: CaptureRoute[]; interactions: CaptureInteractionRoute[]; previewPending: boolean; + /** + * #9464: at least one shot in this capture failed because the RENDERER broke (see `CaptureShotResult. + * renderFailed`) -- browserless down, saturated, or timing out. Distinct from `previewPending`, which means + * the page we want to render does not exist yet. + * + * Both are "we could not obtain evidence", and the screenshot-table gate must defer its one-shot close for + * either. Before this, a renderer failure was swallowed per shot and `buildCapture` returned NORMALLY with + * `previewPending: false` and no throw, so the #9030/#9207 blip guards -- which trigger only on + * previewPending or a thrown error -- never fired, and the gate closed legitimate visual PRs during a + * browserless outage. + */ + renderFailed: boolean; } /** True when `url` is a persisted rendered shot. `capturePage` can also return an on-demand `?url=` @@ -342,6 +354,10 @@ function resolveShotUrl(env: Env, key: string): string { * the cell shows a dash. Reuses an identical cached fingerprint (a deployment_status re-run filling "after" * cells would otherwise re-render the same screenshot — Browser Rendering is the costliest binding). */ +/** What `capturePage` and its no-preview sibling `resolveFallbackAfterShot` both resolve to. `renderFailed` + * is optional because only the live-render path can observe it -- see {@link CaptureResult.renderFailed}. */ +type CapturedShot = { url?: string | undefined; thumbUrl?: string | undefined; png?: Uint8Array | undefined; renderFailed?: boolean | undefined }; + async function capturePage( env: Env, target: CaptureTarget, @@ -361,8 +377,12 @@ async function capturePage( // theming ignores prefers-color-scheme (see shot.ts's CaptureShotOptions.theme doc). Only takes effect // together with `theme`; undefined (every pre-#4109 caller) ⇒ byte-identical to today. themeStorageKey?: string | undefined, -): Promise<{ url?: string | undefined; thumbUrl?: string | undefined; png?: Uint8Array | undefined }> { +): Promise { if (!page) return {}; + // #9464: sticky for the whole call, because the only return that can carry it is the on-demand tail below -- + // every earlier return is a definite outcome (cache hit, auth wall, a real PNG) that by construction cannot + // have come from a broken renderer. + let renderFailed = false; const shotBase = env.PUBLIC_API_ORIGIN; // this worker's public origin (serves /loopover/shot) // Carries the theme (#3678) and, when set, the storage key (#4109) so a LATER on-demand fetch of this // exact URL (e.g. a failed/never-persisted render retried by GitHub's image proxy) still requests the @@ -405,7 +425,14 @@ async function capturePage( const bytes = await new Response(cached.body).arrayBuffer().then((buf) => new Uint8Array(buf)).catch(() => undefined); return { url, ...(thumbUrl ? { thumbUrl } : {}), ...(bytes ? { png: bytes } : {}) }; } - const { png, authWalled } = await captureShot(env, page, viewport, theme ? { theme, ...(themeStorageKey ? { themeStorageKey } : {}) } : {}).catch(() => ({ png: null, authWalled: false })); + // The outer `.catch` reports renderFailed too: captureShot swallows its own renderer errors, so anything + // that still escapes it (a DNS-guard or token-mint throw) is likewise an outcome we could not determine. + const { png, authWalled, renderFailed: shotRenderFailed } = await captureShot(env, page, viewport, theme ? { theme, ...(themeStorageKey ? { themeStorageKey } : {}) } : {}).catch(() => ({ + png: null, + authWalled: false, + renderFailed: true, + })); + renderFailed = shotRenderFailed; // A protected route that redirected to a sign-in wall: show an honest "requires authentication" // placeholder rather than caching/serving a screenshot of the login screen. if (authWalled) { @@ -430,7 +457,7 @@ async function capturePage( return { url, ...(thumbUrl ? { thumbUrl } : {}), ...(includeBytes ? { png } : {}) }; } } - return { url: onDemand }; + return { url: onDemand, renderFailed }; } /** Resolve the "after" shot when there is no real preview page to render (#4112): if `review.visual. @@ -446,7 +473,7 @@ async function resolveFallbackAfterShot( viewportName: "desktop" | "mobile", actionsFallbackEnabled: boolean, placeholder: string | undefined, -): Promise<{ url?: string | undefined; thumbUrl?: string | undefined; png?: Uint8Array | undefined }> { +): Promise { // #6324: never produces a thumbUrl (the actions_fallback artifact is stored as-is, no display downscale // applied) -- typed here purely so this function's return shape matches capturePage's, since buildCapture // uses both interchangeably for the "after" desktop slot. @@ -800,6 +827,10 @@ export async function buildCapture( // `theme` iteration is undefined (the untagged default pass). const themeStorageKey = visualConfig?.themeStorageKey ? visualConfig.themeStorageKey : undefined; const captureRoutes: CaptureRoute[] = []; + // #9464: ORed across every shot of every route/viewport/theme. ONE failed render is enough -- the gate's + // question is "could we have missed evidence?", and a partial outage answers yes. When some routes DID + // render, hasSuccessfulBotCapture already satisfies the gate and this never gets consulted. + let renderFailed = false; for (const theme of themes) { for (const path of routes) { const beforePage = prodBase ? joinUrl(prodBase, path) : ""; @@ -815,6 +846,7 @@ export async function buildCapture( ? capturePage(env, target, afterPage, "after", "mobile", MOBILE_VIEWPORT, diffAvailable, theme, themeStorageKey) : resolveFallbackAfterShot(env, target, path, "mobile", actionsFallbackEnabled, afterPlaceholder), ]); + if ([beforeShot, beforeMobileShot, afterShot, afterMobileShot].some((shot) => shot.renderFailed === true)) renderFailed = true; // A diff needs BOTH sides' real bytes — a placeholder/dash slot (no preview yet, auth-walled, render // failure) has no `png`, so compareCapturedScreenshots degrades to null exactly like a missing shot does. const [desktopDiff, mobileDiff] = diffAvailable @@ -926,5 +958,5 @@ export async function buildCapture( } } - return { routes: captureRoutes, interactions: interactionRoutes, previewPending }; + return { routes: captureRoutes, interactions: interactionRoutes, previewPending, renderFailed }; } diff --git a/src/review/visual/shot.ts b/src/review/visual/shot.ts index 20c457a069..f0a9acbe46 100644 --- a/src/review/visual/shot.ts +++ b/src/review/visual/shot.ts @@ -296,21 +296,38 @@ async function captureBoundedFullPageShot(page: ScreenshotPage, viewport: Viewpo return shot; } +/** + * `captureShot`'s outcome. `renderFailed` (#9464) separates "the RENDERER broke" -- a browserless outage, a + * navigation timeout, a binding quota error, a Chromium crash -- from every other reason `png` can be null: + * an SSRF/redirect refusal, an auth wall, or visual review simply not being configured (`!env.BROWSER`). + * + * The distinction is load-bearing, not cosmetic. The screenshot-table gate reads "no bot capture" as "the + * author supplied no visual evidence" and CLOSES the PR one-shot. Only the renderer-broke case is a false + * negative that must defer that close; the others are accurate answers the gate should keep acting on. Basing + * the signal on "zero pairs were produced" instead would fold all of them together and neuter the gate. + */ +export interface CaptureShotResult { + png: Uint8Array | null; + authWalled: boolean; + /** True ONLY for a thrown renderer error (the `catch` below). Never for a refusal or an unconfigured binding. */ + renderFailed: boolean; +} + /** * Render a page to a PNG via the Browser Rendering binding, also reporting whether the route redirected to a * sign-in wall. `authWalled` is true when the FINAL url looks like a login page that the REQUESTED url was * not — the caller then shows an honest "requires authentication" placeholder instead of a screenshot of the * login screen. `png` is null on any render failure (callers degrade gracefully). */ -export async function captureShot(env: Env, url: string, viewport: Viewport = VIEWPORT, opts: CaptureShotOptions = {}): Promise<{ png: Uint8Array | null; authWalled: boolean }> { +export async function captureShot(env: Env, url: string, viewport: Viewport = VIEWPORT, opts: CaptureShotOptions = {}): Promise { // SSRF defense-in-depth: NEVER navigate the headless browser to a non-public host (loopback / link-local / // private / cloud-metadata 169.254.169.254 / etc.). Callers may resolve `url` from a deployment_status // webhook or a PR-comment preview link, so guard at this choke point regardless of how the URL was obtained. if (!url || !isSafeHttpUrl(url) || (opts.isAllowedUrl && !opts.isAllowedUrl(url)) || !(await isDnsResolutionSafe(url))) { console.log(JSON.stringify({ event: "render_screenshot_blocked", url: String(url).slice(0, 120) })); - return { png: null, authWalled: false }; + return { png: null, authWalled: false, renderFailed: false }; } - if (!env.BROWSER) return { png: null, authWalled: false }; + if (!env.BROWSER) return { png: null, authWalled: false, renderFailed: false }; let browser: Awaited> | null = null; try { browser = await puppeteer.launch(env.BROWSER as unknown as Parameters[0]); @@ -340,14 +357,14 @@ export async function captureShot(env: Env, url: string, viewport: Viewport = VI await page.goto(url, { waitUntil: "networkidle0", timeout: 20000 }); if (!isSafeHttpUrl(page.url()) || (opts.isAllowedUrl && !opts.isAllowedUrl(page.url()))) { console.log(JSON.stringify({ event: "render_screenshot_redirect_blocked", url, final: page.url().slice(0, 200) })); - return { png: null, authWalled: false }; + return { png: null, authWalled: false, renderFailed: false }; } // A protected route that redirected to a login page: don't return a screenshot of the sign-in screen — // flag it so the caller renders an honest auth placeholder. (The requested URL not itself being a login // page guards a PR that legitimately changes the login screen.) if (isAuthWallUrl(page.url()) && !isAuthWallUrl(url)) { console.log(JSON.stringify({ event: "render_screenshot_auth_walled", url, final: page.url().slice(0, 200) })); - return { png: null, authWalled: true }; + return { png: null, authWalled: true, renderFailed: false }; } // A configured themeStorageKey (#4109) ALSO forces the theme via localStorage, then reloads so the // app's own theme-init logic re-runs against the new stored value -- the fallback for a target whose @@ -356,7 +373,9 @@ export async function captureShot(env: Env, url: string, viewport: Viewport = VI if (opts.theme && opts.themeStorageKey) { const storageKey = opts.themeStorageKey; const storageValue = opts.theme; - if (!(await forceThemeStorage(page, storageKey, storageValue))) return { png: null, authWalled: false }; + // A failed localStorage write is the PAGE refusing (storage disabled/partitioned), not the renderer + // breaking -- the browser is demonstrably alive, since it navigated and ran script to get here. + if (!(await forceThemeStorage(page, storageKey, storageValue))) return { png: null, authWalled: false, renderFailed: false }; await page.reload({ waitUntil: "networkidle0", timeout: THEME_STORAGE_RELOAD_TIMEOUT_MS }); } // Full-page (not just the viewport), but bounded: before/after should include the same page position for @@ -364,7 +383,7 @@ export async function captureShot(env: Env, url: string, viewport: Viewport = VI // Chromium raster work on the public screenshot route. const shot = await captureBoundedFullPageShot(page, viewport); incr("loopover_visual_capture_total", { result: "ok" }); - return { png: shot, authWalled: false }; + return { png: shot, authWalled: false, renderFailed: false }; } catch (error) { // Log before degrading to null — otherwise a networkidle0 timeout, a binding quota error, or a render // crash is indistinguishable from "no page" and the cell silently blanks. @@ -373,7 +392,9 @@ export async function captureShot(env: Env, url: string, viewport: Viewport = VI // absent evidence as a close signal, that outage could close legitimate visual PRs before anyone noticed. incr("loopover_visual_capture_total", { result: "error" }); console.log(JSON.stringify({ event: "render_screenshot_error", mode: "binding", url, message: String(error).slice(0, 200) })); - return { png: null, authWalled: false }; + // #9464: the ONLY site that reports renderFailed. Everything above returned a definite answer; this is the + // one path where we do not know what the page looks like because the renderer itself failed to tell us. + return { png: null, authWalled: false, renderFailed: true }; } finally { if (browser) await browser.close().catch(() => undefined); } diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 2fb19a0f76..23ed78894a 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -1443,6 +1443,11 @@ export function isStructuralProviderConfigError(error: unknown): boolean { * a large chunk of model output into Sentry/audit context. */ const UNPARSEABLE_RESPONSE_SNIPPET_MAX_CHARS = 400; +/** Retries per model inside one opinion slot, before the slot falls through to its fallback model (which then + * gets its own full budget). Exported-in-spirit as a named constant because the neuron pre-booking in + * {@link runAiReview} must reserve exactly this worst case -- #9479 was that loop and the budget disagreeing. */ +const REVIEW_ATTEMPTS_PER_MODEL = 3; + /** One reviewer opinion (whichever provider `env.AI` resolves to — self-host Codex/Claude Code/etc, or the * legacy Workers-AI pair) with a per-slot reliable fallback and a 3× retry on the primary. */ async function runWorkersOpinion( @@ -1504,7 +1509,7 @@ async function runWorkersOpinion( // stop-retrying-this-model reasoning as the deliberate-bail/timeout/429 breaks below; the next model // still gets its own full budget. let lastRawText: string | undefined; - for (let attempt = 0; attempt < 3; attempt += 1) { + for (let attempt = 0; attempt < REVIEW_ATTEMPTS_PER_MODEL; attempt += 1) { try { const cliSystemAppend = selfHostCliSystemAppend(model, systemAppend); const result = await ai.run( @@ -2809,8 +2814,16 @@ export async function runLoopOverAiReview( incr("loopover_ai_review_onmerge_clamped_total", { mode: input.mode }); } const dual = combine !== "single" && (!configured || configured.length > 1); + // #9479: ONE call per opinion slot was never the worst case -- runWorkersOpinion retries each model up to + // REVIEW_ATTEMPTS_PER_MODEL times and then falls through to the slot's fallback model with its own full + // budget, so a dual-model block review can make 12 calls where this booked 2. The daily neuron budget is a + // runaway-LOOP backstop; booking the best case made it 6x looser than it reads, which is the one direction a + // backstop must never be wrong in. The tie-break judge below and ai-slop.ts's WORKERS_SLOP_MAX_CALLS both + // already reserve worst-case -- the main review path was the outlier, not the rule. + const slotCalls = (model: string, modelFallback: string): number => REVIEW_ATTEMPTS_PER_MODEL * (modelFallback !== model ? 2 : 1); const freeAiCalls = - (input.mode === "block" ? (dual ? 2 : 1) : 0) + (input.providerKey ? 0 : 1); + (input.mode === "block" ? slotCalls(primary.model, primaryFallback) + (dual ? slotCalls(secondary.model, secondaryFallback) : 0) : 0) + + (input.providerKey ? 0 : 1); // Consensus disagreements may spend extra free calls on the order-swapped tie-break judge. Reserve the // worst-case retry budget up front so the daily limiter remains a hard cap even when judge output is unstable. const tieBreakAiCalls = diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 763a0aca3f..51d178b69e 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -216,6 +216,59 @@ describe("runLoopOverAiReview gating", () => { expect(result.status).toBe("ok"); // NaN → clamped to the 256 floor, review still runs }); + // #9479: the reservation booked ONE call per opinion slot, but runWorkersOpinion retries each model up to + // REVIEW_ATTEMPTS_PER_MODEL times and then falls through to that slot's fallback model with its own full + // budget -- so a dual-model block review can make 12 calls where 2 were booked. The daily neuron budget is a + // runaway-LOOP backstop; booking the best case made it 6x looser than it reads, which is the one direction a + // backstop must never be wrong in. The judge path below and ai-slop.ts's WORKERS_SLOP_MAX_CALLS already + // reserved worst-case; the main review path was the outlier. + it("REGRESSION (#9479): books the worst-case retry x fallback call count, so a budget that only covers the best case is refused", async () => { + const run = vi.fn(); + // 2000 sat comfortably ABOVE the old best-case reservation for this exact input and so let the review + // start, even though actually running it could spend several times that. + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "2000", + }); + + const result = await runLoopOverAiReview(env, { ...baseInput, mode: "block" }); + + expect(result.status).toBe("quota_exceeded"); + expect(result.status === "quota_exceeded" && result.estimatedNeurons).toBeGreaterThan(2000); + expect(run).not.toHaveBeenCalled(); + }); + + it("INVARIANT (#9479): the reservation tracks the real fallback structure — a pair with no distinct fallback books strictly less, not a flat inflation", async () => { + const run = vi.fn(async () => ({ response: reviewJson() })); + // Same budget, same input, but each slot reuses its own model as its fallback (the self-host shape), so the + // worst case is halved and this must still be ADMITTED. A blanket multiplier would have refused it too. + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "2000", + }); + + const result = await runLoopOverAiReview(env, { ...baseInput, mode: "block", reviewers: [{ model: "claude-code" }, { model: "codex" }] }); + + expect(result.status).not.toBe("quota_exceeded"); + expect(run).toHaveBeenCalled(); + }); + + it("INVARIANT (#9479): an ADVISORY review is unaffected — it makes no blocking opinion calls, so its reservation is unchanged", async () => { + const run = vi.fn(async () => ({ response: reviewJson() })); + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "2000", + }); + + await expect(runLoopOverAiReview(env, baseInput)).resolves.not.toMatchObject({ status: "quota_exceeded" }); + }); + it("does NOT count a BYOK advisory against the free neuron budget (it bills the maintainer)", async () => { const fetchMock = vi.fn( async () => diff --git a/test/unit/db-parsers.test.ts b/test/unit/db-parsers.test.ts index 5f8ce79fc4..185ed9409a 100644 --- a/test/unit/db-parsers.test.ts +++ b/test/unit/db-parsers.test.ts @@ -1152,6 +1152,51 @@ describe("database row parser hardening", () => { expect(await recentStaleRecheckDeniedPullNumbers(env, "owner/repo", "2026-06-24T09:00:00.000Z")).toEqual([6]); }); + // #9483: a merge-train waiter had NO path back to evaluation unless its blocker MERGED. The blocker being + // closed unmerged, aging past MERGE_TRAIN_MAX_WAIT_MS, or gaining the manual-review label all leave the + // waiter open indefinitely with no signal, because every one of those fires a webhook about the BLOCKER and + // the two age/label checks are consulted only at the waiter's own evaluation time -- which nothing + // schedules. Under one-shot review that reads to the contributor as a silent rejection. + it("REGRESSION (#9483): returns the PR number for a merge-train wait, which nothing else re-drives", async () => { + const env = createTestEnv(); + await recordAuditEvent(env, { + eventType: "agent.action.merge", + actor: "loopover", + targetKey: "owner/repo#20", + outcome: "denied", + detail: "merge train: waiting for older mergeable sibling #19 — action not executed", + createdAt: "2026-06-24T10:00:00.000Z", + }); + + expect(await recentStaleRecheckDeniedPullNumbers(env, "owner/repo", "2026-06-24T09:00:00.000Z")).toEqual([20]); + }); + + it("INVARIANT (#9483): a merge-train wait matches on the blocking PR number only as part of the WHOLE reason — no partial/suffix match", async () => { + const env = createTestEnv(); + // Missing the " — action not executed" suffix the shared audit() closure appends: this is not a shape the + // executor produces, and anchoring the pattern at both ends is what keeps an attacker-influenced or + // hand-written detail string out of the priority set. + await recordAuditEvent(env, { + eventType: "agent.action.merge", + actor: "loopover", + targetKey: "owner/repo#21", + outcome: "denied", + detail: "merge train: waiting for older mergeable sibling #19", + createdAt: "2026-06-24T10:00:00.000Z", + }); + // A non-numeric blocker reference likewise must not match. + await recordAuditEvent(env, { + eventType: "agent.action.merge", + actor: "loopover", + targetKey: "owner/repo#22", + outcome: "denied", + detail: "merge train: waiting for older mergeable sibling #abc — action not executed", + createdAt: "2026-06-24T10:00:00.000Z", + }); + + expect(await recentStaleRecheckDeniedPullNumbers(env, "owner/repo", "2026-06-24T09:00:00.000Z")).toEqual([]); + }); + it("ignores a denial reason that is NOT one of the three self-resolving staleness reasons", async () => { const env = createTestEnv(); // A manual-review-label denial is durable (needs a human to remove the label) -- retrying fast wastes a @@ -1223,7 +1268,13 @@ describe("database row parser hardening", () => { // agent-actions.test.ts, documents). This reads the real producer source text instead, so a future rewording // at the producer fails this test immediately rather than silently making the hand-typed pattern permanently // unmatchable. - it.each(["duplicate-cluster winner #${action.duplicateWinnerPrNumber} is no longer open", "the base-branch conflict that justified this close has since cleared", "the review thread(s) that justified this close are now all resolved"])( + it.each([ + "duplicate-cluster winner #${action.duplicateWinnerPrNumber} is no longer open", + "the base-branch conflict that justified this close has since cleared", + "the review thread(s) that justified this close are now all resolved", + // #9483: the merge-train wait joined the pattern; it needs the same producer-drift guard as the others. + "merge train: waiting for older mergeable sibling #${decision.blockingPr}", + ])( "%s is still produced (not merely mentioned) in its producer (src/services/agent-action-executor.ts)", (reason) => { const source = readFileSync("src/services/agent-action-executor.ts", "utf8"); diff --git a/test/unit/github-webhook-coalesce.test.ts b/test/unit/github-webhook-coalesce.test.ts index 880aee651c..c5fb9106d4 100644 --- a/test/unit/github-webhook-coalesce.test.ts +++ b/test/unit/github-webhook-coalesce.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { githubWebhookCoalesceKey } from "../../src/github/webhook-coalesce"; +import { githubWebhookCoalesceDelaySeconds, githubWebhookCoalesceKey, PUSH_COALESCE_QUIET_WINDOW_SECONDS } from "../../src/github/webhook-coalesce"; import type { GitHubWebhookPayload } from "../../src/types"; describe("githubWebhookCoalesceKey", () => { @@ -25,6 +25,8 @@ describe("githubWebhookCoalesceKey", () => { }); it("coalesces gate-triggering pull request events and ignores non-actionable or terminal actions", () => { + // #9479: a push gets its OWN PR-scoped key, so consecutive pushes coalesce with each other rather than each + // minting a distinct head-SHA key. See the dedicated pr-push tests below. expect( githubWebhookCoalesceKey("pull_request", { action: "synchronize", @@ -32,7 +34,7 @@ describe("githubWebhookCoalesceKey", () => { number: 99, pull_request: {}, } as never), - ).toBe("github-webhook:pr-refresh:jsonbored/loopover#99"); + ).toBe("github-webhook:pr-push:jsonbored/loopover#99"); expect( githubWebhookCoalesceKey("pull_request", { action: "opened", @@ -56,7 +58,12 @@ describe("githubWebhookCoalesceKey", () => { // "reopened" triggers the same file-refresh path as "opened"/"synchronize" (PR_PUBLIC_SURFACE_ACTIONS in // src/queue/processors.ts) but was missing from the coalescable set — a burst of reopen-adjacent events for // the same PR+head fanned out one `/pulls/{n}/files` fetch per delivery instead of coalescing into one job. - const burstKeys = (["reopened", "synchronize", "ready_for_review"] as const).map((action) => + // + // #9479 narrowed this: "synchronize" now keys separately, deliberately. A push and a lifecycle event + // sharing one key meant a push could overwrite a still-pending "opened"/"ready_for_review" payload (the + // queue's coalesce keeps only the newest), losing that transition entirely -- so the push debounce is kept + // strictly to pushes, and the non-push burst still collapses exactly as it did. + const burstKeys = (["reopened", "ready_for_review"] as const).map((action) => githubWebhookCoalesceKey("pull_request", { action, repository: { full_name: "JSONbored/Loopover" }, @@ -65,6 +72,13 @@ describe("githubWebhookCoalesceKey", () => { ); expect(new Set(burstKeys).size).toBe(1); expect(burstKeys[0]).toBe("github-webhook:pr-refresh:jsonbored/loopover#100@beef123"); + expect( + githubWebhookCoalesceKey("pull_request", { + action: "synchronize", + repository: { full_name: "JSONbored/Loopover" }, + pull_request: { number: 100, head: { sha: "BEEF123" } }, + } as GitHubWebhookPayload), + ).not.toBe(burstKeys[0]); }); it("returns null for malformed or non-coalescible webhook shapes", () => { @@ -239,3 +253,76 @@ describe("githubWebhookCoalesceKey", () => { ).toBeNull(); }); }); + +// #9479: every dedup layer in the pipeline was keyed on the head SHA -- this key, and the AI-review lock's +// `...@${headSha}:${mode}` -- which is precisely the wrong key for a force-push storm, because each push MINTS a +// new SHA. Five amend-and-repush cycles inside a minute therefore looked like five unrelated events and bought +// five full prologues (file list, up to 96k chars of grounding fetch, RAG + impact-map embeddings, an enrichment +// POST) plus five LLM calls, for four heads that no longer exist by the time their reviews land. +describe("force-push debounce (#9479)", () => { + const push = (pr: number, sha: string) => + ({ + action: "synchronize", + repository: { full_name: "JSONbored/Loopover" }, + pull_request: { number: pr, head: { sha } }, + }) as unknown as GitHubWebhookPayload; + + it("REGRESSION: consecutive pushes to the SAME PR share one coalesce key despite different head SHAs", () => { + const keys = ["a".repeat(40), "b".repeat(40), "c".repeat(40), "d".repeat(40), "e".repeat(40)].map((sha) => + githubWebhookCoalesceKey("pull_request", push(7, sha)), + ); + + expect(new Set(keys).size).toBe(1); + expect(keys[0]).toBe("github-webhook:pr-push:jsonbored/loopover#7"); + }); + + it("INVARIANT: a non-push action with no head SHA still keys, just without the @sha suffix", () => { + // The lifecycle branch keeps its head-SHA scoping, but a payload that omits the head (some "edited" + // deliveries) must still produce a usable key rather than dropping out of coalescing entirely. + expect( + githubWebhookCoalesceKey("pull_request", { + action: "edited", + repository: { full_name: "JSONbored/Loopover" }, + pull_request: { number: 7 }, + } as GitHubWebhookPayload), + ).toBe("github-webhook:pr-refresh:jsonbored/loopover#7"); + }); + + it("INVARIANT: pushes to DIFFERENT PRs never coalesce with each other", () => { + expect(githubWebhookCoalesceKey("pull_request", push(7, "a".repeat(40)))).not.toBe( + githubWebhookCoalesceKey("pull_request", push(8, "a".repeat(40))), + ); + }); + + it("INVARIANT: a push never shares a key with a lifecycle event on the same PR, so it cannot overwrite a pending 'opened'", () => { + // The queue's coalesce keeps only the NEWEST payload, so a shared key would let a push inside the window + // discard an "opened"/"ready_for_review" transition entirely -- trading a spend bug for a lost-event bug. + const pushKey = githubWebhookCoalesceKey("pull_request", push(7, "a".repeat(40))); + for (const action of ["opened", "reopened", "edited", "ready_for_review"] as const) { + expect( + githubWebhookCoalesceKey("pull_request", { + action, + repository: { full_name: "JSONbored/Loopover" }, + pull_request: { number: 7, head: { sha: "a".repeat(40) } }, + } as GitHubWebhookPayload), + ).not.toBe(pushKey); + } + }); + + it("REGRESSION: only a push carries the trailing quiet window; every other delivery is enqueued immediately", () => { + expect(githubWebhookCoalesceDelaySeconds("pull_request", push(7, "a".repeat(40)))).toBe(PUSH_COALESCE_QUIET_WINDOW_SECONDS); + expect(PUSH_COALESCE_QUIET_WINDOW_SECONDS).toBeGreaterThan(0); + + for (const action of ["opened", "reopened", "edited", "ready_for_review", "closed", "labeled"] as const) { + expect( + githubWebhookCoalesceDelaySeconds("pull_request", { + action, + repository: { full_name: "JSONbored/Loopover" }, + pull_request: { number: 7, head: { sha: "a".repeat(40) } }, + } as GitHubWebhookPayload), + ).toBe(0); + } + // A same-named action on an unrelated event family must not pick up the delay either. + expect(githubWebhookCoalesceDelaySeconds("check_suite", { action: "synchronize", repository: { full_name: "JSONbored/Loopover" } } as unknown as GitHubWebhookPayload)).toBe(0); + }); +}); diff --git a/test/unit/queue-3.test.ts b/test/unit/queue-3.test.ts index 6bb9598204..d64fc99b17 100644 --- a/test/unit/queue-3.test.ts +++ b/test/unit/queue-3.test.ts @@ -2204,6 +2204,162 @@ describe("queue processors", () => { expect(deferAudit?.n).toBe(1); }); + // #9464: the THIRD blip class, and the one that reached production. #9030 covers a capture that THROWS and + // #9207 covers a preview that is still building; this is "the preview resolved fine, the RENDERER produced + // nothing, and nothing threw". captureShot swallows its own errors per shot, so buildCapture RESOLVES -- + // previewPending false, no exception, no real pair -- which is byte-identical to a legitimately evidence-free + // PR right up until the gate closes it one-shot. Browserless being down must not close contributor PRs. + it("screenshot-table gate (#9464): a RESOLVED capture whose renderer failed defers the close, exactly like a thrown one", async () => { + const buildCaptureSpy = vi.spyOn(visualCaptureModule, "buildCapture").mockResolvedValueOnce({ routes: [{ path: "/app", afterUrl: "https://worker.example/loopover/shot?url=x", afterUrlMobile: "https://worker.example/loopover/shot?url=x" }], interactions: [], previewPending: false, renderFailed: true }); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + LOOPOVER_REVIEW_SCREENSHOTS: "true", + }); + const sentJobs: Array> = []; + env.JOBS = { async send(message: Record) { sentJobs.push(message); } } as unknown as Queue; + await upsertInstallation(env, { + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" }, target_type: "User", repository_selection: "all", permissions: { metadata: "read", pull_requests: "write", issues: "write" }, events: ["pull_request"] }, + repositories: [{ name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }], + }); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + autonomy: { close: "auto", label: "auto" }, + }); + await upsertRepoFocusManifest(env, "JSONbored/gittensory", { settings: { commentMode: "all_prs", publicSurface: "comment_only", checkRunMode: "off", screenshotTableGate: { enabled: true, whenLabels: ["visual"] }, reviewCheckMode: "required" } }, "repo_file"); + const seen = { closed: false }; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") return Response.json([]); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/64/files")) return Response.json([{ filename: "apps/loopover-ui/src/routes/app.index.tsx", status: "modified", additions: 5, deletions: 1, changes: 6, patch: "@@\n+const ok = true;" }]); + if (url.includes("/pulls/64/reviews")) return Response.json([]); + if (url.includes("/pulls/64/commits")) return Response.json([]); + if (url.endsWith("/pulls/64") && method === "PATCH") { seen.closed = JSON.parse(String(init?.body ?? "{}")).state === "closed"; return Response.json({ number: 64, state: "closed" }); } + if (url.endsWith("/pulls/64")) return Response.json({ number: 64, state: "open", user: { login: "visual-contributor" }, head: { sha: "vis64" }, mergeable_state: "clean" }); + if (url.includes("/commits/vis64/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/commits/vis64/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/vis64/check-suites")) return Response.json({ check_suites: [] }); + if (url.includes("/issues/64/labels") && method === "GET") return Response.json([]); + if (url.includes("/issues/64/comments")) return Response.json([]); + if (url.endsWith("/labels") && method === "POST") return Response.json([]); + if (url.endsWith("/check-runs") && method === "POST") return Response.json({ id: 901 }, { status: 201 }); + if (url.includes("/check-runs/901") && method === "PATCH") return Response.json({ id: 901 }); + return new Response("not found", { status: 404 }); + }); + + try { + await processJob(env, { + type: "github-webhook", + deliveryId: "screenshot-table-render-failed-64", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { + number: 64, + title: "Update the app index route", + state: "open", + user: { login: "visual-contributor" }, + head: { sha: "vis64" }, + labels: [{ name: "visual" }], + body: "Changed the route layout, no table here.", + mergeable_state: "clean", + reviewDecision: "APPROVED", + }, + }, + }); + } finally { + buildCaptureSpy.mockRestore(); + } + + // The close must NOT fire: we cannot tell whether this PR has visual evidence, only that we failed to look. + expect(seen.closed).toBe(false); + const closeAudit = await env.DB.prepare("select count(*) as n from audit_events where event_type = 'agent.action.close'").first<{ n: number }>(); + expect(closeAudit?.n).toBe(0); + // Same bounded self-heal the other two blip classes get -- marker persisted for this head, retry scheduled. + const stored = await getPullRequest(env, "JSONbored/gittensory", 64); + expect(stored?.visualCaptureRetryPendingSha).toBe("vis64"); + expect(stored?.visualCaptureSatisfiedSha).toBeNull(); + expect(sentJobs).toContainEqual(expect.objectContaining({ type: "recapture-preview", repoFullName: "JSONbored/gittensory", prNumber: 64, attempt: 1 })); + }); + + // #9464 INVARIANT / #4110 guard: the fix must not neuter the gate. The single distinguishing bit is + // `renderFailed` -- an identical no-evidence capture from a HEALTHY renderer is a true negative and must + // still close. This is the test that an earlier "infer the blip from zero pairs" attempt broke. + it("screenshot-table gate (#9464 INVARIANT): a healthy renderer that found no evidence still CLOSES — the fix does not neuter the gate", async () => { + const buildCaptureSpy = vi.spyOn(visualCaptureModule, "buildCapture").mockResolvedValueOnce({ routes: [], interactions: [], previewPending: false, renderFailed: false }); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + LOOPOVER_REVIEW_SCREENSHOTS: "true", + }); + const sentJobs: Array> = []; + env.JOBS = { async send(message: Record) { sentJobs.push(message); } } as unknown as Queue; + await upsertInstallation(env, { + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" }, target_type: "User", repository_selection: "all", permissions: { metadata: "read", pull_requests: "write", issues: "write" }, events: ["pull_request"] }, + repositories: [{ name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }], + }); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + autonomy: { close: "auto", label: "auto" }, + }); + await upsertRepoFocusManifest(env, "JSONbored/gittensory", { settings: { commentMode: "all_prs", publicSurface: "comment_only", checkRunMode: "off", screenshotTableGate: { enabled: true, whenLabels: ["visual"] }, reviewCheckMode: "required" } }, "repo_file"); + const seen = { closed: false }; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") return Response.json([]); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/65/files")) return Response.json([{ filename: "apps/loopover-ui/src/routes/app.index.tsx", status: "modified", additions: 5, deletions: 1, changes: 6, patch: "@@\n+const ok = true;" }]); + if (url.includes("/pulls/65/reviews")) return Response.json([]); + if (url.includes("/pulls/65/commits")) return Response.json([]); + if (url.endsWith("/pulls/65") && method === "PATCH") { seen.closed = JSON.parse(String(init?.body ?? "{}")).state === "closed"; return Response.json({ number: 65, state: "closed" }); } + if (url.endsWith("/pulls/65")) return Response.json({ number: 65, state: "open", user: { login: "visual-contributor" }, head: { sha: "vis65" }, mergeable_state: "clean" }); + if (url.includes("/commits/vis65/status")) return Response.json({ state: "success", statuses: [] }); + if (url.includes("/commits/vis65/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/vis65/check-suites")) return Response.json({ check_suites: [] }); + if (url.includes("/issues/65/labels") && method === "GET") return Response.json([]); + if (url.includes("/issues/65/comments")) return Response.json([]); + if (url.endsWith("/labels") && method === "POST") return Response.json([]); + if (url.endsWith("/check-runs") && method === "POST") return Response.json({ id: 901 }, { status: 201 }); + if (url.includes("/check-runs/901") && method === "PATCH") return Response.json({ id: 901 }); + return new Response("not found", { status: 404 }); + }); + + try { + await processJob(env, { + type: "github-webhook", + deliveryId: "screenshot-table-render-failed-65", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { + number: 65, + title: "Update the app index route", + state: "open", + user: { login: "visual-contributor" }, + head: { sha: "vis65" }, + labels: [{ name: "visual" }], + body: "Changed the route layout, no table here.", + mergeable_state: "clean", + reviewDecision: "APPROVED", + }, + }, + }); + } finally { + buildCaptureSpy.mockRestore(); + } + + expect(seen.closed).toBe(true); + // No retry marker and no recapture job: there is nothing to retry, the answer was already definite. + const stored = await getPullRequest(env, "JSONbored/gittensory", 65); + expect(stored?.visualCaptureRetryPendingSha).toBeNull(); + expect(sentJobs.some((job) => job.type === "recapture-preview")).toBe(false); + }); + // #9030: sibling of the test above, but the recapture retry BUDGET is already exhausted (this delivery IS // itself the final recapture-preview attempt) -- proves the deferral cannot hold a PR closed forever: once // no retry chance remains, a still-erroring capture pipeline falls through to the gate's normal, accurate diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 692d4a6600..9c4592087e 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -1027,7 +1027,8 @@ describe("self-host queue common helpers", () => { }, }), ), - ).toBe("github-webhook:pr-refresh:jsonbored/gittensory#99"); + // #9479: a push keys on the PR alone so a force-push storm coalesces into one review. + ).toBe("github-webhook:pr-push:jsonbored/gittensory#99"); expect( jobCoalesceKey( payload({ diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 96a6ad265b..54bb473d8d 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -2,6 +2,7 @@ import { DatabaseSync } from "node:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; +import { PUSH_COALESCE_QUIET_WINDOW_SECONDS } from "../../src/github/webhook-coalesce"; import { PrActuationLockContendedError } from "../../src/queue/transient-locks"; import { ATTEMPT_FREE_RETRY_DEADLINE_MS } from "../../src/queue/retryable"; import { jobCoalesceKey, queueSnapshotFromBinding } from "../../src/selfhost/queue-common"; @@ -28,7 +29,7 @@ const webhook = (sender: { login: string; type: string }, eventName = "issue_com eventName, payload: { action, sender }, }) as unknown as JobMessage; -const prWebhook = (deliveryId: string, action = "synchronize", sha = "a".repeat(40)): JobMessage => +const prWebhook = (deliveryId: string, action = "synchronize", sha = "a".repeat(40), prNumber = 1629): JobMessage => ({ type: "github-webhook", deliveryId, @@ -36,7 +37,7 @@ const prWebhook = (deliveryId: string, action = "synchronize", sha = "a".repeat( payload: { action, repository: { full_name: "JSONbored/gittensory" }, - pull_request: { number: 1629, head: { sha } }, + pull_request: { number: prNumber, head: { sha } }, }, }) as unknown as JobMessage; const ciWebhook = (deliveryId: string, eventName: "check_suite" | "check_run" = "check_suite", sha = "b".repeat(40)): JobMessage => @@ -865,7 +866,8 @@ describe("createSqliteQueue (durable #980)", () => { expect(rows.map((row) => row.job_key).sort()).toEqual([ "agent-regate-sweep:all", `github-webhook:ci-completed:jsonbored/gittensory@${"b".repeat(40)}#1629`, - `github-webhook:pr-refresh:jsonbored/gittensory#1629@${"a".repeat(40)}`, + // #9479: a push keys on the PR alone (no head SHA), so a re-push collapses into the pending review. + "github-webhook:pr-push:jsonbored/gittensory#1629", ]); expect(rows.map((row) => JSON.parse(row.payload).deliveryId).filter(Boolean).sort()).toEqual(["ci-2", "pr-2"]); expect(await q.stats()).toMatchObject({ @@ -947,12 +949,46 @@ describe("createSqliteQueue (durable #980)", () => { expect(event?.status).toBe("queued"); // untouched -- there is nothing genuinely superseded here }); + // #9479: the end-to-end shape of a force-push storm. Each push mints a new head SHA, so under the old + // SHA-scoped key these were five independent jobs -- five full review prologues and five LLM calls, four of + // them for heads that no longer exist. The PR-scoped key plus the trailing quiet window collapses them into + // ONE review, of the head that actually survived. + it("REGRESSION (#9479): five rapid pushes to one PR collapse into a single job carrying the FINAL head", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + const shas = ["a", "b", "c", "d", "e"].map((c) => c.repeat(40)); + for (const [index, sha] of shas.entries()) { + await q.binding.send(prWebhook(`push-${index + 1}`, "synchronize", sha), { delaySeconds: PUSH_COALESCE_QUIET_WINDOW_SECONDS }); + } + + const rows = driver.query("SELECT payload, job_key FROM _selfhost_jobs ORDER BY id", []).rows as Array<{ payload: string; job_key: string }>; + expect(rows).toHaveLength(1); + expect(rows[0]!.job_key).toBe("github-webhook:pr-push:jsonbored/gittensory#1629"); + const payload = JSON.parse(rows[0]!.payload) as { deliveryId: string; payload: { pull_request: { head: { sha: string } } } }; + // The SURVIVING head, not the first one -- reviewing an abandoned intermediate SHA is the waste being fixed. + expect(payload.deliveryId).toBe("push-5"); + expect(payload.payload.pull_request.head.sha).toBe(shas[4]); + }); + + it("INVARIANT (#9479): the quiet window EXTENDS on each push, so the review runs after the storm settles rather than mid-burst", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + await q.binding.send(prWebhook("push-1", "synchronize", "a".repeat(40)), { delaySeconds: PUSH_COALESCE_QUIET_WINDOW_SECONDS }); + const first = (driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }).run_after; + // A later push must push the deadline OUT (GREATEST(run_after, ...)), never pull it in -- otherwise the + // first push in a burst would fix the review time and the debounce would only ever save the tail. + await q.binding.send(prWebhook("push-2", "synchronize", "b".repeat(40)), { delaySeconds: PUSH_COALESCE_QUIET_WINDOW_SECONDS * 2 }); + + const second = (driver.query("SELECT run_after FROM _selfhost_jobs", []).rows[0] as { run_after: number }).run_after; + expect(second).toBeGreaterThan(first); + }); + it("tolerates an unparseable existing payload (a corrupted row) without throwing", async () => { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined); driver.query( `INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority, job_key) VALUES (?, 'pending', 0, ?, ?, 5, ?)`, - ["not valid json{", Date.now() + 60_000, Date.now(), `github-webhook:pr-refresh:jsonbored/gittensory#1629@${"a".repeat(40)}`], + ["not valid json{", Date.now() + 60_000, Date.now(), "github-webhook:pr-push:jsonbored/gittensory#1629"], ); await expect(q.binding.send(prWebhook("pr-2"), { delaySeconds: 1 })).resolves.toBeUndefined(); @@ -1491,10 +1527,11 @@ describe("createSqliteQueue (durable #980)", () => { const seen: string[] = []; const q = createSqliteQueue(driver, async (m) => void seen.push((m as unknown as { deliveryId: string }).deliveryId), { concurrency: 1 }); for (let i = 1; i <= 6; i++) await q.binding.send(backlogJob("owner/repo", i)); - // Distinct head SHAs -- prWebhook's coalesce key is repo#pr@headSha, so two same-PR events with the - // SAME default sha would coalesce into one row instead of two. - await q.binding.send(prWebhook("fresh-1", "synchronize", "a".repeat(40))); - await q.binding.send(prWebhook("fresh-2", "synchronize", "b".repeat(40))); + // Distinct PR NUMBERS -- #9479 keys a push on repo#pr alone (deliberately, so a force-push storm + // coalesces), so two same-PR pushes now collapse into one row no matter how their head SHAs differ. This + // test is about the fairness RATIO, so it needs two genuinely independent fresh-intake jobs. + await q.binding.send(prWebhook("fresh-1", "synchronize", "a".repeat(40), 1629)); + await q.binding.send(prWebhook("fresh-2", "synchronize", "b".repeat(40), 1630)); await q.drain(); expect(seen).toEqual([ "backlog-convergence:owner/repo#1", diff --git a/test/unit/visual-capture.test.ts b/test/unit/visual-capture.test.ts index 3175d96654..2ef7a5821d 100644 --- a/test/unit/visual-capture.test.ts +++ b/test/unit/visual-capture.test.ts @@ -770,7 +770,7 @@ describe("buildCapture pixel-diff wiring (#3674)", () => { }); it("returns just the URL (no bytes) for a fresh successful render when diffing is unavailable — the real default", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([5, 5, 5]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([5, 5, 5]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", @@ -796,7 +796,7 @@ describe("buildCapture pixel-diff wiring (#3674)", () => { it("threads fresh screenshot bytes to the diff provider right after a successful render, not just on a cache hit", async () => { const availableSpy = vi.spyOn(pixelDiffModule, "isVisualDiffAvailable").mockReturnValue(true); const compareSpy = vi.spyOn(pixelDiffModule, "compareCapturedScreenshots").mockResolvedValue(null); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", @@ -854,7 +854,7 @@ describe("buildCapture pixel-diff wiring (#3674)", () => { describe("buildCapture display-thumbnail wiring (#6324)", () => { it("never attempts a thumbnail when display downscaling is unavailable (the real, unmocked default) — byte-identical to pre-#6324", async () => { const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay"); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const result = await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -874,7 +874,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { it("generates + stores a thumbnail and threads beforeThumbUrl/afterThumbUrl when downscaling is available and genuinely shrinks the image", async () => { const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockResolvedValue(new Uint8Array([1])); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }); const result = await buildCapture( @@ -900,7 +900,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { it("also generates a thumbnail for the mobile viewport when downscaling is available (bug fix — a full-page mobile capture is height-unbounded even though 390px width is already narrow)", async () => { const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockResolvedValue(new Uint8Array([1])); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }); const result = await buildCapture( @@ -929,7 +929,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { const same = new Uint8Array([9, 9, 9]); const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockResolvedValue(same); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: same, authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: same, authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }); const result = await buildCapture( @@ -951,7 +951,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { it("degrades to the original bytes (never throws) when downscaleForDisplay itself rejects", async () => { const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockRejectedValue(new Error("simulated decode failure")); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const result = await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1021,7 +1021,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { const originalAfter = new Uint8Array([40, 50, 60]); const captureShotSpy = vi .spyOn(shotModule, "captureShot") - .mockImplementation(async (_env, url: string) => ({ png: url.includes("preview.example.com") ? originalAfter : originalBefore, authWalled: false })); + .mockImplementation(async (_env, url: string) => ({ png: url.includes("preview.example.com") ? originalAfter : originalBefore, authWalled: false, renderFailed: false })); try { await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1044,7 +1044,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { it("falls back to no thumbUrl (never throws) when the thumb-key WRITE itself fails on a fresh render, even though the original write succeeded", async () => { const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockResolvedValue(new Uint8Array([1])); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const beforeThumb = await thumbKey(49, "before", "desktop", "https://prod.example.com/app"); const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit({ failPutKeys: [beforeThumb] }) }); @@ -1089,7 +1089,7 @@ describe("buildCapture display-thumbnail wiring (#6324)", () => { it("links a thumbnail directly at the bucket instead of this instance's /loopover/shot proxy when REVIEW_AUDIT_S3_PUBLIC_URL is configured", async () => { const availableSpy = vi.spyOn(imageDownscaleModule, "isDisplayDownscaleAvailable").mockReturnValue(true); const downscaleSpy = vi.spyOn(imageDownscaleModule, "downscaleForDisplay").mockResolvedValue(new Uint8Array([1])); - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", @@ -1283,7 +1283,7 @@ describe("buildCapture theme matrix (#3678)", () => { }); it("passes the configured theme through to captureShot's render options", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false, renderFailed: false }); try { await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1302,7 +1302,7 @@ describe("buildCapture theme matrix (#3678)", () => { }); it("never passes a theme option to captureShot when no themes are configured", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false, renderFailed: false }); try { await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1352,7 +1352,7 @@ describe("buildCapture theme matrix (#3678)", () => { describe("buildCapture theme-storage-key wiring (#4109)", () => { it("passes themeStorageKey through to captureShot's render options when both themes and theme_storage_key are configured", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false, renderFailed: false }); try { await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1371,7 +1371,7 @@ describe("buildCapture theme-storage-key wiring (#4109)", () => { }); it("never passes themeStorageKey to captureShot when no themes are configured, even if theme_storage_key is set", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false, renderFailed: false }); try { await buildCapture( createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }), @@ -1417,7 +1417,7 @@ describe("buildCapture theme-storage-key wiring (#4109)", () => { }); it("threads the theme storage key into the shot fingerprint too, so it never collides with an untagged-key capture of the same theme", async () => { - const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false }); + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); try { const env = createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }); const result = await buildCapture( @@ -3204,3 +3204,82 @@ describe("fetchExternalScreenshotContentBlock", () => { await expect(fetchExternalScreenshotContentBlock("https://example.com/first.png")).resolves.toBeUndefined(); }); }); + +// #9464: a browserless outage was invisible to the screenshot-table gate. captureShot swallows its own renderer +// errors and returns a null PNG, so buildCapture came back NORMALLY -- previewPending false, nothing thrown -- +// and neither the #9030 (thrown-error) nor the #9207 (previewPending) blip guard fired. The maintenance pass then +// read "no visual evidence, no retry pending" and CLOSED the PR one-shot, which is unrecoverable by policy: a +// contributor's only remedy is opening a fresh PR. The last time this gate misfired it closed five of them. +// +// The signal is deliberately "a render THREW", not "zero pairs were produced". The latter is also true when the +// author genuinely supplied no visual evidence, which is the case the gate exists to close -- inferring the blip +// from an empty result would have neutered the gate entirely. +describe("renderer-failure blip signal (#9464)", () => { + const target = { repoFullName: "owner/repo", prNumber: 9464, previewUrl: "https://preview.example.com" }; + const capturedEnv = () => + createTestEnv({ PUBLIC_API_ORIGIN: "https://worker.example", PUBLIC_SITE_ORIGIN: "https://prod.example.com", REVIEW_AUDIT: memoryReviewAudit() }); + + it("REGRESSION: a renderer that fails on every shot reports renderFailed, so the gate's close is deferred instead of firing on a browserless outage", async () => { + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: false, renderFailed: true }); + try { + const result = await buildCapture(capturedEnv(), "installation-token", target, ["apps/loopover-ui/src/routes/app.index.tsx"]); + + expect(result.renderFailed).toBe(true); + // The precise shape that used to fool the gate: nothing pending, nothing thrown, and no real pair. + expect(result.previewPending).toBe(false); + expect(hasSuccessfulBotCapture(result.routes)).toBe(false); + } finally { + captureShotSpy.mockRestore(); + } + }); + + it("INVARIANT: a healthy renderer never reports renderFailed, so a genuine no-evidence PR still closes (#4110 guard)", async () => { + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false }); + try { + const result = await buildCapture(capturedEnv(), "installation-token", target, ["apps/loopover-ui/src/routes/app.index.tsx"]); + + expect(result.renderFailed).toBe(false); + } finally { + captureShotSpy.mockRestore(); + } + }); + + it("INVARIANT: an auth-walled route is a DEFINITE answer, not a renderer failure — it must not defer the gate", async () => { + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockResolvedValue({ png: null, authWalled: true, renderFailed: false }); + try { + const result = await buildCapture(capturedEnv(), "installation-token", target, ["apps/loopover-ui/src/routes/app.index.tsx"]); + + // No real pair either -- which is exactly why "zero pairs" could not have been the signal. + expect(hasSuccessfulBotCapture(result.routes)).toBe(false); + expect(result.renderFailed).toBe(false); + } finally { + captureShotSpy.mockRestore(); + } + }); + + it("REGRESSION: ONE failed shot among otherwise-healthy ones still reports renderFailed — a partial outage can still have hidden evidence", async () => { + let call = 0; + const captureShotSpy = vi + .spyOn(shotModule, "captureShot") + .mockImplementation(async () => (call++ === 0 ? { png: null, authWalled: false, renderFailed: true } : { png: new Uint8Array([9, 9, 9]), authWalled: false, renderFailed: false })); + try { + const result = await buildCapture(capturedEnv(), "installation-token", target, ["apps/loopover-ui/src/routes/app.index.tsx"]); + + expect(result.renderFailed).toBe(true); + } finally { + captureShotSpy.mockRestore(); + } + }); + + it("REGRESSION: captureShot REJECTING (rather than degrading) also reports renderFailed — the outer catch must not launder it into a clean result", async () => { + const captureShotSpy = vi.spyOn(shotModule, "captureShot").mockRejectedValue(new Error("dns guard exploded")); + try { + const result = await buildCapture(capturedEnv(), "installation-token", target, ["apps/loopover-ui/src/routes/app.index.tsx"]); + + expect(result.renderFailed).toBe(true); + } finally { + captureShotSpy.mockRestore(); + } + }); + +}); diff --git a/test/unit/visual-shot.test.ts b/test/unit/visual-shot.test.ts index b6151644aa..ac3230ef2a 100644 --- a/test/unit/visual-shot.test.ts +++ b/test/unit/visual-shot.test.ts @@ -256,7 +256,7 @@ describe("visual screenshot on-demand SSRF guard", () => { const result = captureShot(env(), "https://preview.pages.dev/page"); await vi.advanceTimersByTimeAsync(10_000); - await expect(result).resolves.toEqual({ png: null, authWalled: false }); + await expect(result).resolves.toEqual({ png: null, authWalled: false, renderFailed: false }); } finally { vi.useRealTimers(); } @@ -271,7 +271,7 @@ describe("visual screenshot on-demand SSRF guard", () => { const result = captureShot(env(), "https://preview.pages.dev/page"); await vi.advanceTimersByTimeAsync(2_000); - await expect(result).resolves.toEqual({ png: null, authWalled: false }); + await expect(result).resolves.toEqual({ png: null, authWalled: false, renderFailed: false }); expect(mocks.screenshot).not.toHaveBeenCalled(); expect(mocks.close).toHaveBeenCalled(); } finally { @@ -336,7 +336,7 @@ describe("visual screenshot on-demand SSRF guard", () => { const result = captureShot(env(), "https://preview.pages.dev/page", undefined, { theme: "dark", themeStorageKey: "theme" }); await vi.advanceTimersByTimeAsync(2_000); - await expect(result).resolves.toEqual({ png: null, authWalled: false }); + await expect(result).resolves.toEqual({ png: null, authWalled: false, renderFailed: false }); expect(mocks.reload).not.toHaveBeenCalled(); expect(mocks.screenshot).not.toHaveBeenCalled(); expect(mocks.close).toHaveBeenCalled(); @@ -374,7 +374,7 @@ describe("visual screenshot on-demand SSRF guard", () => { it("captureShot rejects an unsafe target before launching the browser (defense-in-depth)", async () => { const result = await captureShot(env(), "http://127.0.0.1/admin"); - expect(result).toEqual({ png: null, authWalled: false }); + expect(result).toEqual({ png: null, authWalled: false, renderFailed: false }); expect(mocks.launch).not.toHaveBeenCalled(); }); @@ -528,6 +528,13 @@ describe("captureScrollFrames (#3612 scroll-through GIF evidence)", () => { expect(mocks.close).toHaveBeenCalled(); }); + // #9464: an auth wall is a DEFINITE answer about the page, not a renderer failure -- it must never set + // renderFailed, or a repo with one protected route would defer the screenshot-table gate on every review. + it("flags authWalled WITHOUT renderFailed on a login-page redirect (#9464)", async () => { + mocks.finalUrl = "https://preview.pages.dev/login"; + await expect(captureShot(env(), "https://preview.pages.dev/dashboard")).resolves.toEqual({ png: null, authWalled: true, renderFailed: false }); + }); + it("flags authWalled and captures no frames on a login-page redirect", async () => { mocks.finalUrl = "https://preview.pages.dev/login"; const result = await captureScrollFrames(env(), "https://preview.pages.dev/dashboard", { width: 1440, height: 900 }); @@ -949,7 +956,7 @@ describe("DNS-resolution pin (#9044, best-effort SSRF defense-in-depth)", () => it("captureShot rejects a public hostname whose DNS resolves to a disallowed address, before launching the browser", async () => { mocks.dnsLookup.mockResolvedValueOnce({ address: "169.254.169.254", family: 4 }); // cloud metadata const result = await captureShot(env(), "https://attacker-controlled.pages.dev/page"); - expect(result).toEqual({ png: null, authWalled: false }); + expect(result).toEqual({ png: null, authWalled: false, renderFailed: false }); expect(mocks.launch).not.toHaveBeenCalled(); }); @@ -979,7 +986,7 @@ describe("DNS-resolution pin (#9044, best-effort SSRF defense-in-depth)", () => mocks.dnsLookup.mockReturnValueOnce(new Promise(() => undefined)); const result = captureShot(env(), "https://preview.pages.dev/page"); await vi.advanceTimersByTimeAsync(1_500); - await expect(result).resolves.toEqual({ png: expect.any(Uint8Array), authWalled: false }); + await expect(result).resolves.toEqual({ png: expect.any(Uint8Array), authWalled: false, renderFailed: false }); } finally { vi.useRealTimers(); } @@ -1100,7 +1107,7 @@ describe("visual capture result metric (#9487)", () => { resetMetrics(); mocks.screenshot.mockRejectedValueOnce(new Error("Protocol error: Target closed")); - await expect(captureShot(env(), "https://preview.pages.dev/page")).resolves.toEqual({ png: null, authWalled: false }); + await expect(captureShot(env(), "https://preview.pages.dev/page")).resolves.toEqual({ png: null, authWalled: false, renderFailed: true }); expect(counterValue("loopover_visual_capture_total", { result: "error" })).toBe(1); }); diff --git a/test/unit/webhook.test.ts b/test/unit/webhook.test.ts index 0a689f8431..983f2d738e 100644 --- a/test/unit/webhook.test.ts +++ b/test/unit/webhook.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Context } from "hono"; import { enqueueWebhookByEnv, handleGitHubWebhook, handleOrbRelay } from "../../src/github/webhook"; +import { PUSH_COALESCE_QUIET_WINDOW_SECONDS } from "../../src/github/webhook-coalesce"; import { getWebhookEvent, recordWebhookEvent } from "../../src/db/repositories"; import { relaySignature } from "../../src/orb/relay"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; @@ -818,3 +819,35 @@ async function signWebhook(body: string, secret: string | undefined): Promise byte.toString(16).padStart(2, "0")).join("")}`; } + +// #9479: on a coalescing queue the delay is what CREATES the debounce window -- without it the first push of a +// burst is claimed immediately and every later push finds no pending row to merge into, so the PR-scoped +// coalesce key alone would save nothing. +describe("push quiet window on enqueue (#9479)", () => { + const enqueue = (env: Env, deliveryId: string, action: string) => + enqueueWebhookByEnv( + env, + deliveryId, + "pull_request", + JSON.stringify({ action, repository: { full_name: "JSONbored/gittensory" }, installation: { id: 1 }, pull_request: { number: 3, head: { sha: "a".repeat(40) } } }), + ); + + it("REGRESSION: a push is enqueued with the trailing quiet window", async () => { + const env = createTestEnv(); + const delays: Array = []; + env.WEBHOOKS = { send: async (_m: unknown, opts?: { delaySeconds?: number }) => void delays.push(opts?.delaySeconds) } as unknown as Queue; + + expect(await enqueue(env, "push-window-1", "synchronize")).toBe("queued"); + expect(delays).toEqual([PUSH_COALESCE_QUIET_WINDOW_SECONDS]); + }); + + it("INVARIANT: every other action is still enqueued with no delay — review latency is unchanged for them", async () => { + const env = createTestEnv(); + const delays: Array = []; + env.WEBHOOKS = { send: async (_m: unknown, opts?: { delaySeconds?: number }) => void delays.push(opts?.delaySeconds) } as unknown as Queue; + + expect(await enqueue(env, "push-window-2", "opened")).toBe("queued"); + expect(await enqueue(env, "push-window-3", "closed")).toBe("queued"); + expect(delays).toEqual([0, 0]); + }); +});