diff --git a/packages/loopover-miner/lib/calibration.ts b/packages/loopover-miner/lib/calibration.ts index c50a8f8e5..d07553867 100644 --- a/packages/loopover-miner/lib/calibration.ts +++ b/packages/loopover-miner/lib/calibration.ts @@ -58,6 +58,33 @@ function recordKey(project: string, targetId: string): string { return `${project} ${targetId}`; } +/** Build the same `(project, targetId)` → normalized-outcome map {@link buildCalibrationReport} uses internally. + * Malformed outcome records are skipped. Exported so metrics-cli (#8315) can reuse the join without a second + * implementation. */ +export function buildOutcomeDecisionMap(outcomes: ObservedOutcomeRecord[]): Map { + const outcomeByKey = new Map(); + for (const outcome of Array.isArray(outcomes) ? outcomes : []) { + if (!isObservedOutcomeRecord(outcome)) continue; + outcomeByKey.set(recordKey(outcome.project, outcome.targetId), normalizeDecision(outcome.outcomeDecision)); + } + return outcomeByKey; +} + +/** Resolve one prediction's `correct` flag using the same join rules as {@link buildCalibrationReport}: only + * directional merge/close predictions with a realized merge/close outcome are scored; hold, pending, and + * unclassifiable rows return `undefined` (unset). Malformed predictions return `undefined`. */ +export function resolvePredictionCorrectness( + prediction: PredictedVerdictRecord, + outcomeByKey: Map, +): boolean | undefined { + if (!isPredictedVerdictRecord(prediction)) return undefined; + const observed = outcomeByKey.get(recordKey(prediction.project, prediction.targetId)); + if (observed !== "merge" && observed !== "close") return undefined; + const predicted = normalizeDecision(prediction.predictedDecision); + if (predicted !== "merge" && predicted !== "close") return undefined; + return predicted === observed; +} + /** * Join predicted-verdict records with realized-outcome records into a per-project calibration report. Pure and * read-only. A prediction counts as "decided" only when a realized outcome for the SAME `(project, targetId)` @@ -70,11 +97,7 @@ export function buildCalibrationReport( predictions: PredictedVerdictRecord[], outcomes: ObservedOutcomeRecord[], ): CalibrationReport { - const outcomeByKey = new Map(); - for (const outcome of Array.isArray(outcomes) ? outcomes : []) { - if (!isObservedOutcomeRecord(outcome)) continue; - outcomeByKey.set(recordKey(outcome.project, outcome.targetId), normalizeDecision(outcome.outcomeDecision)); - } + const outcomeByKey = buildOutcomeDecisionMap(outcomes); const byProject = new Map(); for (const prediction of Array.isArray(predictions) ? predictions : []) { diff --git a/packages/loopover-miner/lib/metrics-cli.ts b/packages/loopover-miner/lib/metrics-cli.ts index a94bb75bc..640e2763a 100644 --- a/packages/loopover-miner/lib/metrics-cli.ts +++ b/packages/loopover-miner/lib/metrics-cli.ts @@ -1,25 +1,40 @@ import { renderMinerPredictionMetrics } from "@loopover/engine"; import type { MinerPredictionMetricRow } from "@loopover/engine"; +import { buildOutcomeDecisionMap, resolvePredictionCorrectness } from "./calibration.js"; +import type { ObservedOutcomeRecord } from "./calibration-types.js"; +import { toOutcomeRecords, toPredictionRecords } from "./calibration-cli.js"; +import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; +import type { EventLedger } from "./event-ledger.js"; import { initPredictionLedger } from "./prediction-ledger.js"; import type { PredictionLedger } from "./prediction-ledger.js"; import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js"; // `metrics` (#4838): render the miner's prediction-calibration counters as Prometheus text-exposition to stdout, // for a scrape wrapper or cron redirect. The counters are produced by the engine's already-built -// renderMinerPredictionMetrics (packages/loopover-engine/src/miner-prediction-metrics.ts) -- this command only -// reads the local prediction ledger and feeds it in, never touching the renderer itself. Strictly local + offline: -// no network, no writes. +// renderMinerPredictionMetrics (packages/loopover-engine/src/miner-prediction-metrics.ts) -- this command reads +// the local prediction ledger, joins each row with realized `pr_outcome` events from the event ledger +// (calibration-cli.js's toPredictionRecords/toOutcomeRecords + calibration.js's join), and feeds the resolved +// rows to the renderer. Strictly local + offline: no network, no writes. const METRICS_USAGE = "Usage: loopover-miner metrics"; /** - * Project prediction-ledger rows onto the engine renderer's metric-row shape -- the predicted `conclusion` only. - * The realized-outcome pairing (`correct`) is intentionally left unset: the miner has no outcome-join yet, so the - * correct/incorrect counters stay zero and only `predictions_total{conclusion}` moves -- exactly how the renderer - * is designed to degrade before outcome-pairing exists (see its header comment). + * Project prediction-ledger rows onto the engine renderer's metric-row shape, pairing each predicted `conclusion` + * with a realized outcome when one exists for the same `(repoFullName, targetId)`. Reuses calibration-cli.js's + * record mappers and calibration.js's {@link resolvePredictionCorrectness} so the join matches + * `buildCalibrationReport` exactly. */ -export function collectPredictionMetricRows(ledger: PredictionLedger): MinerPredictionMetricRow[] { - return ledger.readPredictions().map((entry) => ({ conclusion: entry.conclusion })); +export function collectPredictionMetricRows( + ledger: PredictionLedger, + outcomes: ObservedOutcomeRecord[] = [], +): MinerPredictionMetricRow[] { + const outcomeByKey = buildOutcomeDecisionMap(outcomes); + return toPredictionRecords(ledger.readPredictions()).map((prediction) => { + const correct = resolvePredictionCorrectness(prediction, outcomeByKey); + const row: MinerPredictionMetricRow = { conclusion: prediction.predictedDecision }; + if (correct !== undefined) row.correct = correct; + return row; + }); } // Open the local prediction ledger (or a test-injected one) for the duration of `run`, closing it only when we @@ -37,19 +52,34 @@ function withPredictionLedger( } } -export function runMetrics(args: string[], options: { initPredictionLedger?: () => PredictionLedger } = {}): number { +export function runMetrics( + args: string[], + options: { + initPredictionLedger?: () => PredictionLedger; + initEventLedger?: () => EventLedger; + env?: Record; + } = {}, +): number { if (args.length > 0) { return reportCliFailure(argsWantJson(args), METRICS_USAGE); } + const env = options.env ?? process.env; + let eventLedger: EventLedger | undefined; + const ownsEventLedger = options.initEventLedger === undefined; + try { return withPredictionLedger(options, (ledger) => { + eventLedger = (options.initEventLedger ?? (() => initEventLedger(resolveEventLedgerDbPath(env))))(); + const outcomes = toOutcomeRecords(eventLedger.readEvents()); // renderMinerPredictionMetrics returns a newline-terminated document; console.log re-adds the terminator, so // trim it to emit exactly one trailing newline. - console.log(renderMinerPredictionMetrics(collectPredictionMetricRows(ledger)).trimEnd()); + console.log(renderMinerPredictionMetrics(collectPredictionMetricRows(ledger, outcomes)).trimEnd()); return 0; }); } catch (error) { return reportCliFailure(argsWantJson(args), describeCliError(error)); + } finally { + if (ownsEventLedger) eventLedger?.close(); } } diff --git a/test/unit/miner-calibration.test.ts b/test/unit/miner-calibration.test.ts index 01e333f6f..db8228041 100644 --- a/test/unit/miner-calibration.test.ts +++ b/test/unit/miner-calibration.test.ts @@ -1,13 +1,18 @@ import { describe, expect, it } from "vitest"; -import { - buildCalibrationReport, - isCalibrationReport, -} from "../../packages/loopover-miner/lib/calibration.js"; import type { ObservedOutcomeRecord, PredictedVerdictRecord, } from "../../packages/loopover-miner/lib/calibration.js"; +// Same .ts-via-variable import as miner-metrics-cli.test.ts (#8315) — CI grades patch on the .ts paths. +const CALIBRATION_MODULE = "../../packages/loopover-miner/lib/calibration.ts"; +const { + buildCalibrationReport, + buildOutcomeDecisionMap, + isCalibrationReport, + resolvePredictionCorrectness, +} = (await import(CALIBRATION_MODULE)) as typeof import("../../packages/loopover-miner/lib/calibration.js"); + const TS = "2026-07-12T00:00:00.000Z"; const pred = (project: string, targetId: string, predictedDecision: string): PredictedVerdictRecord => ({ project, @@ -113,3 +118,21 @@ describe("buildCalibrationReport (#4849)", () => { expect(report.hasSignal).toBe(false); // outcome belongs to a different project }); }); + +describe("resolvePredictionCorrectness (#8315)", () => { + it("scores directional predictions and leaves hold/pending/unrecognized unset", () => { + const outcomeByKey = buildOutcomeDecisionMap([ + out("a/b", "1", "merged"), + out("a/b", "2", "closed"), + out("a/b", "3", "closed"), + out("a/b", "4", "unknown"), + ]); + expect(resolvePredictionCorrectness(pred("a/b", "1", "merge"), outcomeByKey)).toBe(true); + expect(resolvePredictionCorrectness(pred("a/b", "2", "merge"), outcomeByKey)).toBe(false); + expect(resolvePredictionCorrectness(pred("a/b", "3", "close"), outcomeByKey)).toBe(true); + expect(resolvePredictionCorrectness(pred("a/b", "5", "merge"), outcomeByKey)).toBeUndefined(); + expect(resolvePredictionCorrectness(pred("a/b", "3", "hold"), outcomeByKey)).toBeUndefined(); + expect(resolvePredictionCorrectness(pred("a/b", "4", "merge"), outcomeByKey)).toBeUndefined(); + expect(resolvePredictionCorrectness({ project: "a/b" } as never, outcomeByKey)).toBeUndefined(); + }); +}); diff --git a/test/unit/miner-metrics-cli.test.ts b/test/unit/miner-metrics-cli.test.ts index 9fbf1b800..373bc28d2 100644 --- a/test/unit/miner-metrics-cli.test.ts +++ b/test/unit/miner-metrics-cli.test.ts @@ -2,13 +2,20 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { toOutcomeRecords } from "../../packages/loopover-miner/lib/calibration-cli.js"; +import { initEventLedger } from "../../packages/loopover-miner/lib/event-ledger.js"; +import type { LedgerEntry } from "../../packages/loopover-miner/lib/event-ledger.js"; import { initPredictionLedger } from "../../packages/loopover-miner/lib/prediction-ledger.js"; -import { - collectPredictionMetricRows, - runMetrics, -} from "../../packages/loopover-miner/lib/metrics-cli.js"; import type { PredictionLedger } from "../../packages/loopover-miner/lib/prediction-ledger.d.ts"; +// Import the .ts SOURCE (not the build-time .js) via a non-literal specifier. After `build:miner`, a plain +// `.js` import loads the compiled artifact and leaves coverage.include's `.ts` entry at 0% under CI's +// `--changed=origin/main --coverage.all=false` run (#8315, same pattern as miner-replay-snapshot.test.ts). +const METRICS_CLI_MODULE = "../../packages/loopover-miner/lib/metrics-cli.ts"; +const { collectPredictionMetricRows, runMetrics } = (await import(METRICS_CLI_MODULE)) as typeof import("../../packages/loopover-miner/lib/metrics-cli.js"); + +const REPO = "acme/widgets"; + const roots: string[] = []; const ledgers: Array<{ close(): void }> = []; @@ -20,6 +27,14 @@ function tempLedger(): PredictionLedger { return ledger; } +function tempEventLedger() { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-metrics-cli-event-")); + roots.push(root); + const ledger = initEventLedger(join(root, "event-ledger.sqlite3")); + ledgers.push(ledger); + return ledger; +} + function tempDbPath() { const root = mkdtempSync(join(tmpdir(), "loopover-miner-metrics-cli-")); roots.push(root); @@ -27,7 +42,18 @@ function tempDbPath() { } function appendPrediction(ledger: PredictionLedger, targetId: number, conclusion: string) { - ledger.appendPrediction({ repoFullName: "acme/widgets", targetId, conclusion, pack: "gittensor", engineVersion: "0.2.0" }); + ledger.appendPrediction({ repoFullName: REPO, targetId, conclusion, pack: "gittensor", engineVersion: "0.2.0" }); +} + +function prOutcome(prNumber: number, decision: string, repoFullName = REPO): LedgerEntry { + return { + id: prNumber, + seq: prNumber, + type: "pr_outcome", + repoFullName, + payload: { prNumber, decision }, + createdAt: "2026-07-08T00:00:00.000Z", + }; } afterEach(() => { @@ -36,57 +62,122 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -describe("loopover-miner metrics CLI (#4838)", () => { - it("collectPredictionMetricRows projects ledger rows onto the renderer's conclusion-only shape", () => { +describe("loopover-miner metrics CLI (#4838, #8315 outcome join)", () => { + it("collectPredictionMetricRows leaves correct unset when no outcomes are supplied", () => { const ledger = tempLedger(); appendPrediction(ledger, 1, "merge"); appendPrediction(ledger, 2, "close"); expect(collectPredictionMetricRows(ledger)).toEqual([{ conclusion: "merge" }, { conclusion: "close" }]); }); - it("runMetrics renders prediction counters as Prometheus text and returns 0", () => { + it("collectPredictionMetricRows resolves merge/close predictions against realized outcomes", () => { + const ledger = tempLedger(); + appendPrediction(ledger, 1, "merge"); // confirmed + appendPrediction(ledger, 2, "merge"); // false — closed instead + appendPrediction(ledger, 3, "close"); // confirmed + appendPrediction(ledger, 4, "close"); // false — merged instead + appendPrediction(ledger, 5, "hold"); // never scored + appendPrediction(ledger, 6, "merge"); // pending — no outcome yet + + const outcomes = toOutcomeRecords([ + prOutcome(1, "merged"), + prOutcome(2, "closed"), + prOutcome(3, "closed"), + prOutcome(4, "merged"), + prOutcome(5, "closed"), + // pr 6 intentionally omitted + // malformed pr_outcome — skipped by toOutcomeRecords + { id: 99, seq: 99, type: "pr_outcome", repoFullName: REPO, payload: { prNumber: 7, decision: "unknown" }, createdAt: "2026-07-08T00:00:00.000Z" }, + { id: 100, seq: 100, type: "pr_outcome", repoFullName: REPO, payload: { prNumber: "bad" }, createdAt: "2026-07-08T00:00:00.000Z" }, + ]); + + expect(collectPredictionMetricRows(ledger, outcomes)).toEqual([ + { conclusion: "merge", correct: true }, + { conclusion: "merge", correct: false }, + { conclusion: "close", correct: true }, + { conclusion: "close", correct: false }, + { conclusion: "hold" }, + { conclusion: "merge" }, + ]); + }); + + it("collectPredictionMetricRows does not join an outcome from a different repo (strict project match)", () => { const ledger = tempLedger(); appendPrediction(ledger, 1, "merge"); - appendPrediction(ledger, 2, "close"); - appendPrediction(ledger, 3, "merge"); + const outcomes = toOutcomeRecords([prOutcome(1, "merged", "other/repo")]); + expect(collectPredictionMetricRows(ledger, outcomes)).toEqual([{ conclusion: "merge" }]); + }); + + it("runMetrics renders resolved correct/incorrect counters from both ledgers and returns 0", () => { + const predictionLedger = tempLedger(); + const eventLedger = tempEventLedger(); + appendPrediction(predictionLedger, 1, "merge"); + appendPrediction(predictionLedger, 2, "close"); + appendPrediction(predictionLedger, 3, "merge"); + eventLedger.appendEvent({ type: "pr_outcome", repoFullName: REPO, payload: { prNumber: 1, decision: "merged" } }); + eventLedger.appendEvent({ type: "pr_outcome", repoFullName: REPO, payload: { prNumber: 2, decision: "closed" } }); + eventLedger.appendEvent({ type: "pr_outcome", repoFullName: REPO, payload: { prNumber: 3, decision: "closed" } }); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); - expect(runMetrics([], { initPredictionLedger: () => ledger })).toBe(0); + expect( + runMetrics([], { + initPredictionLedger: () => predictionLedger, + initEventLedger: () => eventLedger, + }), + ).toBe(0); const text = String(log.mock.calls[0]?.[0]); expect(text).toContain("# TYPE loopover_miner_predictions_total counter"); - // Series are emitted in sorted conclusion order, so "close" precedes "merge". expect(text).toContain('loopover_miner_predictions_total{conclusion="close"} 1'); expect(text).toContain('loopover_miner_predictions_total{conclusion="merge"} 2'); - // No outcome-join exists yet, so both the correct and incorrect counters stay zero. - expect(text).toContain("loopover_miner_prediction_correct_total 0"); - expect(text).toContain("loopover_miner_prediction_incorrect_total 0"); - // The output is a single, once-terminated document (no doubled trailing blank line). + expect(text).toContain("loopover_miner_prediction_correct_total 2"); + expect(text).toContain("loopover_miner_prediction_incorrect_total 1"); expect(text.endsWith("\n")).toBe(false); }); - it("runMetrics opens and closes its own default ledger when none is injected", () => { - const dbPath = tempDbPath(); - const seed = initPredictionLedger(dbPath); - appendPrediction(seed, 1, "hold"); - seed.close(); + it("runMetrics opens and closes its own default ledgers when none are injected", () => { + const root = mkdtempSync(join(tmpdir(), "loopover-miner-metrics-cli-default-")); + roots.push(root); + const predictionDbPath = join(root, "prediction-ledger.sqlite3"); + const eventDbPath = join(root, "event-ledger.sqlite3"); + + const seedPrediction = initPredictionLedger(predictionDbPath); + appendPrediction(seedPrediction, 1, "hold"); + seedPrediction.close(); - const prev = process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB; - process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB = dbPath; + const seedEvent = initEventLedger(eventDbPath); + seedEvent.appendEvent({ type: "pr_outcome", repoFullName: REPO, payload: { prNumber: 1, decision: "merged" } }); + seedEvent.close(); + + const prevPrediction = process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB; + const prevEvent = process.env.LOOPOVER_MINER_EVENT_LEDGER_DB; + process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB = predictionDbPath; + process.env.LOOPOVER_MINER_EVENT_LEDGER_DB = eventDbPath; const log = vi.spyOn(console, "log").mockImplementation(() => undefined); try { expect(runMetrics([])).toBe(0); } finally { - if (prev === undefined) delete process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB; - else process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB = prev; + if (prevPrediction === undefined) delete process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB; + else process.env.LOOPOVER_MINER_PREDICTION_LEDGER_DB = prevPrediction; + if (prevEvent === undefined) delete process.env.LOOPOVER_MINER_EVENT_LEDGER_DB; + else process.env.LOOPOVER_MINER_EVENT_LEDGER_DB = prevEvent; } - expect(String(log.mock.calls[0]?.[0])).toContain('loopover_miner_predictions_total{conclusion="hold"} 1'); + const text = String(log.mock.calls[0]?.[0]); + expect(text).toContain('loopover_miner_predictions_total{conclusion="hold"} 1'); + // Hold is never scored even when an outcome exists. + expect(text).toContain("loopover_miner_prediction_correct_total 0"); + expect(text).toContain("loopover_miner_prediction_incorrect_total 0"); }); it("runMetrics rejects unexpected arguments with a usage error", () => { const error = vi.spyOn(console, "error").mockImplementation(() => undefined); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); - expect(runMetrics(["--json"], { initPredictionLedger: () => tempLedger() })).toBe(2); + expect( + runMetrics(["--json"], { + initPredictionLedger: () => tempLedger(), + initEventLedger: () => tempEventLedger(), + }), + ).toBe(2); expect(error).not.toHaveBeenCalled(); expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toEqual({ ok: false, @@ -94,7 +185,12 @@ describe("loopover-miner metrics CLI (#4838)", () => { }); error.mockClear(); log.mockClear(); - expect(runMetrics(["--nope"], { initPredictionLedger: () => tempLedger() })).toBe(2); + expect( + runMetrics(["--nope"], { + initPredictionLedger: () => tempLedger(), + initEventLedger: () => tempEventLedger(), + }), + ).toBe(2); expect(error).toHaveBeenCalledWith("Usage: loopover-miner metrics"); expect(log).not.toHaveBeenCalled(); }); @@ -106,6 +202,7 @@ describe("loopover-miner metrics CLI (#4838)", () => { initPredictionLedger: () => { throw new Error("prediction ledger is locked"); }, + initEventLedger: () => tempEventLedger(), }), ).toBe(2); expect(error).toHaveBeenCalledWith("prediction ledger is locked"); @@ -118,8 +215,25 @@ describe("loopover-miner metrics CLI (#4838)", () => { initPredictionLedger: () => { throw "prediction-ledger-unavailable"; }, + initEventLedger: () => tempEventLedger(), }), ).toBe(2); expect(error).toHaveBeenCalledWith("prediction-ledger-unavailable"); }); + + it("runMetrics closes an injected event ledger only when it opened the default store", () => { + const predictionLedger = tempLedger(); + const eventLedger = tempEventLedger(); + const closeSpy = vi.spyOn(eventLedger, "close"); + appendPrediction(predictionLedger, 1, "merge"); + + vi.spyOn(console, "log").mockImplementation(() => undefined); + expect( + runMetrics([], { + initPredictionLedger: () => predictionLedger, + initEventLedger: () => eventLedger, + }), + ).toBe(0); + expect(closeSpy).not.toHaveBeenCalled(); + }); });