Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 78 additions & 1 deletion packages/loopover-miner/lib/discover-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,14 +292,89 @@ function initDefaultSignalTrackingStore(): SignalStore {
return createSignalTrackingStore({ appendEvent, readEvents });
}

// #8544: bounded raw context for eligibility-exclusion backtest replay — mirrors ORB's #8129/#8130 posture.
// Caps are fixed so a captured exclusion can be re-evaluated under a candidate ContributionProfile without
// storing whole candidate objects or unbounded label/assignee lists.
export const ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS = 50;
export const ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES = 25;
export const ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS = 200;

type EligibilityExclusionCaptureCandidate = {
labels?: string[];
assignees?: string[];
owner?: string;
};

function truncateEligibilityMetadataString(value: string): { value: string; truncated: boolean } {
if (value.length <= ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS) {
return { value, truncated: false };
}
return {
value: value.slice(0, ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS),
truncated: true,
};
}

function boundedEligibilityMetadataStrings(
items: string[] | undefined,
maxCount: number,
): { values: string[] | undefined; truncated: boolean } {
if (items === undefined) return { values: undefined, truncated: false };
const strings = items.filter((entry): entry is string => typeof entry === "string");
if (strings.length === 0) return { values: undefined, truncated: false };
let truncated = strings.length > maxCount;
const bounded = strings.slice(0, maxCount);
const values: string[] = [];
for (const entry of bounded) {
const next = truncateEligibilityMetadataString(entry);
if (next.truncated) truncated = true;
values.push(next.value);
}
return { values, truncated };
}

/** Build bounded metadata for an eligibility-exclusion fired event (#8544). Omits absent/empty fields; adds
* `truncated: true` only when an array-length or string-length cap actually clamped input. */
export function buildEligibilityExclusionMetadata(
candidate: EligibilityExclusionCaptureCandidate,
): Record<string, unknown> | undefined {
let truncated = false;
const metadata: Record<string, unknown> = {};
if (typeof candidate.owner === "string" && candidate.owner) {
const owner = truncateEligibilityMetadataString(candidate.owner);
if (owner.truncated) truncated = true;
metadata.owner = owner.value;
}
const labels = boundedEligibilityMetadataStrings(candidate.labels, ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS);
if (labels.values !== undefined) metadata.labels = labels.values;
if (labels.truncated) truncated = true;
const assignees = boundedEligibilityMetadataStrings(
candidate.assignees,
ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES,
);
if (assignees.values !== undefined) metadata.assignees = assignees.values;
if (assignees.truncated) truncated = true;
if (truncated) metadata.truncated = true;
return Object.keys(metadata).length > 0 ? metadata : undefined;
}

// #7982: records each real-run eligibility exclusion as a rule-fired signal (ruleId = the exclusion reason,
// e.g. "missing_eligibility_label"), so it can later be scored for precision. Best-effort: a store-open
// failure or a single write failure never aborts discovery -- same discipline as the caches/stores in
// runDiscover's real-run branch (own try/catch, degrade silently). Deliberately NOT called from the dry-run
// branch: a dry run previews what a real run would do (including its own noopQueueStore for the portfolio
// queue) and must not itself contribute real data to a precision report.
async function recordEligibilityExclusionSignals(
excluded: ReadonlyArray<{ candidate: { repoFullName: string; issueNumber: number }; reason: string }>,
excluded: ReadonlyArray<{
candidate: {
repoFullName: string;
issueNumber: number;
labels?: string[];
assignees?: string[];
owner?: string;
};
reason: string;
}>,
options: Pick<RunDiscoverOptions, "initSignalTrackingStore" | "nowMs">,
): Promise<void> {
if (excluded.length === 0) return;
Expand All @@ -312,12 +387,14 @@ async function recordEligibilityExclusionSignals(
if (!store) return;
const occurredAt = new Date(options.nowMs ?? Date.now()).toISOString();
for (const entry of excluded) {
const metadata = buildEligibilityExclusionMetadata(entry.candidate);
await store
.recordRuleFired({
ruleId: entry.reason,
targetKey: `${entry.candidate.repoFullName}#issue-${entry.candidate.issueNumber}`,
outcome: "exclude",
occurredAt,
...(metadata ? { metadata } : {}),
})
.catch(() => undefined);
}
Expand Down
98 changes: 98 additions & 0 deletions test/unit/miner-discover-cli-eligibility-metadata.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { describe, expect, it } from "vitest";

// Import the .ts SOURCE via a non-literal specifier so CI's `--coverage.all=false` run grades discover-cli.ts,
// not a stale post-build .js artifact (#8544, same pattern as miner-replay-snapshot.test.ts / #8510).
const DISCOVER_CLI_MODULE = "../../packages/loopover-miner/lib/discover-cli.ts";
const {
ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES,
ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS,
ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS,
buildEligibilityExclusionMetadata,
} = (await import(DISCOVER_CLI_MODULE)) as typeof import("../../packages/loopover-miner/lib/discover-cli.js");

function labelList(count: number, prefix = "label"): string[] {
return Array.from({ length: count }, (_, index) => `${prefix}-${index}`);
}

describe("buildEligibilityExclusionMetadata (#8544)", () => {
it("records labels, assignees, and owner when all three are present", () => {
expect(
buildEligibilityExclusionMetadata({
owner: "acme",
labels: ["help wanted", "bug"],
assignees: ["alice", "bob"],
}),
).toEqual({
owner: "acme",
labels: ["help wanted", "bug"],
assignees: ["alice", "bob"],
});
});

it("omits each field when absent from the candidate", () => {
expect(buildEligibilityExclusionMetadata({})).toBeUndefined();
expect(buildEligibilityExclusionMetadata({ labels: [] })).toBeUndefined();
expect(buildEligibilityExclusionMetadata({ assignees: [] })).toBeUndefined();
expect(buildEligibilityExclusionMetadata({ owner: "" })).toBeUndefined();
expect(buildEligibilityExclusionMetadata({ labels: ["bug"] })).toEqual({ labels: ["bug"] });
expect(buildEligibilityExclusionMetadata({ assignees: ["alice"] })).toEqual({ assignees: ["alice"] });
expect(buildEligibilityExclusionMetadata({ owner: "acme" })).toEqual({ owner: "acme" });
expect(buildEligibilityExclusionMetadata({ labels: [null as unknown as string, "bug"] })).toEqual({
labels: ["bug"],
});
});

it("keeps exactly-max label and assignee counts without a truncated key", () => {
expect(
buildEligibilityExclusionMetadata({
labels: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS),
assignees: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES, "user"),
}),
).toEqual({
labels: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS),
assignees: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES, "user"),
});
});

it("clamps one-over-max label and assignee counts and sets truncated", () => {
const metadata = buildEligibilityExclusionMetadata({
labels: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS + 1),
assignees: labelList(ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES + 1, "user"),
});
expect(metadata?.labels).toHaveLength(ELIGIBILITY_EXCLUSION_METADATA_MAX_LABELS);
expect(metadata?.assignees).toHaveLength(ELIGIBILITY_EXCLUSION_METADATA_MAX_ASSIGNEES);
expect(metadata?.truncated).toBe(true);
});

it("keeps exactly-max-length strings without truncated", () => {
const exact = "x".repeat(ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS);
expect(
buildEligibilityExclusionMetadata({
owner: exact,
labels: [exact],
assignees: [exact],
}),
).toEqual({
owner: exact,
labels: [exact],
assignees: [exact],
});
});

it("truncates one-over-max-length owner, label, and assignee strings and sets truncated", () => {
const over = "y".repeat(ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS + 1);
const expected = "y".repeat(ELIGIBILITY_EXCLUSION_METADATA_MAX_STRING_CHARS);
expect(buildEligibilityExclusionMetadata({ owner: over })).toEqual({
owner: expected,
truncated: true,
});
expect(buildEligibilityExclusionMetadata({ labels: [over] })).toEqual({
labels: [expected],
truncated: true,
});
expect(buildEligibilityExclusionMetadata({ assignees: [over] })).toEqual({
assignees: [expected],
truncated: true,
});
});
});
85 changes: 80 additions & 5 deletions test/unit/miner-discover-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1644,12 +1644,27 @@ describe("runDiscover onResult hook (#6522)", () => {

describe("eligibility-exclusion signal tracking (#7982)", () => {
function fakeSignalStore() {
const fired: Array<{ ruleId: string; targetKey: string; outcome: string }> = [];
const fired: Array<{
ruleId: string;
targetKey: string;
outcome: string;
metadata?: Record<string, unknown>;
}> = [];
return {
fired,
store: {
recordRuleFired: vi.fn(async (event: { ruleId: string; targetKey: string; outcome: string }) => {
fired.push({ ruleId: event.ruleId, targetKey: event.targetKey, outcome: event.outcome });
recordRuleFired: vi.fn(async (event: {
ruleId: string;
targetKey: string;
outcome: string;
metadata?: Record<string, unknown>;
}) => {
fired.push({
ruleId: event.ruleId,
targetKey: event.targetKey,
outcome: event.outcome,
...(event.metadata ? { metadata: event.metadata } : {}),
});
}),
recordHumanOverride: vi.fn(async () => undefined),
queryRuleHistory: vi.fn(async () => ({ fired: [], overrides: [] })),
Expand All @@ -1669,8 +1684,18 @@ describe("runDiscover onResult hook (#6522)", () => {
const exitCode = await runDiscover(["acme/widgets", "--json"], { ...opts, initSignalTrackingStore: () => store });
expect(exitCode).toBe(0);
expect(fired).toEqual([
{ ruleId: "exclusion_label", targetKey: "acme/widgets#issue-2", outcome: "exclude" },
{ ruleId: "missing_eligibility_label", targetKey: "acme/widgets#issue-3", outcome: "exclude" },
{
ruleId: "exclusion_label",
targetKey: "acme/widgets#issue-2",
outcome: "exclude",
metadata: { owner: "acme", labels: ["blocked"] },
},
{
ruleId: "missing_eligibility_label",
targetKey: "acme/widgets#issue-3",
outcome: "exclude",
metadata: { owner: "acme", labels: ["bug"] },
},
]);
});

Expand Down Expand Up @@ -1742,6 +1767,56 @@ describe("runDiscover onResult hook (#6522)", () => {
expect(exitCode).toBe(0);
expect(recordRuleFired).toHaveBeenCalledTimes(2);
});

it("captures bounded candidate context in fired-event metadata on a real run (#8544)", async () => {
const issues = [
fanOutIssue({
issueNumber: 2,
labels: ["blocked"],
assignees: ["alice"],
owner: "acme",
}),
];
const { opts } = discoverWith(issues, new Map([["acme/widgets", trustworthyProfile]]));
const { fired, store } = fakeSignalStore();
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
const exitCode = await runDiscover(["acme/widgets", "--json"], { ...opts, initSignalTrackingStore: () => store });
expect(exitCode).toBe(0);
expect(fired).toEqual([
{
ruleId: "exclusion_label",
targetKey: "acme/widgets#issue-2",
outcome: "exclude",
metadata: {
owner: "acme",
labels: ["blocked"],
assignees: ["alice"],
},
},
]);
});

it("omits metadata entirely when the excluded candidate has no capturable context (#8544)", async () => {
const issues = [
fanOutIssue({
issueNumber: 3,
labels: [123 as unknown as string],
owner: undefined,
assignees: undefined,
}),
];
const { opts } = discoverWith(issues, new Map([["acme/widgets", trustworthyProfile]]));
const { fired, store } = fakeSignalStore();
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
await runDiscover(["acme/widgets", "--json"], { ...opts, initSignalTrackingStore: () => store });
expect(fired).toEqual([
{
ruleId: "missing_eligibility_label",
targetKey: "acme/widgets#issue-3",
outcome: "exclude",
},
]);
});
});

it("SAFE DEFAULT: filters nothing for a low-confidence/empty profile", async () => {
Expand Down