diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6914465..67f88de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,8 @@ Keep in mind: - Codemm’s “agent logic” is backend-owned; the IDE should remain a thin shell over backend contracts. - Judging relies on Docker; don’t add a path that executes untrusted code outside Docker. - For generation APIs, preserve IPC naming stability (`threads.generate`, `threads.generateV2`, `threads.regenerateSlot`, `threads.getGenerationDiagnostics`). +- Generation progress is `runId`-scoped. Do not add thread-wide progress reducers or subscriptions that can mix overlapping runs. +- Persistent generation state lives in `generation_runs`, `generation_slot_runs`, and `generation_slot_transitions`. Avoid rebuilding run state from transient UI reducers or `problems_json` length. ## Style / Guardrails diff --git a/README.md b/README.md index 8253110..ee3b4c6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,17 @@ There is no internal HTTP API for engine calls. UI → engine is IPC only. ## Local State & Persistence - Per-workspace DB: `/codemm.db` (preferred: `/.codemm/codemm.db`) -- Key tables (IDE-first): `threads`, `thread_messages`, `activities`, `runs`, `run_events` +- Key tables (IDE-first): `threads`, `thread_messages`, `activities`, `runs`, `run_events`, `generation_runs`, `generation_slot_runs`, `generation_slot_transitions` + +## Generation State Machine + +Generation is now persisted as three explicit aggregates: + +- `threads`: user-facing lifecycle (`DRAFT` -> `CLARIFYING` -> `READY` -> `GENERATE_PENDING` -> `GENERATING` -> terminal outcome) +- `generation_runs`: one durable activity-generation run per request, correlated by `runId` +- `generation_slot_runs`: one durable slot record per run, with stage transitions and terminal status per slot + +Thread state is derived from persisted run outcomes. Slot failures no longer abort the full thread on first exception, and stale `RUNNING` generation state is reconciled on engine startup. ## Security Model (Practical) @@ -99,6 +109,7 @@ Builds are typically produced on the target OS (mac builds on macOS, win builds - IDE-first mental model + topology: `docs/architecture/IDE_FIRST.md` - Local runtime orchestration: `docs/architecture/LOCAL_LLM_ORCHESTRATION.md` +- Generation state machine: `docs/architecture/GENERATION_STATE_MACHINE.md` - Migration phases: `docs/architecture/MIGRATION.md` - Wrapper behavior: `docs/FUNCTIONS.md` - Security notes: `docs/SECURITY.md` diff --git a/apps/backend/scripts/runTests.js b/apps/backend/scripts/runTests.js index 834768a..ebc8b1c 100644 --- a/apps/backend/scripts/runTests.js +++ b/apps/backend/scripts/runTests.js @@ -54,6 +54,13 @@ function main(argv) { return 1; } + const contractsBuild = spawnSync("npm", ["--workspace", "@codemm/shared-contracts", "run", "build"], { + stdio: "inherit", + }); + if ((contractsBuild.status ?? 1) !== 0) { + return contractsBuild.status ?? 1; + } + // ts-node compilation dominates cost when Node runs each file in an isolated worker. // Running with no isolation keeps a single module cache and is significantly faster. // Note: Node's test runner flags vary across Node versions. Prefer broad compatibility. diff --git a/apps/backend/src/contracts/generationOutcome.ts b/apps/backend/src/contracts/generationOutcome.ts index 986719a..9786711 100644 --- a/apps/backend/src/contracts/generationOutcome.ts +++ b/apps/backend/src/contracts/generationOutcome.ts @@ -1,7 +1,12 @@ +import type { GenerationFailureKind, GenerationSlotTerminalStatus } from "@codemm/shared-contracts"; + export type GenerationOutcome = { slotIndex: number; success: boolean; + status: GenerationSlotTerminalStatus; retries: number; + failureKind?: GenerationFailureKind; + failureCode?: string; + message?: string; appliedFallback?: string; }; - diff --git a/apps/backend/src/contracts/session.ts b/apps/backend/src/contracts/session.ts index 17661bd..32df761 100644 --- a/apps/backend/src/contracts/session.ts +++ b/apps/backend/src/contracts/session.ts @@ -10,9 +10,13 @@ export const SessionStateSchema = z.enum([ "DRAFT", "CLARIFYING", "READY", + "GENERATE_PENDING", "GENERATING", - "SAVED", - "FAILED", + "COMPLETED", + "INCOMPLETE", + "PARTIAL_SUCCESS", + "RETRYABLE_FAILURE", + "HARD_FAILURE", ]); export type SessionState = z.infer; @@ -61,10 +65,14 @@ export type Session = z.infer; const ALLOWED_TRANSITIONS: Record = { DRAFT: ["CLARIFYING"], CLARIFYING: ["CLARIFYING", "READY"], - READY: ["GENERATING"], - GENERATING: ["SAVED", "FAILED", "READY"], - SAVED: [], - FAILED: ["READY"], + READY: ["GENERATE_PENDING"], + GENERATE_PENDING: ["GENERATING", "RETRYABLE_FAILURE", "HARD_FAILURE"], + GENERATING: ["COMPLETED", "INCOMPLETE", "PARTIAL_SUCCESS", "RETRYABLE_FAILURE", "HARD_FAILURE"], + COMPLETED: ["GENERATE_PENDING"], + INCOMPLETE: ["GENERATE_PENDING"], + PARTIAL_SUCCESS: ["GENERATE_PENDING"], + RETRYABLE_FAILURE: ["GENERATE_PENDING", "READY"], + HARD_FAILURE: ["GENERATE_PENDING", "READY"], }; export function canTransition(from: SessionState, to: SessionState): boolean { diff --git a/apps/backend/src/database.ts b/apps/backend/src/database.ts index 68d15cb..cf9dd14 100644 --- a/apps/backend/src/database.ts +++ b/apps/backend/src/database.ts @@ -10,6 +10,22 @@ export { activityRepository as activityDb, submissionRepository as submissionDb, } from "./database/repositories/activityRepository"; +export type { + DBGenerationExecutionAttempt, + DBGenerationRunFailureCacheEntry, + DBGenerationRun, + DBGenerationSlotRun, + DBGenerationSlotDiagnosis, + DBGenerationSlotTransition, +} from "./database/repositories/generationRunRepository"; +export { + generationExecutionAttemptRepository as generationExecutionAttemptDb, + generationRunFailureCacheRepository as generationRunFailureCacheDb, + generationRunRepository as generationRunDb, + generationSlotRunRepository as generationSlotRunDb, + generationSlotDiagnosisRepository as generationSlotDiagnosisDb, + generationSlotTransitionRepository as generationSlotTransitionDb, +} from "./database/repositories/generationRunRepository"; export type { DBSession, DBSessionCollector, diff --git a/apps/backend/src/database/migrations.ts b/apps/backend/src/database/migrations.ts index 3737c97..96d966c 100644 --- a/apps/backend/src/database/migrations.ts +++ b/apps/backend/src/database/migrations.ts @@ -158,6 +158,126 @@ export function initializeDatabase() { ) `); + db.exec(` + CREATE TABLE IF NOT EXISTS generation_runs ( + id TEXT PRIMARY KEY, + thread_id TEXT NOT NULL, + status TEXT NOT NULL, + activity_id TEXT, + total_slots INTEGER NOT NULL DEFAULT 0, + completed_slots INTEGER NOT NULL DEFAULT 0, + successful_slots INTEGER NOT NULL DEFAULT 0, + failed_slots INTEGER NOT NULL DEFAULT 0, + last_failure_kind TEXT, + last_failure_code TEXT, + last_failure_message TEXT, + meta_json TEXT, + started_at TEXT, + finished_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (thread_id) REFERENCES threads(id) ON DELETE CASCADE, + FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE SET NULL + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS generation_slot_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + slot_index INTEGER NOT NULL, + status TEXT NOT NULL, + current_stage TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + title TEXT, + topic TEXT, + language TEXT, + started_at TEXT, + ended_at TEXT, + last_failure_kind TEXT, + last_failure_code TEXT, + last_failure_message TEXT, + last_artifact_hash TEXT, + result_json TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(run_id, slot_index), + FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS generation_slot_transitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + slot_index INTEGER NOT NULL, + attempt INTEGER, + stage TEXT, + status TEXT NOT NULL, + payload_json TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS generation_execution_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + slot_index INTEGER NOT NULL, + attempt INTEGER NOT NULL, + execution_phase TEXT NOT NULL, + bundle_hash TEXT NOT NULL, + strategy TEXT, + budget_profile_json TEXT, + started_at TEXT NOT NULL, + finished_at TEXT, + exit_code INTEGER, + timeout_stage TEXT, + watchdog_source TEXT, + failure_category TEXT, + stdout_hash TEXT, + stderr_hash TEXT, + stdout_snippet TEXT, + stderr_snippet TEXT, + parsed_failures_json TEXT, + trace_json TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS generation_slot_diagnoses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + slot_index INTEGER NOT NULL, + attempt INTEGER NOT NULL, + diagnosis_class TEXT NOT NULL, + recoverability TEXT NOT NULL, + normalized_symptom TEXT NOT NULL, + recommended_repair_strategy TEXT, + source_execution_attempt_id INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE, + FOREIGN KEY (source_execution_attempt_id) REFERENCES generation_execution_attempts(id) ON DELETE SET NULL + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS generation_run_failure_cache ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + language TEXT NOT NULL, + topic_signature TEXT NOT NULL, + failure_class TEXT NOT NULL, + normalized_symptom TEXT NOT NULL, + guardrail_patch_json TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE + ) + `); + db.exec(` CREATE INDEX IF NOT EXISTS idx_threads_state ON threads(state); CREATE INDEX IF NOT EXISTS idx_thread_messages_thread_id ON thread_messages(thread_id); @@ -165,6 +285,13 @@ export function initializeDatabase() { CREATE INDEX IF NOT EXISTS idx_submissions_activity_id ON submissions(activity_id); CREATE INDEX IF NOT EXISTS idx_runs_thread_id ON runs(thread_id); CREATE INDEX IF NOT EXISTS idx_run_events_run_id ON run_events(run_id); + CREATE INDEX IF NOT EXISTS idx_generation_runs_thread_id ON generation_runs(thread_id); + CREATE INDEX IF NOT EXISTS idx_generation_runs_status ON generation_runs(status); + CREATE INDEX IF NOT EXISTS idx_generation_slot_runs_run_id ON generation_slot_runs(run_id); + CREATE INDEX IF NOT EXISTS idx_generation_slot_transitions_run_id ON generation_slot_transitions(run_id); + CREATE INDEX IF NOT EXISTS idx_generation_execution_attempts_run_slot ON generation_execution_attempts(run_id, slot_index, attempt); + CREATE INDEX IF NOT EXISTS idx_generation_slot_diagnoses_run_slot ON generation_slot_diagnoses(run_id, slot_index, attempt); + CREATE INDEX IF NOT EXISTS idx_generation_run_failure_cache_run_id ON generation_run_failure_cache(run_id); `); console.log("Database initialized successfully"); diff --git a/apps/backend/src/database/repositories/activityRepository.ts b/apps/backend/src/database/repositories/activityRepository.ts index bbbb892..d68270f 100644 --- a/apps/backend/src/database/repositories/activityRepository.ts +++ b/apps/backend/src/database/repositories/activityRepository.ts @@ -36,7 +36,7 @@ export const activityRepository = { title: string, problems: string, prompt?: string, - opts?: { status?: "DRAFT" | "PUBLISHED"; timeLimitSeconds?: number | null } + opts?: { status?: "DRAFT" | "INCOMPLETE" | "PUBLISHED"; timeLimitSeconds?: number | null } ) => { const stmt = db.prepare( `INSERT INTO activities (id, title, prompt, problems, status, time_limit_seconds, created_at) @@ -78,7 +78,7 @@ export const activityRepository = { prompt?: string; problems?: string; time_limit_seconds?: number | null; - status?: "DRAFT" | "PUBLISHED"; + status?: "DRAFT" | "INCOMPLETE" | "PUBLISHED"; } ): DBActivity | undefined => { const sets: string[] = []; diff --git a/apps/backend/src/database/repositories/generationRunRepository.ts b/apps/backend/src/database/repositories/generationRunRepository.ts new file mode 100644 index 0000000..ceb8757 --- /dev/null +++ b/apps/backend/src/database/repositories/generationRunRepository.ts @@ -0,0 +1,505 @@ +import type { + GenerationFailureKind, + GenerationRunStatus, + GenerationSlotStage, + GenerationSlotTerminalStatus, +} from "@codemm/shared-contracts"; +import db from "../db"; + +export type DBGenerationRun = { + id: string; + thread_id: string; + status: GenerationRunStatus | string; + activity_id: string | null; + total_slots: number; + completed_slots: number; + successful_slots: number; + failed_slots: number; + last_failure_kind: GenerationFailureKind | null; + last_failure_code: string | null; + last_failure_message: string | null; + meta_json: string | null; + started_at: string | null; + finished_at: string | null; + created_at: string; + updated_at: string; +}; + +export type DBGenerationSlotRun = { + id: number; + run_id: string; + slot_index: number; + status: GenerationSlotStage | string; + current_stage: string | null; + attempt_count: number; + title: string | null; + topic: string | null; + language: string | null; + started_at: string | null; + ended_at: string | null; + last_failure_kind: GenerationFailureKind | null; + last_failure_code: string | null; + last_failure_message: string | null; + last_artifact_hash: string | null; + result_json: string | null; + created_at: string; + updated_at: string; +}; + +export type DBGenerationSlotTransition = { + id: number; + run_id: string; + slot_index: number; + attempt: number | null; + stage: string | null; + status: string; + payload_json: string | null; + created_at: string; +}; + +export type DBGenerationExecutionAttempt = { + id: number; + run_id: string; + slot_index: number; + attempt: number; + execution_phase: "compile" | "test_exec" | "quality_gate" | string; + bundle_hash: string; + strategy: string | null; + budget_profile_json: string | null; + started_at: string; + finished_at: string | null; + exit_code: number | null; + timeout_stage: "compile" | "execute" | "overall" | string | null; + watchdog_source: "inner" | "outer" | "unknown" | string | null; + failure_category: string | null; + stdout_hash: string | null; + stderr_hash: string | null; + stdout_snippet: string | null; + stderr_snippet: string | null; + parsed_failures_json: string | null; + trace_json: string | null; + created_at: string; +}; + +export type DBGenerationSlotDiagnosis = { + id: number; + run_id: string; + slot_index: number; + attempt: number; + diagnosis_class: string; + recoverability: "recoverable" | "fatal" | "quarantine" | string; + normalized_symptom: string; + recommended_repair_strategy: string | null; + source_execution_attempt_id: number | null; + created_at: string; +}; + +export type DBGenerationRunFailureCacheEntry = { + id: number; + run_id: string; + language: string; + topic_signature: string; + failure_class: string; + normalized_symptom: string; + guardrail_patch_json: string | null; + created_at: string; +}; + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value ?? null); + } catch { + return "null"; + } +} + +export const generationRunRepository = { + create(args: { + id: string; + threadId: string; + totalSlots: number; + metaJson?: string | null; + }) { + const stmt = db.prepare( + `INSERT INTO generation_runs ( + id, thread_id, status, activity_id, total_slots, completed_slots, successful_slots, failed_slots, + last_failure_kind, last_failure_code, last_failure_message, meta_json, started_at, finished_at, created_at, updated_at + ) VALUES (?, ?, 'PENDING', NULL, ?, 0, 0, 0, NULL, NULL, NULL, ?, NULL, NULL, datetime('now'), datetime('now'))` + ); + stmt.run(args.id, args.threadId, args.totalSlots, args.metaJson ?? null); + }, + + markRunning(id: string) { + const stmt = db.prepare( + `UPDATE generation_runs + SET status = 'RUNNING', + started_at = COALESCE(started_at, datetime('now')), + updated_at = datetime('now') + WHERE id = ?` + ); + stmt.run(id); + }, + + finish(args: { + id: string; + status: GenerationRunStatus; + activityId?: string | null; + completedSlots: number; + successfulSlots: number; + failedSlots: number; + lastFailureKind?: GenerationFailureKind | null; + lastFailureCode?: string | null; + lastFailureMessage?: string | null; + }) { + const stmt = db.prepare( + `UPDATE generation_runs + SET status = ?, + activity_id = ?, + completed_slots = ?, + successful_slots = ?, + failed_slots = ?, + last_failure_kind = ?, + last_failure_code = ?, + last_failure_message = ?, + finished_at = datetime('now'), + updated_at = datetime('now') + WHERE id = ?` + ); + stmt.run( + args.status, + args.activityId ?? null, + args.completedSlots, + args.successfulSlots, + args.failedSlots, + args.lastFailureKind ?? null, + args.lastFailureCode ?? null, + args.lastFailureMessage ?? null, + args.id + ); + }, + + findById(id: string) { + const stmt = db.prepare(`SELECT * FROM generation_runs WHERE id = ?`); + return stmt.get(id) as DBGenerationRun | undefined; + }, + + latestByThread(threadId: string) { + const stmt = db.prepare( + `SELECT * FROM generation_runs WHERE thread_id = ? ORDER BY created_at DESC, id DESC LIMIT 1` + ); + return stmt.get(threadId) as DBGenerationRun | undefined; + }, + + listStaleActiveRuns() { + const stmt = db.prepare( + `SELECT * FROM generation_runs WHERE status IN ('PENDING', 'RUNNING') ORDER BY created_at ASC` + ); + return stmt.all() as DBGenerationRun[]; + }, +}; + +export const generationSlotRunRepository = { + seed(runId: string, slots: Array<{ slotIndex: number; topic?: string | null; language?: string | null }>) { + const insert = db.prepare( + `INSERT OR REPLACE INTO generation_slot_runs ( + run_id, slot_index, status, current_stage, attempt_count, title, topic, language, + started_at, ended_at, last_failure_kind, last_failure_code, last_failure_message, last_artifact_hash, + result_json, created_at, updated_at + ) VALUES (?, ?, 'QUEUED', 'QUEUED', 0, NULL, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, datetime('now'), datetime('now'))` + ); + const tx = db.transaction((rows: Array<{ slotIndex: number; topic?: string | null; language?: string | null }>) => { + for (const row of rows) { + insert.run(runId, row.slotIndex, row.topic ?? null, row.language ?? null); + } + }); + tx(slots); + }, + + find(runId: string, slotIndex: number) { + const stmt = db.prepare(`SELECT * FROM generation_slot_runs WHERE run_id = ? AND slot_index = ?`); + return stmt.get(runId, slotIndex) as DBGenerationSlotRun | undefined; + }, + + listByRun(runId: string) { + const stmt = db.prepare(`SELECT * FROM generation_slot_runs WHERE run_id = ? ORDER BY slot_index ASC`); + return stmt.all(runId) as DBGenerationSlotRun[]; + }, + + beginSlot(args: { + runId: string; + slotIndex: number; + topic: string; + language: string; + }) { + const stmt = db.prepare( + `UPDATE generation_slot_runs + SET status = 'SKELETON_RUNNING', + current_stage = 'SKELETON_RUNNING', + topic = ?, + language = ?, + started_at = COALESCE(started_at, datetime('now')), + updated_at = datetime('now') + WHERE run_id = ? AND slot_index = ?` + ); + stmt.run(args.topic, args.language, args.runId, args.slotIndex); + }, + + updateStage(args: { + runId: string; + slotIndex: number; + status: GenerationSlotStage; + currentStage?: string | null; + attemptCount?: number; + title?: string | null; + lastFailureKind?: GenerationFailureKind | null; + lastFailureCode?: string | null; + lastFailureMessage?: string | null; + lastArtifactHash?: string | null; + result?: unknown; + ended?: boolean; + }) { + const stmt = db.prepare( + `UPDATE generation_slot_runs + SET status = ?, + current_stage = ?, + attempt_count = ?, + title = COALESCE(?, title), + last_failure_kind = ?, + last_failure_code = ?, + last_failure_message = ?, + last_artifact_hash = COALESCE(?, last_artifact_hash), + result_json = COALESCE(?, result_json), + ended_at = CASE WHEN ? = 1 THEN datetime('now') ELSE ended_at END, + updated_at = datetime('now') + WHERE run_id = ? AND slot_index = ?` + ); + const current = this.find(args.runId, args.slotIndex); + stmt.run( + args.status, + args.currentStage ?? args.status, + args.attemptCount ?? current?.attempt_count ?? 0, + args.title ?? null, + args.lastFailureKind ?? null, + args.lastFailureCode ?? null, + args.lastFailureMessage ?? null, + args.lastArtifactHash ?? null, + typeof args.result === "undefined" ? null : safeJsonStringify(args.result), + args.ended ? 1 : 0, + args.runId, + args.slotIndex + ); + }, + + appendTransition(args: { + runId: string; + slotIndex: number; + attempt?: number | null; + stage?: string | null; + status: string; + payload?: unknown; + }) { + const stmt = db.prepare( + `INSERT INTO generation_slot_transitions (run_id, slot_index, attempt, stage, status, payload_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, datetime('now'))` + ); + stmt.run( + args.runId, + args.slotIndex, + typeof args.attempt === "number" ? args.attempt : null, + args.stage ?? null, + args.status, + typeof args.payload === "undefined" ? null : safeJsonStringify(args.payload) + ); + }, + + markTerminal(args: { + runId: string; + slotIndex: number; + status: GenerationSlotTerminalStatus; + attemptCount: number; + title?: string | null; + result?: unknown; + lastFailureKind?: GenerationFailureKind | null; + lastFailureCode?: string | null; + lastFailureMessage?: string | null; + }) { + const nextArgs: Parameters[0] = { + runId: args.runId, + slotIndex: args.slotIndex, + status: args.status, + currentStage: args.status, + attemptCount: args.attemptCount, + ended: true, + }; + if (typeof args.title !== "undefined") nextArgs.title = args.title; + if (typeof args.result !== "undefined") nextArgs.result = args.result; + if (typeof args.lastFailureKind !== "undefined") nextArgs.lastFailureKind = args.lastFailureKind; + if (typeof args.lastFailureCode !== "undefined") nextArgs.lastFailureCode = args.lastFailureCode; + if (typeof args.lastFailureMessage !== "undefined") nextArgs.lastFailureMessage = args.lastFailureMessage; + this.updateStage(nextArgs); + }, + + reconcileIncomplete(runId: string) { + const stmt = db.prepare( + `UPDATE generation_slot_runs + SET status = CASE + WHEN status = 'QUEUED' THEN 'SKIPPED' + ELSE 'RETRYABLE_FAILURE' + END, + current_stage = CASE + WHEN status = 'QUEUED' THEN 'SKIPPED' + ELSE 'RETRYABLE_FAILURE' + END, + last_failure_code = COALESCE(last_failure_code, 'ENGINE_RESTART'), + last_failure_message = COALESCE(last_failure_message, 'Generation was interrupted before slot completion.'), + ended_at = COALESCE(ended_at, datetime('now')), + updated_at = datetime('now') + WHERE run_id = ? + AND status NOT IN ('SUCCEEDED', 'RETRYABLE_FAILURE', 'HARD_FAILURE', 'SKIPPED')` + ); + stmt.run(runId); + }, +}; + +export const generationSlotTransitionRepository = { + listByRun(runId: string) { + const stmt = db.prepare(`SELECT * FROM generation_slot_transitions WHERE run_id = ? ORDER BY id ASC`); + return stmt.all(runId) as DBGenerationSlotTransition[]; + }, +}; + +export const generationExecutionAttemptRepository = { + create(args: { + runId: string; + slotIndex: number; + attempt: number; + executionPhase: "compile" | "test_exec" | "quality_gate"; + bundleHash: string; + strategy?: string | null; + budgetProfile?: unknown; + startedAt: string; + finishedAt?: string | null; + exitCode?: number | null; + timeoutStage?: "compile" | "execute" | "overall" | null; + watchdogSource?: "inner" | "outer" | "unknown" | null; + failureCategory?: string | null; + stdoutHash?: string | null; + stderrHash?: string | null; + stdoutSnippet?: string | null; + stderrSnippet?: string | null; + parsedFailures?: unknown; + trace?: unknown; + }) { + const stmt = db.prepare( + `INSERT INTO generation_execution_attempts ( + run_id, slot_index, attempt, execution_phase, bundle_hash, strategy, budget_profile_json, + started_at, finished_at, exit_code, timeout_stage, watchdog_source, failure_category, + stdout_hash, stderr_hash, stdout_snippet, stderr_snippet, parsed_failures_json, trace_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ); + const result = stmt.run( + args.runId, + args.slotIndex, + args.attempt, + args.executionPhase, + args.bundleHash, + args.strategy ?? null, + typeof args.budgetProfile === "undefined" ? null : safeJsonStringify(args.budgetProfile), + args.startedAt, + args.finishedAt ?? null, + typeof args.exitCode === "number" ? args.exitCode : null, + args.timeoutStage ?? null, + args.watchdogSource ?? null, + args.failureCategory ?? null, + args.stdoutHash ?? null, + args.stderrHash ?? null, + args.stdoutSnippet ?? null, + args.stderrSnippet ?? null, + typeof args.parsedFailures === "undefined" ? null : safeJsonStringify(args.parsedFailures), + typeof args.trace === "undefined" ? null : safeJsonStringify(args.trace) + ); + return Number(result.lastInsertRowid); + }, + + listByRun(runId: string, slotIndex?: number) { + if (typeof slotIndex === "number") { + const stmt = db.prepare( + `SELECT * FROM generation_execution_attempts WHERE run_id = ? AND slot_index = ? ORDER BY id ASC` + ); + return stmt.all(runId, slotIndex) as DBGenerationExecutionAttempt[]; + } + const stmt = db.prepare(`SELECT * FROM generation_execution_attempts WHERE run_id = ? ORDER BY id ASC`); + return stmt.all(runId) as DBGenerationExecutionAttempt[]; + }, +}; + +export const generationSlotDiagnosisRepository = { + create(args: { + runId: string; + slotIndex: number; + attempt: number; + diagnosisClass: string; + recoverability: "recoverable" | "fatal" | "quarantine"; + normalizedSymptom: string; + recommendedRepairStrategy?: string | null; + sourceExecutionAttemptId?: number | null; + }) { + const stmt = db.prepare( + `INSERT INTO generation_slot_diagnoses ( + run_id, slot_index, attempt, diagnosis_class, recoverability, normalized_symptom, + recommended_repair_strategy, source_execution_attempt_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ); + const result = stmt.run( + args.runId, + args.slotIndex, + args.attempt, + args.diagnosisClass, + args.recoverability, + args.normalizedSymptom, + args.recommendedRepairStrategy ?? null, + args.sourceExecutionAttemptId ?? null + ); + return Number(result.lastInsertRowid); + }, + + listByRun(runId: string, slotIndex?: number) { + if (typeof slotIndex === "number") { + const stmt = db.prepare(`SELECT * FROM generation_slot_diagnoses WHERE run_id = ? AND slot_index = ? ORDER BY id ASC`); + return stmt.all(runId, slotIndex) as DBGenerationSlotDiagnosis[]; + } + const stmt = db.prepare(`SELECT * FROM generation_slot_diagnoses WHERE run_id = ? ORDER BY id ASC`); + return stmt.all(runId) as DBGenerationSlotDiagnosis[]; + }, +}; + +export const generationRunFailureCacheRepository = { + create(args: { + runId: string; + language: string; + topicSignature: string; + failureClass: string; + normalizedSymptom: string; + guardrailPatch?: unknown; + }) { + const stmt = db.prepare( + `INSERT INTO generation_run_failure_cache ( + run_id, language, topic_signature, failure_class, normalized_symptom, guardrail_patch_json + ) VALUES (?, ?, ?, ?, ?, ?)` + ); + const result = stmt.run( + args.runId, + args.language, + args.topicSignature, + args.failureClass, + args.normalizedSymptom, + typeof args.guardrailPatch === "undefined" ? null : safeJsonStringify(args.guardrailPatch) + ); + return Number(result.lastInsertRowid); + }, + + listByRun(runId: string) { + const stmt = db.prepare(`SELECT * FROM generation_run_failure_cache WHERE run_id = ? ORDER BY id ASC`); + return stmt.all(runId) as DBGenerationRunFailureCacheEntry[]; + }, +}; diff --git a/apps/backend/src/database/repositories/threadRepository.ts b/apps/backend/src/database/repositories/threadRepository.ts index 49698c0..2807af3 100644 --- a/apps/backend/src/database/repositories/threadRepository.ts +++ b/apps/backend/src/database/repositories/threadRepository.ts @@ -65,6 +65,11 @@ export const threadRepository = { return stmt.get(id) as DBSession | undefined; }, + findByActivityId: (activityId: string): DBSession | undefined => { + const stmt = db.prepare(`SELECT * FROM threads WHERE activity_id = ? ORDER BY updated_at DESC LIMIT 1`); + return stmt.get(activityId) as DBSession | undefined; + }, + updateState: (id: string, state: string) => { const stmt = db.prepare(`UPDATE threads SET state = ?, updated_at = datetime('now') WHERE id = ?`); stmt.run(state, id); @@ -155,6 +160,13 @@ export const threadRepository = { `); return stmt.all(safeLimit) as DBSessionSummary[]; }, + + listByStates: (states: string[]) => { + if (!Array.isArray(states) || states.length === 0) return [] as DBSession[]; + const placeholders = states.map(() => "?").join(", "); + const stmt = db.prepare(`SELECT * FROM threads WHERE state IN (${placeholders}) ORDER BY updated_at DESC`); + return stmt.all(...states) as DBSession[]; + }, }; export const threadCollectorRepository = { diff --git a/apps/backend/src/generation/legacyAdapter.ts b/apps/backend/src/generation/legacyAdapter.ts deleted file mode 100644 index d5defb7..0000000 --- a/apps/backend/src/generation/legacyAdapter.ts +++ /dev/null @@ -1,136 +0,0 @@ -import crypto from "crypto"; -import type { ProblemPlan } from "../planner/types"; -import { generateSingleProblem } from "./perSlotGenerator"; -import { - ReferenceSolutionValidationError, - validateReferenceSolution, -} from "./referenceSolutionValidator"; -import { runTestStrengthGate, TestStrengthGateError } from "./testStrengthGate"; -import { deriveSlotObligations } from "./obligations"; -import { progressSummaryForFailure, validateInjectedDraftContract } from "./services/validationService"; -import type { SlotIntent } from "../contracts/generationDiagnostics"; -import type { GenerationProgressEvent } from "../contracts/generationProgress"; -import type { SlotPromptContext } from "../languages/types"; - -function computeExpensiveFingerprint(draft: unknown): string { - const h = crypto.createHash("sha256"); - const generatedDraft = draft as Record; - h.update(String(generatedDraft.language ?? "")); - h.update("\n==test_suite==\n"); - h.update(String(generatedDraft.test_suite ?? "")); - - if ("reference_solution" in generatedDraft) { - h.update("\n==reference_solution==\n"); - h.update(String(generatedDraft.reference_solution ?? "")); - } - - if ("reference_workspace" in generatedDraft && (generatedDraft.reference_workspace as any)?.files) { - const files = Array.isArray((generatedDraft.reference_workspace as any).files) - ? [...(generatedDraft.reference_workspace as any).files] - : []; - files.sort((a: any, b: any) => String(a?.path ?? "").localeCompare(String(b?.path ?? ""))); - h.update("\n==reference_workspace==\n"); - for (const file of files) { - h.update(String(file?.path ?? "")); - h.update("\0"); - h.update(String(file?.content ?? "")); - h.update("\n"); - } - } - - return h.digest("hex"); -} - -export async function runLegacySlotAdapter(args: { - slot: ProblemPlan[number]; - promptContext: SlotPromptContext; - slotIntent: SlotIntent; - onProgress?: (event: GenerationProgressEvent) => void; - deps?: { - generateSingleProblem?: typeof generateSingleProblem; - validateReferenceSolution?: typeof validateReferenceSolution; - runTestStrengthGate?: typeof runTestStrengthGate; - }; -}): Promise<{ generated: Awaited>; attempt: number }> { - // Remaining legacy-only behavior stays here while the staged pipeline absorbs: - // 1. injected test seams around generateSingleProblem/validation - // 2. deterministic re-validation caching via expensive draft fingerprints - // 3. quality-gate retry handling shared with older integration coverage - const defaultMaxAttempts = 3; - const generateSingleProblemFn = args.deps?.generateSingleProblem ?? generateSingleProblem; - const validateReferenceSolutionFn = args.deps?.validateReferenceSolution ?? validateReferenceSolution; - const runTestStrengthGateFn = args.deps?.runTestStrengthGate ?? runTestStrengthGate; - - let qualityFailureFingerprint: string | undefined; - let cachedQualityFailure: unknown; - let validatedFingerprint: string | undefined; - - for (let attempt = 1; attempt <= defaultMaxAttempts; attempt++) { - args.onProgress?.({ type: "slot_llm_attempt_started", slotIndex: args.slot.index, attempt }); - args.onProgress?.({ type: "attempt_started", index: args.slot.index, attempt }); - - let generated: Awaited> | undefined; - try { - generated = await generateSingleProblemFn(args.slot, { promptContext: args.promptContext }); - validateInjectedDraftContract(args.slot, generated.draft); - args.onProgress?.({ type: "slot_contract_validated", slotIndex: args.slot.index, attempt }); - args.onProgress?.({ - type: "slot_evidence", - slotIndex: args.slot.index, - attempt, - obligations: deriveSlotObligations(args.slot).map((id) => ({ id, ok: true })), - }); - args.onProgress?.({ type: "slot_docker_validation_started", slotIndex: args.slot.index, attempt }); - args.onProgress?.({ type: "validation_started", index: args.slot.index, attempt }); - - const fingerprint = computeExpensiveFingerprint(generated.draft); - if (validatedFingerprint !== fingerprint) { - await validateReferenceSolutionFn(generated.draft); - validatedFingerprint = fingerprint; - } - - if (qualityFailureFingerprint === fingerprint && cachedQualityFailure) { - throw cachedQualityFailure; - } - - await runTestStrengthGateFn(generated.draft, args.slot); - return { generated, attempt }; - } catch (err) { - const fingerprint = generated ? computeExpensiveFingerprint(generated.draft) : undefined; - if (err instanceof TestStrengthGateError && fingerprint) { - qualityFailureFingerprint = fingerprint; - cachedQualityFailure = err; - } - if (err instanceof ReferenceSolutionValidationError) { - args.onProgress?.({ - type: "slot_docker_validation_failed", - slotIndex: args.slot.index, - attempt, - shortError: err.message, - }); - args.onProgress?.({ type: "validation_failed", index: args.slot.index, attempt }); - } - const emitted = progressSummaryForFailure({ - slotIndex: args.slot.index, - attempt, - maxAttempts: defaultMaxAttempts, - err, - ...(typeof generated?.meta?.llmOutputHash === "string" ? { llmOutputHash: generated.meta.llmOutputHash } : {}), - ...(generated?.meta?.llm ? { llm: generated.meta.llm } : {}), - slotIntent: args.slotIntent, - final: attempt >= defaultMaxAttempts, - }); - args.onProgress?.(emitted.summary); - args.onProgress?.(emitted.failure); - args.onProgress?.({ - type: "attempt_failed", - index: args.slot.index, - attempt, - phase: err instanceof ReferenceSolutionValidationError || err instanceof TestStrengthGateError ? "validate" : "generate", - }); - if (attempt >= defaultMaxAttempts) throw err; - } - } - - throw new Error(`Failed to generate slot ${args.slot.index}.`); -} diff --git a/apps/backend/src/generation/orchestrator.ts b/apps/backend/src/generation/orchestrator.ts index c305a4b..254e692 100644 --- a/apps/backend/src/generation/orchestrator.ts +++ b/apps/backend/src/generation/orchestrator.ts @@ -2,15 +2,10 @@ import type { ProblemPlan } from "../planner/types"; import type { GeneratedProblem } from "../contracts/problem"; import type { GenerationOutcome } from "../contracts/generationOutcome"; import type { GenerationProgressEvent } from "../contracts/generationProgress"; -import { createExecutionContext } from "../engine/execution/ExecutionContext"; -import { ExecutionEngine } from "../engine/execution/ExecutionEngine"; -import type { Step } from "../engine/execution/Step"; import { trace } from "../utils/trace"; -import { GenerationSlotFailureError } from "./errors"; import { deriveSlotObligations } from "./obligations"; import { runSlotPipeline, SlotPipelineTerminalError } from "../pipeline/slotStages"; import { applyGuidedScaffoldingAsync } from "./services/scaffoldingService"; -import { runLegacySlotAdapter } from "./legacyAdapter"; import { buildArtifactSet, buildSlotIntent, @@ -21,6 +16,7 @@ import { progressSummaryForFailure, } from "./services/validationService"; import type { SlotPromptContext } from "../languages/types"; +import type { SlotExecutionFailure, SlotExecutionResult } from "../services/threads/generationState"; const DOMAIN_POOL = [ "smart home", @@ -63,42 +59,101 @@ function pickDomain(seed: string, usedDomains: string[]): string { return DOMAIN_POOL[start]!; } -type OrchestratorState = { - problems: GeneratedProblem[]; - outcomes: GenerationOutcome[]; - usedDomains: string[]; - usedTitles: string[]; -}; +function isHardFailureKind(kind: string): boolean { + return kind === "infra" || kind === "judge_infra_failure" || kind === "spec_error" || kind === "run_policy_failure"; +} + +function resolveSlotConcurrency(explicit?: number | null): number { + const raw = + typeof explicit === "number" && Number.isFinite(explicit) + ? explicit + : Number.parseInt(process.env.CODEMM_GENERATION_SLOT_CONCURRENCY ?? "", 10); + if (!Number.isFinite(raw)) return 2; + return Math.max(1, Math.min(4, Math.trunc(raw))); +} + +function buildProblemMapFromResume( + problems: GeneratedProblem[], + outcomes: GenerationOutcome[], +): Map { + const bySlot = new Map(); + let problemCursor = 0; + for (const outcome of outcomes) { + if (!outcome?.success) continue; + const problem = problems[problemCursor]; + if (problem) { + bySlot.set(outcome.slotIndex, problem); + problemCursor += 1; + } + } + return bySlot; +} + +function synthesizeFailureStage(status: GenerationOutcome["status"]): SlotExecutionFailure["stage"] { + if (status === "QUARANTINED") return "FAILURE_DIAGNOSED"; + if (status === "HARD_FAILURE" || status === "FATAL_FAILED") return "QUALITY_GATE_RUNNING"; + if (status === "SKIPPED") return "EXECUTION_BUNDLE_READY"; + return "VALIDATING_REFERENCE"; +} + +function synthesizeResultFromOutcome( + outcome: GenerationOutcome, + problem: GeneratedProblem | undefined, +): SlotExecutionResult { + if (outcome.success && problem) { + return { + slotIndex: outcome.slotIndex, + terminalStatus: "SUCCEEDED", + retries: outcome.retries, + problem, + outcome, + title: problem.title, + }; + } + + const terminalStatus = + outcome.status === "QUARANTINED" || outcome.status === "SKIPPED" + ? outcome.status + : outcome.status === "HARD_FAILURE" || outcome.status === "FATAL_FAILED" + ? "HARD_FAILURE" + : "RETRYABLE_FAILURE"; + + return { + slotIndex: outcome.slotIndex, + terminalStatus, + retries: outcome.retries, + outcome, + failure: { + kind: outcome.failureKind ?? "unknown", + code: outcome.failureCode ?? "RESUMED_SLOT_FAILURE", + message: outcome.message ?? "Slot did not complete successfully.", + stage: synthesizeFailureStage(outcome.status), + }, + }; +} async function runSlotGenerationStep(args: { slot: ProblemPlan[number]; - state: OrchestratorState; onProgress?: (event: GenerationProgressEvent) => void; - onCheckpoint?: (state: { - problems: GeneratedProblem[]; - outcomes: GenerationOutcome[]; - completedSlotIndex: number; - }) => void; customInstructionsMd?: string; - useLegacyAdapter: boolean; + promptContext?: SlotPromptContext; deps?: { - generateSingleProblem?: (...args: any[]) => Promise; - validateReferenceSolution?: (...args: any[]) => Promise; - runTestStrengthGate?: (...args: any[]) => Promise; + runSlotPipeline?: typeof runSlotPipeline; }; -}) { +}): Promise { const { slot } = args; const slotIntent = buildSlotIntent(slot); - const domainSeed = pickDomain( - `${slot.language}:${slot.difficulty}:${slot.topics.join(",")}:${slot.index}`, - args.state.usedDomains - ); - const promptContext: SlotPromptContext = { - domain: domainSeed, - avoidDomains: args.state.usedDomains.slice(-4), - avoidTitles: args.state.usedTitles.slice(-4), - ...(args.customInstructionsMd ? { customInstructionsMd: args.customInstructionsMd } : {}), - }; + const domainSeed = + args.promptContext?.domain ?? + pickDomain(`${slot.language}:${slot.difficulty}:${slot.topics.join(",")}:${slot.index}`, []); + const promptContext: SlotPromptContext = + args.promptContext ?? + ({ + domain: domainSeed, + avoidDomains: [], + avoidTitles: [], + ...(args.customInstructionsMd ? { customInstructionsMd: args.customInstructionsMd } : {}), + } satisfies SlotPromptContext); const topic = slot.topics[0] ?? "topic"; args.onProgress?.({ @@ -119,25 +174,18 @@ async function runSlotGenerationStep(args: { }); try { - const generatedResult = args.useLegacyAdapter - ? await runLegacySlotAdapter({ - slot, - promptContext, - slotIntent, - ...(args.onProgress ? { onProgress: args.onProgress } : {}), - ...(args.deps ? { deps: args.deps } : {}), - }) - : { - generated: await runSlotPipeline({ - slot, - promptContext, - ...(args.onProgress ? { onProgress: args.onProgress } : {}), - }), - attempt: 1, - }; + const pipelineRunner = args.deps?.runSlotPipeline ?? runSlotPipeline; + const generatedResult = { + generated: await pipelineRunner({ + slot, + promptContext, + ...(args.onProgress ? { onProgress: args.onProgress } : {}), + }), + attempt: 1, + }; const { generated, attempt: finalAttempt } = generatedResult; - if (args.useLegacyAdapter && slot.pedagogy) { + if (slot.pedagogy && !generated.draft.pedagogy) { generated.draft = { ...(await applyGuidedScaffoldingAsync(generated.draft, slot)), pedagogy: slot.pedagogy, @@ -156,22 +204,30 @@ async function runSlotGenerationStep(args: { slotIntent, artifactSet: buildArtifactSet(generated.draft), }); - if (!args.useLegacyAdapter) { - args.onProgress?.({ - type: "slot_evidence", - slotIndex: slot.index, - attempt: 1, - obligations: deriveSlotObligations(slot).map((id) => ({ id, ok: true })), - }); - } + args.onProgress?.({ + type: "slot_evidence", + slotIndex: slot.index, + attempt: 1, + obligations: deriveSlotObligations(slot).map((id) => ({ id, ok: true })), + }); args.onProgress?.({ type: "slot_completed", slotIndex: slot.index }); args.onProgress?.({ type: "problem_validated", index: slot.index }); - args.state.problems.push(problem); - args.state.outcomes.push({ slotIndex: slot.index, success: true, retries: Math.max(0, finalAttempt - 1) }); - args.state.usedDomains.push(domainSeed); - args.state.usedTitles.push(problem.title); - args.onCheckpoint?.({ problems: args.state.problems, outcomes: args.state.outcomes, completedSlotIndex: slot.index }); + const outcome: GenerationOutcome = { + slotIndex: slot.index, + success: true, + status: "SUCCEEDED", + retries: Math.max(0, finalAttempt - 1), + }; + const result: SlotExecutionResult = { + slotIndex: slot.index, + terminalStatus: "SUCCEEDED", + retries: outcome.retries, + problem, + outcome, + title: problem.title, + }; trace("generation.attempt.success", { slotIndex: slot.index, title: generated.draft.title }); + return result; } catch (err: any) { console.warn(`Slot ${slot.index} staged pipeline failed:`, err?.message ?? err); const finalKind = err instanceof SlotPipelineTerminalError ? err.kind : inferFailureKind(err); @@ -201,55 +257,112 @@ async function runSlotGenerationStep(args: { slotIndex: slot.index, success: false, retries: 0, + status: finalKind === "repair_no_progress" ? "QUARANTINED" : isHardFailureKind(finalKind) ? "HARD_FAILURE" : "RETRYABLE_FAILURE", + failureKind: finalKind, + failureCode: err instanceof SlotPipelineTerminalError ? `STAGE_${err.stage.toUpperCase()}` : "SLOT_PIPELINE_FAILED", + message: err instanceof Error ? err.message : String(err), + }; + const failure: SlotExecutionFailure = { + kind: finalKind, + code: err instanceof SlotPipelineTerminalError ? `STAGE_${err.stage.toUpperCase()}` : "SLOT_PIPELINE_FAILED", + message: err instanceof Error ? err.message : String(err), + stage: + err instanceof SlotPipelineTerminalError + ? err.stage === "validate" + ? "VALIDATING_REFERENCE" + : err.stage === "repair" + ? "REPAIRING_REFERENCE" + : err.stage === "reference" + ? "REFERENCE_RUNNING" + : err.stage === "tests" + ? "TESTS_RUNNING" + : "SKELETON_RUNNING" + : "HARD_FAILURE", + ...(typeof err?.title === "string" ? { title: err.title } : {}), + ...(typeof err?.llmOutputHash === "string" ? { llmOutputHash: err.llmOutputHash } : {}), + }; + const terminalStatus = + failOutcome.status === "HARD_FAILURE" + ? "HARD_FAILURE" + : failOutcome.status === "QUARANTINED" + ? "QUARANTINED" + : "RETRYABLE_FAILURE"; + const result: SlotExecutionResult = { + slotIndex: slot.index, + terminalStatus, + retries: 0, + outcome: failOutcome, + failure, + ...(typeof err?.title === "string" ? { title: err.title } : {}), }; - throw new GenerationSlotFailureError( - `Failed to generate slot ${slot.index}. Last error: ${err instanceof Error ? err.message : String(err)}`, - { - slotIndex: slot.index, - kind: finalKind, - attempts: 1, - ...(typeof err?.title === "string" ? { title: err.title } : {}), - ...(typeof err?.llmOutputHash === "string" ? { llmOutputHash: err.llmOutputHash } : {}), - ...(err?.llm ? { llm: err.llm } : {}), - outcomesSoFar: [...args.state.outcomes, failOutcome], - problemsSoFar: [...args.state.problems], - } - ); + return result; } } +function buildPromptContextForSlot(args: { + slot: ProblemPlan[number]; + assignedDomains: Map; + priorSuccessfulTitles: string[]; + customInstructionsMd?: string; +}): SlotPromptContext { + const priorDomains = [...args.assignedDomains.entries()] + .filter(([slotIndex]) => slotIndex < args.slot.index) + .sort((a, b) => a[0] - b[0]) + .map(([, domain]) => domain); + + return { + domain: args.assignedDomains.get(args.slot.index) ?? pickDomain( + `${args.slot.language}:${args.slot.difficulty}:${args.slot.topics.join(",")}:${args.slot.index}`, + priorDomains + ), + avoidDomains: priorDomains.slice(-4), + avoidTitles: args.priorSuccessfulTitles.slice(-4), + ...(args.customInstructionsMd ? { customInstructionsMd: args.customInstructionsMd } : {}), + }; +} + +function buildOrderedSnapshot(args: { + plan: ProblemPlan; + resultBySlot: Map; +}): { problems: GeneratedProblem[]; outcomes: GenerationOutcome[]; slotResults: SlotExecutionResult[] } { + const ordered = args.plan + .map((slot) => args.resultBySlot.get(slot.index)) + .filter((result): result is SlotExecutionResult => Boolean(result)); + + const outcomes = ordered.map((result) => result.outcome); + const problems = ordered + .filter((result): result is Extract => result.terminalStatus === "SUCCEEDED") + .map((result) => result.problem); + + return { problems, outcomes, slotResults: ordered }; +} + export async function generateProblemsFromPlan( plan: ProblemPlan, opts?: { onProgress?: (event: GenerationProgressEvent) => void; customInstructionsMd?: string | null; resume?: { problems: GeneratedProblem[]; outcomes: GenerationOutcome[] }; + targetSlotIndexes?: number[]; + concurrency?: number; onCheckpoint?: (state: { problems: GeneratedProblem[]; outcomes: GenerationOutcome[]; completedSlotIndex: number; }) => void; deps?: { - generateSingleProblem?: (...args: any[]) => Promise; - validateReferenceSolution?: (...args: any[]) => Promise; - runTestStrengthGate?: (...args: any[]) => Promise; + runSlotPipeline?: typeof runSlotPipeline; }; } -): Promise<{ problems: GeneratedProblem[]; outcomes: GenerationOutcome[] }> { - const resumeProblems = Array.isArray(opts?.resume?.problems) ? opts!.resume!.problems : []; - const resumeOutcomes = Array.isArray(opts?.resume?.outcomes) ? opts!.resume!.outcomes : []; +): Promise<{ problems: GeneratedProblem[]; outcomes: GenerationOutcome[]; slotResults: SlotExecutionResult[] }> { + const resumeProblems = Array.isArray(opts?.resume?.problems) ? opts.resume!.problems : []; + const resumeOutcomes = Array.isArray(opts?.resume?.outcomes) ? opts.resume!.outcomes : []; const initialCount = resumeProblems.length === resumeOutcomes.length && resumeProblems.length <= plan.length ? resumeProblems.length : 0; - - const problems: GeneratedProblem[] = initialCount ? [...resumeProblems.slice(0, initialCount)] : []; - const outcomes: GenerationOutcome[] = initialCount ? [...resumeOutcomes.slice(0, initialCount)] : []; const onProgress = opts?.onProgress; const onCheckpoint = opts?.onCheckpoint; - const useLegacyAdapter = typeof opts?.deps?.generateSingleProblem === "function"; - const usedDomains: string[] = []; - const usedTitles: string[] = []; const customInstructionsMd = (() => { const raw = typeof opts?.customInstructionsMd === "string" ? opts.customInstructionsMd : ""; const trimmed = raw.trim(); @@ -257,36 +370,82 @@ export async function generateProblemsFromPlan( const maxLen = 8000; return trimmed.length > maxLen ? `${trimmed.slice(0, maxLen)}…(truncated)` : trimmed; })(); + const resultBySlot = new Map(); + const resumeProblemBySlot = buildProblemMapFromResume(resumeProblems, resumeOutcomes); + const targetSlotIndexes = (() => { + const explicit = Array.isArray(opts?.targetSlotIndexes) + ? [...new Set(opts!.targetSlotIndexes.filter((value) => Number.isInteger(value) && value >= 0 && value < plan.length))].sort((a, b) => a - b) + : null; + if (explicit && explicit.length > 0) return explicit; + if (initialCount > 0) return plan.slice(initialCount).map((slot) => slot.index); + return plan.map((slot) => slot.index); + })(); + const targetSlotIndexSet = new Set(targetSlotIndexes); - for (let i = 0; i < initialCount; i++) { - const slot = plan[i]; - if (!slot) continue; - const domainSeed = pickDomain(`${slot.language}:${slot.difficulty}:${slot.topics.join(",")}:${slot.index}`, usedDomains); - usedDomains.push(domainSeed); - const title = problems[i]?.title; - if (typeof title === "string" && title.trim()) usedTitles.push(title); + for (const outcome of resumeOutcomes) { + if (targetSlotIndexSet.has(outcome.slotIndex)) continue; + const synthesized = synthesizeResultFromOutcome(outcome, resumeProblemBySlot.get(outcome.slotIndex)); + resultBySlot.set(outcome.slotIndex, synthesized); } - const context = createExecutionContext>({ - workflowId: `generation-plan:${plan.length}:${initialCount}`, - loggerName: "generation.orchestrator", - initialState: { problems, outcomes, usedDomains, usedTitles }, + const assignedDomains = new Map(); + for (const slot of plan) { + const priorDomains = [...assignedDomains.values()]; + assignedDomains.set( + slot.index, + pickDomain(`${slot.language}:${slot.difficulty}:${slot.topics.join(",")}:${slot.index}`, priorDomains), + ); + } + + const priorSuccessfulTitles = plan + .map((slot) => resumeProblemBySlot.get(slot.index)) + .filter((problem): problem is GeneratedProblem => Boolean(problem)) + .map((problem) => problem.title) + .filter((title): title is string => typeof title === "string" && title.trim().length > 0); + + const slotsToRun = plan.filter((slot) => targetSlotIndexSet.has(slot.index)); + const concurrency = Math.min(resolveSlotConcurrency(opts?.concurrency ?? null), Math.max(1, slotsToRun.length || 1)); + trace("generation.orchestrator.schedule", { + totalSlots: plan.length, + targetSlotIndexes, + concurrency, }); - const steps: Step>[] = plan.slice(initialCount).map((slot) => ({ - id: `slot:${slot.index}`, - run: async () => { - await runSlotGenerationStep({ - slot, - state: context.state, + + let cursor = 0; + async function worker() { + while (cursor < slotsToRun.length) { + const next = slotsToRun[cursor]; + cursor += 1; + if (!next) continue; + + const result = await runSlotGenerationStep({ + slot: next, ...(onProgress ? { onProgress } : {}), - ...(onCheckpoint ? { onCheckpoint } : {}), ...(customInstructionsMd ? { customInstructionsMd } : {}), - useLegacyAdapter, + promptContext: buildPromptContextForSlot({ + slot: next, + assignedDomains, + priorSuccessfulTitles, + ...(customInstructionsMd ? { customInstructionsMd } : {}), + }), ...(opts?.deps ? { deps: opts.deps } : {}), }); - }, - })); + resultBySlot.set(next.index, result); + + if (onCheckpoint) { + const snapshot = buildOrderedSnapshot({ plan, resultBySlot }); + onCheckpoint({ + problems: snapshot.problems, + outcomes: snapshot.outcomes, + completedSlotIndex: next.index, + }); + } + } + } + + if (slotsToRun.length > 0) { + await Promise.all(Array.from({ length: concurrency }, () => worker())); + } - await new ExecutionEngine(steps).run(context); - return { problems: context.state.problems, outcomes: context.state.outcomes }; + return buildOrderedSnapshot({ plan, resultBySlot }); } diff --git a/apps/backend/src/generation/perSlotGenerator.ts b/apps/backend/src/generation/perSlotGenerator.ts deleted file mode 100644 index 84ac1a2..0000000 --- a/apps/backend/src/generation/perSlotGenerator.ts +++ /dev/null @@ -1,1824 +0,0 @@ -import crypto from "crypto"; -import { createCodemmCompletion } from "../infra/llm"; -import type { CompletionMeta } from "../infra/llm"; -import { tryParseJson } from "../utils/jsonParser"; -import { buildDefaultClassSkeleton, inferClassName } from "../utils/javaCodegen"; -import { - hasBrittleWhitespaceStringExpectations, - isValidJUnit5TestSuite, - javaTestSuiteCapturesStdout, - javaTestSuiteSetsStdin, -} from "../languages/java/rules"; -import { assertJavaStructuralTopicRequirements } from "../languages/java/structuralTopics"; -import { diagnoseCppTestSuite, hasCppStdoutWrites, looksLikeCppTestSuiteCapturesStdout } from "../languages/cpp/rules"; -import { hasPythonStdoutWrites, isValidPytestTestSuiteForStyle } from "../languages/python/rules"; -import { GeneratedProblemDraftSchema, type GeneratedProblemDraft } from "../contracts/problem"; -import type { ProblemSlot } from "../planner/types"; -import { buildSlotPromptWithContext, getSystemPromptForSlot } from "./prompts"; -import { trace, traceText } from "../utils/trace"; -import { GenerationContractError } from "./errors"; -import { getTopLevelPublicTypeNames, javaUsesStdout, javaUsesStdin } from "../utils/javaSource"; -import type { SlotPromptContext } from "../languages/types"; -import { coerceSqlTestSuiteToJsonString } from "../languages/sql/rules"; -import { ObligationViolationError, type ObligationId } from "./obligations"; -import { demoteExtraTopLevelPublicTypes, promoteOneTopLevelTypeToPublic, rewriteJavaTopLevelPublicClassName } from "../utils/javaRewrite"; -import { buildJavaStdinSampleDrivenJUnitTestSuite, computeJavaStdoutSamplesByExecutingReference } from "../languages/java/sampleDrivenTests"; - -const MAX_TOKENS = 5000; -const TEMPERATURE = 0.3; -const REPAIR_TEMPERATURE = 0.4; - -type ProblemStyle = "stdout" | "return" | "mixed"; -function normalizeProblemStyle(raw: string): ProblemStyle { - const s = String(raw ?? "").trim().toLowerCase(); - if (s === "stdout" || s === "return" || s === "mixed") return s; - if (s.includes("stdout")) return "stdout"; - if (s.includes("mixed")) return "mixed"; - return "return"; -} - -function stripCppComments(source: string): string { - const withoutBlock = source.replace(/\/\*[\s\S]*?\*\//g, ""); - return withoutBlock.replace(/\/\/.*$/gm, ""); -} - -function extractCppSolveSignature(referenceSolution: string): string | null { - const src = String(referenceSolution ?? ""); - if (!src.trim()) return null; - - // Best-effort: match a solve(...) function definition (brace may be on same line). - const reSameLine = - /(^|\n)\s*([A-Za-z_][\w:<>\s*&]+?)\s+solve\s*\(([\s\S]*?)\)\s*(?:const\s*)?\{/m; - const m1 = reSameLine.exec(src); - const m = m1; - if (!m) return null; - - const returnType = m[2]?.replace(/\s+/g, " ").trim(); - const params = m[3]?.replace(/\s+/g, " ").trim(); - if (!returnType || params == null) return null; - return `${returnType} solve(${params})`; -} - -function synthesizeCppStarterCodeFromReference(args: { referenceSolution: string; fallbackTopic: string }): string | null { - const signature = extractCppSolveSignature(args.referenceSolution); - if (!signature) return null; - - return `#include - -${signature} { - // BEGIN STUDENT TODO - // TODO: Implement the missing core logic (${args.fallbackTopic}). - // Hint: Use the problem description as your spec. - // Hint: Let the tests drive edge cases. - // END STUDENT TODO - throw std::runtime_error("TODO"); -} -`; -} - -export const __test__ = { - stripCppComments, - extractCppSolveSignature, - synthesizeCppStarterCodeFromReference, -}; - -function sanitizeJavaStringLiteralsBoundaryWhitespace(testSuite: string): { testSuite: string; changed: boolean } { - // Deterministic de-brittling: remove leading/trailing *space/tab* characters in Java string literals. - // This aligns with `hasBrittleWhitespaceStringExpectations()` (which rejects literals like " Bob "). - // - // We intentionally do NOT try to interpret escapes; we only trim literal boundary characters. - let changed = false; - const out = String(testSuite ?? "").replace(/"((?:\\.|[^"\\])*)"/g, (_m, inner: string) => { - const raw = inner ?? ""; - if (!/\S/.test(raw)) return `"${raw}"`; // ignore all-whitespace strings (allowed by rule) - const trimmed = raw.replace(/^[ \t]+|[ \t]+$/g, ""); - if (trimmed === raw) return `"${raw}"`; - changed = true; - return `"${trimmed}"`; - }); - return { testSuite: out, changed }; -} - -function summarizeJUnitFailures(output: string): string[] { - const text = String(output ?? ""); - const lines = text.split(/\r?\n/); - const out: string[] = []; - - for (const line of lines) { - const m = - /^\|\s+.*--\s+([A-Za-z0-9_]+)\([^)]*\)\s+\[X\].*expected:\s*<([^>]*)>\s*but was:\s*<([^>]*)>/i.exec(line) || - /^\s*=>.*expected:\s*<([^>]*)>\s*but was:\s*<([^>]*)>/i.exec(line); - if (!m) continue; - if (m.length === 4) { - out.push(`${m[1]}: expected "${m[2]}", got "${m[3]}"`); - } else if (m.length === 3) { - out.push(`expected "${m[1]}", got "${m[2]}"`); - } - if (out.length >= 10) break; - } - - // Also surface failing test names even if no expected/actual is shown in-line. - if (out.length === 0) { - for (const line of lines) { - const name = /^\|\s+.*--\s+([A-Za-z0-9_]+)\([^)]*\)\s+\[X\]/.exec(line)?.[1]; - if (name) out.push(`${name}: failed`); - if (out.length >= 10) break; - } - } - - return out; -} - -async function repairJavaReferenceSolution(args: { - slot: ProblemSlot; - draft: Extract; - errorMessage: string; - judgeStdout?: string; - judgeStderr?: string; - ctx?: SlotPromptContext; -}): Promise<{ reference_solution: string; llmOutputHash: string; llm?: CompletionMeta }> { - const title = args.draft.title; - const failures = summarizeJUnitFailures(`${args.judgeStdout ?? ""}\n${args.judgeStderr ?? ""}`); - - const system = ` -You are Codemm's Java reference solution repairer. - -Your job: -- Fix ONLY the hidden reference_solution so it passes the provided JUnit test_suite. -- Do NOT change the test_suite. -- Do NOT change the problem description/constraints. - -Hard rules: -- Java 17, no package declarations. -- reference_solution must declare at most ONE top-level public type. -- Return ONLY valid JSON (no markdown/no prose) with this exact schema: - { "reference_solution": "..." } -- Encode newlines as "\\n" (single backslash). -`; - - const stdoutSnippet = String(args.judgeStdout ?? "").slice(0, 2400); - const stderrSnippet = String(args.judgeStderr ?? "").slice(0, 2400); - const errorMessage = String(args.errorMessage ?? "").slice(0, 800); - const failureSummary = failures.length ? `\nFailing assertions summary:\n- ${failures.join("\n- ")}\n` : ""; - - const user = ` -Title: ${title} -Difficulty: ${args.slot.difficulty} -Topics: ${args.slot.topics.join(", ")} -Problem style: ${args.slot.problem_style} -Constraints: ${args.draft.constraints} -${args.ctx?.domain ? `Scenario seed: ${args.ctx.domain}\n` : ""}${args.ctx?.avoidDomains?.length ? `Avoid repeating domains: ${args.ctx.avoidDomains.join(", ")}\n` : ""}${args.ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${args.ctx.avoidTitles.join(" | ")}\n` : ""} - -Description: -${args.draft.description} - -Starter code (student-facing; keep semantics consistent): -${args.draft.starter_code} - -JUnit test_suite (DO NOT CHANGE): -${args.draft.test_suite} -${failureSummary} - -Last failure reason: -${errorMessage} - -Docker/JUnit output (may be truncated): -STDOUT: -${stdoutSnippet || "(empty)"} - -STDERR: -${stderrSnippet || "(empty)"} - -Previous reference_solution (you must change it to make tests pass): -${args.draft.reference_solution} - -Return JSON: {"reference_solution":"..."} only. -`; - - const completion = await createCodemmCompletion({ - system, - user, - temperature: REPAIR_TEMPERATURE, - maxTokens: MAX_TOKENS, - }); - - const text = completion.content.map((b) => (b.type === "text" ? b.text : "")).join("\n"); - const llmOutputHash = sha256(text); - const parsed = tryParseJson(text) as any; - const repaired = typeof parsed?.reference_solution === "string" ? parsed.reference_solution.trim() : ""; - if (!repaired) throw new Error("Java reference_solution repair failed: missing reference_solution."); - return { reference_solution: repaired, llmOutputHash, ...(completion.meta ? { llm: completion.meta } : {}) }; -} - -export type RepairContext = { - previousDraft?: GeneratedProblemDraft; - previousRaw?: string; - errorMessage?: string; - judgeStdout?: string; - judgeStderr?: string; - history?: Array<{ - attempt: number; - phase: "contract" | "validate" | "quality" | "unknown"; - message: string; - strategy?: string; - obligationId?: string; - }>; -}; - -function formatRepairHistory( - history?: Array<{ - attempt: number; - phase: "contract" | "validate" | "quality" | "unknown"; - message: string; - strategy?: string; - obligationId?: string; - }> -): string { - if (!Array.isArray(history) || history.length === 0) return ""; - const lines = history.slice(-3).map((entry) => { - const parts = [`Attempt ${entry.attempt}`, `phase=${entry.phase}`]; - if (entry.obligationId) parts.push(`obligation=${entry.obligationId}`); - if (entry.strategy) parts.push(`repair=${entry.strategy}`); - return `- ${parts.join(", ")}: ${String(entry.message ?? "").slice(0, 320)}`; - }); - return `\nRecent failure history (do not repeat these mistakes):\n${lines.join("\n")}\n`; -} - -export type GeneratedDraftWithMeta = { - draft: GeneratedProblemDraft; - meta: { llmOutputHash: string; llm?: CompletionMeta; rewrites?: Array<{ id: string; applied: boolean; detail?: string }> }; -}; - -async function repairCppTestSuite(args: { - slot: ProblemSlot; - title: string; - description: string; - constraints: string; - starterCode: string; - referenceSolution: string; - previousTestSuite: string; - errorMessage: string; -}): Promise { - const style = normalizeProblemStyle(args.slot.problem_style); - const system = ` -You are Codemm's C++ test suite repairer. - -Your job: -- Produce a VALID C++20 test.cpp for a problem, using the required harness. -- The test suite MUST compile against solution.cpp and MUST be deterministic. - -Hard rules: -- Return ONLY valid JSON (no markdown, no code fences, no prose) -- Output schema: { "test_suite": "..." } -- test_suite must be based on this exact template (copy/paste; only edit inside the TODO blocks): - #include - #include "solution.cpp" - - static int __codem_failures = 0; - #define RUN_TEST(name, ...) do { \\ - try { __VA_ARGS__; std::cout << "[PASS] " << (name) << "\\\\n"; } \\ - catch (const std::exception&) { std::cout << "[FAIL] " << (name) << "\\\\n"; __codem_failures++; } \\ - catch (...) { std::cout << "[FAIL] " << (name) << "\\\\n"; __codem_failures++; } \\ - } while (0) - - int main() { - RUN_TEST("test_case_1", { /* TODO */ }); - RUN_TEST("test_case_2", { /* TODO */ }); - RUN_TEST("test_case_3", { /* TODO */ }); - RUN_TEST("test_case_4", { /* TODO */ }); - RUN_TEST("test_case_5", { /* TODO */ }); - RUN_TEST("test_case_6", { /* TODO */ }); - RUN_TEST("test_case_7", { /* TODO */ }); - RUN_TEST("test_case_8", { /* TODO */ }); - return __codem_failures ? 1 : 0; - } - -Additional rules: -- Each TODO block must contain deterministic assertions (use std::runtime_error on failure). -- Problem style for this activity is "${style}": - - return: tests should call solve(...) and compare returned values. - - stdout: tests should call solve(...), capture std::cout output (redirect rdbuf), and compare printed output. - - mixed: tests should compare BOTH the returned value and captured std::cout output. -`.trim(); - - const user = ` -Slot: -${JSON.stringify({ difficulty: args.slot.difficulty, topics: args.slot.topics, style: args.slot.problem_style })} - -Title: -${args.title} - -Description: -${args.description} - -Constraints: -${args.constraints} - -Starter code (learner edits): -${args.starterCode} - -Reference solution (must pass all tests): -${args.referenceSolution} - -Previous invalid test_suite: -${args.previousTestSuite} - -Error: -${args.errorMessage} - -Return JSON: {"test_suite":"..."} only. -`.trim(); - - const completion = await createCodemmCompletion({ - system, - user, - temperature: 0.2, - maxTokens: 2400, - }); - - const text = completion.content.map((b) => (b.type === "text" ? b.text : "")).join("\n"); - traceText("generation.cpp.testSuite.repair.raw", text, { extra: { slotIndex: args.slot.index } }); - const parsed = tryParseJson(text) as any; - const repaired = typeof parsed?.test_suite === "string" ? parsed.test_suite.trim() : ""; - if (!repaired) throw new Error("C++ test_suite repair failed: missing test_suite."); - return repaired; -} - -async function repairPythonTestSuite(args: { - slot: ProblemSlot; - title: string; - description: string; - constraints: string; - starterCode: string; - referenceSolution: string; - previousTestSuite: string; - errorMessage: string; -}): Promise { - const style = normalizeProblemStyle(args.slot.problem_style); - const system = ` -You are Codemm's Python pytest test suite repairer. - -Your job: -- Produce a VALID pytest test suite for the given problem. -- The suite MUST be deterministic and MUST pass against the provided reference_solution. - -Hard rules: -- Return ONLY valid JSON (no markdown, no code fences, no prose) -- Output schema: { "test_suite": "..." } -- Python 3.11, pytest -- test_suite MUST start with: - import pytest - from solution import solve -- Exactly 8 tests named: test_case_1 ... test_case_8 -- Tests MUST NOT use input(), print(), open(), randomness, or pytest.approx - -Problem style for this activity is "${style}": -- return: each test must assert solve(...) == expected -- stdout: each test must call solve(...), then use capsys.readouterr() and assert on captured.out -- mixed: each test must assert solve(...) == expected AND assert captured.out (after calling solve) -`.trim(); - - const user = ` -Slot: -${JSON.stringify({ difficulty: args.slot.difficulty, topics: args.slot.topics, style: args.slot.problem_style })} - -Title: -${args.title} - -Description: -${args.description} - -Constraints: -${args.constraints} - -Starter code (learner edits): -${args.starterCode} - -Reference solution (must pass all tests): -${args.referenceSolution} - -Previous invalid test_suite: -${args.previousTestSuite} - -Error: -${args.errorMessage} - -Return JSON: {"test_suite":"..."} only. -`.trim(); - - const completion = await createCodemmCompletion({ - system, - user, - temperature: 0, - maxTokens: 2000, - }); - - const text = completion.content.map((b) => (b.type === "text" ? b.text : "")).join("\n"); - traceText("generation.python.testSuite.repair.raw", text, { extra: { slotIndex: args.slot.index } }); - const parsed = tryParseJson(text) as any; - const repaired = typeof parsed?.test_suite === "string" ? parsed.test_suite.trim() : ""; - if (!repaired) throw new Error("Python test_suite repair failed: missing test_suite."); - return repaired; -} - -function sha256(text: string): string { - return crypto.createHash("sha256").update(text).digest("hex"); -} - -function coerceNonEmptySamplePairs( - raw: any, - fallbackLabel: string -): { sampleInputs: string[]; sampleOutputs: string[]; changed: boolean } { - const placeholder = "(see problem description)"; - const rawInputs = Array.isArray(raw?.sample_inputs) ? raw.sample_inputs : []; - const rawOutputs = Array.isArray(raw?.sample_outputs) ? raw.sample_outputs : []; - - const inputs = rawInputs - .map((x: any) => String(x ?? "").trim()) - .filter(Boolean) - .slice(0, 10); - const outputs = rawOutputs - .map((x: any) => String(x ?? "").trim()) - .filter(Boolean) - .slice(0, 10); - - let nextInputs = inputs; - let nextOutputs = outputs; - let changed = false; - - if (nextInputs.length === 0) { - nextInputs = [`${fallbackLabel}: ${placeholder}`]; - changed = true; - } - if (nextOutputs.length === 0) { - nextOutputs = [placeholder]; - changed = true; - } - - if (nextInputs.length !== nextOutputs.length) { - nextInputs = [nextInputs[0] ?? `${fallbackLabel}: ${placeholder}`]; - nextOutputs = [nextOutputs[0] ?? placeholder]; - changed = true; - } - - return { sampleInputs: nextInputs, sampleOutputs: nextOutputs, changed }; -} - -function inferPrimaryClassName(starterCode: string, fallback: string): string { - const topLevelPublic = getTopLevelPublicTypeNames(starterCode)[0]; - if (topLevelPublic) return topLevelPublic; - return inferClassName(starterCode, fallback); -} - -function mapJavaStructuralTopicErrorToObligationId(message: string): ObligationId | null { - const m = /Structural topic requirement failed \((polymorphism|inheritance|abstraction|encapsulation|composition)\)/i.exec( - String(message ?? "") - ); - const key = m?.[1]?.toLowerCase(); - if (!key) return null; - return `java.structural_topic.${key}` as ObligationId; -} - -function assertJavaLegacyDraftInvariants(slot: ProblemSlot, draft: Extract) { - if (!("starter_code" in draft)) return; - - const starterCode = String((draft as any).starter_code ?? ""); - const testSuite = String((draft as any).test_suite ?? ""); - const referenceSolution = String((draft as any).reference_solution ?? ""); - - const starterPublicTypes = getTopLevelPublicTypeNames(starterCode); - if (starterPublicTypes.length !== 1) { - throw new Error("starter_code must declare exactly one top-level public type."); - } - const className = inferPrimaryClassName(starterCode, `Problem${slot.index + 1}`); - - const expectedTestClassName = `${className}Test`; - const actualTestClassName = inferPrimaryClassName(testSuite, expectedTestClassName); - if (actualTestClassName !== expectedTestClassName) { - throw new Error(`Test suite class name "${actualTestClassName}" must match "${expectedTestClassName}".`); - } - - if (/^\s*package\s+/m.test(referenceSolution)) { - throw new Error(`reference_solution for slot ${slot.index} contains package declaration.`); - } - const refPublicTypes = getTopLevelPublicTypeNames(referenceSolution); - if (refPublicTypes.length !== 1) { - throw new Error("reference_solution must declare exactly one top-level public type."); - } - const refClassName = inferPrimaryClassName(referenceSolution, ""); - if (refClassName !== className) { - throw new Error( - `reference_solution class name "${refClassName}" does not match starter_code class name "${className}".` - ); - } - - // Avoid pathological patterns that are guaranteed to fail compilation. - if (/\bwhile\s*\(\s*false\s*\)\s*\{?/.test(referenceSolution)) { - throw new Error('reference_solution must not include "while(false)" (unreachable statement).'); - } -} - -function hasJavaMainMethod(source: string): boolean { - const s = String(source ?? ""); - const withoutBlockComments = s.replace(/\/\*[\s\S]*?\*\//g, ""); - const withoutLineComments = withoutBlockComments.replace(/\/\/.*$/gm, ""); - return /public\s+static\s+void\s+main\s*\(\s*(?:final\s+)?String\s*(?:(?:\[\s*\]|\.\.\.)\s*\w+|\w+\s*\[\s*\])\s*\)/.test( - withoutLineComments - ); -} - -function hasJavaStructuralTopics(topics: string[]): boolean { - const lower = topics.map((t) => String(t ?? "").toLowerCase()); - const keys = ["polymorphism", "inheritance", "abstraction", "encapsulation", "composition"]; - return keys.some((k) => lower.some((t) => t.includes(k))); -} - -function assertJavaFilenameMatchesPublicClass(filename: string, source: string) { - const publicType = getTopLevelPublicTypeNames(source)[0]; - if (!publicType) return; // no public top-level type is okay - const expected = filename.replace(/\.java$/i, ""); - if (publicType !== expected) { - throw new Error(`Public type "${publicType}" must match filename "${filename}".`); - } -} - -function getWorkspaceTargetFile(draft: any): { path: string; role: string; content: string } | null { - const files = draft?.workspace?.files; - if (!Array.isArray(files) || files.length === 0) return null; - const nonEntry = files.find((f: any) => f && typeof f === "object" && f.role !== "entry"); - return (nonEntry ?? files[0]) as any; -} - -function buildJavaRepairPrompt(slot: ProblemSlot, repair: RepairContext, ctx?: SlotPromptContext): string { - const previousJson = - repair.previousDraft != null ? JSON.stringify(repair.previousDraft, null, 2) : null; - const stdoutSnippet = (repair.judgeStdout ?? "").slice(0, 1600); - const stderrSnippet = (repair.judgeStderr ?? "").slice(0, 1600); - const rawSnippet = (repair.previousRaw ?? "").slice(0, 2400); - const errorMessage = (repair.errorMessage ?? "").slice(0, 600); - const historyBlock = formatRepairHistory(repair.history); - const isContractRepair = !repair.previousDraft && !stdoutSnippet && !stderrSnippet; - - if (isContractRepair) { - return `You previously generated a problem JSON for this slot, but it FAILED deterministic validation (before any Docker/JUnit run). - -Slot requirements: -- Difficulty: ${slot.difficulty} -- Topics: ${slot.topics.join(", ")} -- Problem style: ${slot.problem_style} -- Constraints: ${slot.constraints} -- Java 17, no package declarations -- test_suite must have exactly 8 @Test methods (JUnit 5) -${ctx?.domain ? `\nScenario seed: ${ctx.domain}\n` : ""} -${ctx?.avoidDomains?.length ? `Avoid repeating domains: ${ctx.avoidDomains.join(", ")}\n` : ""} -${ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${ctx.avoidTitles.join(" | ")}\n` : ""} - -Validation failure reason: -${errorMessage || "(not provided)"} -${historyBlock} - -Hard structure rules (do not violate): -- Return ONLY valid JSON (no markdown, no code fences, no prose). -- If using legacy fields: starter_code + reference_solution must be valid Java 17 with no package declarations. -- If using workspace fields: workspace + reference_workspace must be valid Java 17 with no package declarations, and reference_workspace must include the same file paths as workspace. -- Each Java file must not declare more than one public class. -- Keep exactly 8 @Test methods. -- If asserting on strings, do NOT use assertEquals() with string literals that have leading/trailing spaces. If whitespace behavior matters, normalize the actual value first (e.g. actual.trim()). - -Here is your previous output (may be truncated): -${rawSnippet || "(not provided)"} - -Goal: -- Return corrected JSON with the exact same fields. -- Keep id/title/description/starter_code stable when possible. -- Fix test_suite structure and/or brittle assertions to satisfy validation. - -Return ONLY valid JSON. No markdown. No code fences. No prose.`; - } - - const failedArtifact = repair.previousDraft - ? ("reference_workspace" in repair.previousDraft ? "reference_workspace" : "reference_solution") - : "reference_solution"; - - return `You previously generated a problem JSON for this slot, but the ${failedArtifact} FAILED when executed against the test_suite in Docker/JUnit. - -Slot requirements: -- Difficulty: ${slot.difficulty} -- Topics: ${slot.topics.join(", ")} -- Problem style: ${slot.problem_style} -- Constraints: ${slot.constraints} -- Java 17, no package declarations -- test_suite must have exactly 8 @Test methods (JUnit 5) -${ctx?.domain ? `\nScenario seed: ${ctx.domain}\n` : ""} -${ctx?.avoidDomains?.length ? `Avoid repeating domains: ${ctx.avoidDomains.join(", ")}\n` : ""} -${ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${ctx.avoidTitles.join(" | ")}\n` : ""} - -Failure output (may include the real assertion failure): -STDOUT: -${stdoutSnippet || "(empty)"} - -STDERR: -${stderrSnippet || "(empty)"} - -Error reason: -${errorMessage || "(not provided)"} -${historyBlock} - -Hard structure rules (do not violate): -- If using legacy fields: starter_code + reference_solution must be valid Java 17 with no package declarations. -- If using workspace fields: workspace + reference_workspace must be valid Java 17 with no package declarations, and reference_workspace must include the same file paths as workspace. -- Each Java file must not declare more than one public class. -- Keep exactly 8 @Test methods. -- Avoid brittle whitespace expectations like assertEquals(" Bob White ", ...) unless the problem explicitly specifies whitespace behavior. - -Here is your previous output (may be truncated): -${rawSnippet || "(not provided)"} - -Here is your previous JSON (preferred to edit if present): -${previousJson || "(not provided)"} - -Goal: -- Return corrected JSON with the exact same fields. -- Prefer keeping id/title/description/starter_code stable. -- Prefer fixing the reference solution artifact to satisfy the existing tests. -- Only change test_suite if it is clearly inconsistent with the description or contains an obvious mistake; otherwise keep tests stable. -- The final test_suite + reference artifact MUST compile and MUST pass in Docker/JUnit. -- Keep tests meaningful (no trivial assertions). - -Return ONLY valid JSON. No markdown. No code fences. No prose.`; -} - -function buildPythonRepairPrompt(slot: ProblemSlot, repair: RepairContext, ctx?: SlotPromptContext): string { - const previousJson = - repair.previousDraft != null ? JSON.stringify(repair.previousDraft, null, 2) : null; - const stdoutSnippet = (repair.judgeStdout ?? "").slice(0, 1600); - const stderrSnippet = (repair.judgeStderr ?? "").slice(0, 1600); - const rawSnippet = (repair.previousRaw ?? "").slice(0, 2400); - const errorMessage = (repair.errorMessage ?? "").slice(0, 600); - const historyBlock = formatRepairHistory(repair.history); - - return `You previously generated a problem JSON for this slot, but the reference_solution FAILED when executed against the test_suite in Docker/pytest. - -Slot requirements: -- Difficulty: ${slot.difficulty} -- Topics: ${slot.topics.join(", ")} -- Problem style: ${slot.problem_style} -- Constraints: ${slot.constraints} -- Python 3.11 -- test_suite must use pytest and define exactly 8 tests named test_case_1..test_case_8 - -${ctx?.domain ? `\nScenario seed: ${ctx.domain}\n` : ""} -${ctx?.avoidDomains?.length ? `Avoid repeating domains: ${ctx.avoidDomains.join(", ")}\n` : ""} -${ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${ctx.avoidTitles.join(" | ")}\n` : ""} - -Failure output: -STDOUT: -${stdoutSnippet || "(empty)"} - -STDERR: -${stderrSnippet || "(empty)"} - -Error reason: -${errorMessage || "(not provided)"} -${historyBlock} - -Hard structure rules (do not violate): -- starter_code and reference_solution must define solve(...) -- solve(...) must NOT read from stdin (no input(), no sys.stdin.*) and must not use networking or randomness -- For problem_style=return: solve(...) must NOT print; tests must assert solve(...) == expected -- For problem_style=stdout: solve(...) should print the answer; tests must capture stdout via capsys and assert on captured.out -- For problem_style=mixed: solve(...) should return the answer AND print it; tests must assert both return and captured.out -- test_suite must import solve via: from solution import solve -- No print-based tests, no randomness, no pytest.approx -- Keep exactly 8 tests: test_case_1..test_case_8 - -Here is your previous output (may be truncated): -${rawSnippet || "(not provided)"} - -Here is your previous JSON (preferred to edit if present): -${previousJson || "(not provided)"} - -Goal: -- Return corrected JSON with the exact same fields. -- Prefer keeping id/title/description/starter_code stable. -- You MAY update test_suite and/or reference_solution, but the final pair MUST pass in Docker/pytest. - -Return ONLY valid JSON. No markdown. No code fences. No prose.`; -} - -function buildCppRepairPrompt(slot: ProblemSlot, repair: RepairContext, ctx?: SlotPromptContext): string { - const previousJson = - repair.previousDraft != null ? JSON.stringify(repair.previousDraft, null, 2) : null; - const stdoutSnippet = (repair.judgeStdout ?? "").slice(0, 1600); - const stderrSnippet = (repair.judgeStderr ?? "").slice(0, 1600); - const rawSnippet = (repair.previousRaw ?? "").slice(0, 2400); - const errorMessage = (repair.errorMessage ?? "").slice(0, 600); - const historyBlock = formatRepairHistory(repair.history); - - return `You previously generated a problem JSON for this slot, but the reference_solution FAILED when executed against the test_suite in Docker/g++. - -Slot requirements: -- Difficulty: ${slot.difficulty} -- Topics: ${slot.topics.join(", ")} -- Problem style: ${slot.problem_style} -- Constraints: ${slot.constraints} -- C++20 (g++) -- test_suite must include exactly 8 RUN_TEST("test_case_1".. "test_case_8", ...) tests - -${ctx?.domain ? `\nScenario seed: ${ctx.domain}\n` : ""} -${ctx?.avoidDomains?.length ? `Avoid repeating domains: ${ctx.avoidDomains.join(", ")}\n` : ""} -${ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${ctx.avoidTitles.join(" | ")}\n` : ""} - -Failure output: -STDOUT: -${stdoutSnippet || "(empty)"} - -STDERR: -${stderrSnippet || "(empty)"} - -Error reason: -${errorMessage || "(not provided)"} -${historyBlock} - -Hard structure rules (do not violate): -- starter_code and reference_solution must define solve(...) (no main()) -- test_suite must #include "solution.cpp" and define main() -- Keep exactly 8 tests: test_case_1..test_case_8 using RUN_TEST("test_case_N", { ... }) -- IMPORTANT: RUN_TEST must be a VARIADIC macro: #define RUN_TEST(name, ...) ... __VA_ARGS__ ... - (otherwise commas inside test blocks break compilation) -- Tests must be deterministic. -- solve(...) must NOT read from stdin (no cin/scanf/getline/etc). -- For problem_style=return: tests should compare returned values (no output capture). -- For problem_style=stdout: tests should capture std::cout output (redirect rdbuf) and compare printed output. -- For problem_style=mixed: tests should compare BOTH the returned value and captured std::cout output. -- Tests must print one line per test: [PASS] test_case_N or [FAIL] test_case_N - -Here is your previous output (may be truncated): -${rawSnippet || "(not provided)"} - -Here is your previous JSON (preferred to edit if present): -${previousJson || "(not provided)"} - -Goal: -- Return corrected JSON with the exact same fields. -- Prefer keeping id/title/description/starter_code stable. -- You MAY update test_suite and/or reference_solution, but the final pair MUST pass in Docker/g++. - -Return ONLY valid JSON. No markdown. No code fences. No prose.`; -} - -function buildSqlRepairPrompt(slot: ProblemSlot, repair: RepairContext, ctx?: SlotPromptContext): string { - const previousJson = - repair.previousDraft != null ? JSON.stringify(repair.previousDraft, null, 2) : null; - const stdoutSnippet = (repair.judgeStdout ?? "").slice(0, 1600); - const stderrSnippet = (repair.judgeStderr ?? "").slice(0, 1600); - const rawSnippet = (repair.previousRaw ?? "").slice(0, 2400); - const errorMessage = (repair.errorMessage ?? "").slice(0, 600); - const historyBlock = formatRepairHistory(repair.history); - - return `You previously generated a problem JSON for this slot, but the reference_solution FAILED when executed against the test_suite in Docker/SQLite. - -Slot requirements: -- Difficulty: ${slot.difficulty} -- Topics: ${slot.topics.join(", ")} -- Problem style: ${slot.problem_style} -- Constraints: ${slot.constraints} -- SQLite 3 -- test_suite must be valid JSON with schema_sql + exactly 8 cases: test_case_1..test_case_8 - -${ctx?.domain ? `\nScenario seed: ${ctx.domain}\n` : ""} -${ctx?.avoidDomains?.length ? `Avoid repeating domains: ${ctx.avoidDomains.join(", ")}\n` : ""} -${ctx?.avoidTitles?.length ? `Avoid reusing titles too similar to: ${ctx.avoidTitles.join(" | ")}\n` : ""} - -Failure output: -STDOUT: -${stdoutSnippet || "(empty)"} - -STDERR: -${stderrSnippet || "(empty)"} - -Error reason: -${errorMessage || "(not provided)"} -${historyBlock} - -Hard structure rules (do not violate): -- starter_code and reference_solution must be a single read-only query (WITH/SELECT only) -- test_suite must be valid JSON (not code); include schema_sql + 8 cases -- Each case expected.columns must match actual output column names -- KEY FIX: If "Expected rows" mismatches "Actual rows" by order, you MUST add "ORDER BY" to the query and set "order_matters": true. -- KEY FIX: If "Actual rows" are empty or wrong, check your JOIN/WHERE logic. - -Here is your previous output (may be truncated): -${rawSnippet || "(not provided)"} - -Here is your previous JSON (preferred to edit if present): -${previousJson || "(not provided)"} - -Goal: -- Return corrected JSON with the exact same fields. -- Prefer keeping id/title/description/starter_code stable. -- You MAY update test_suite and/or reference_solution, but the final pair MUST pass in Docker/SQLite. - -Return ONLY valid JSON. No markdown. No code fences. No prose.`; -} - -function buildRepairPrompt(slot: ProblemSlot, repair: RepairContext, ctx?: SlotPromptContext): string { - if (slot.language === "python") return buildPythonRepairPrompt(slot, repair, ctx); - if (slot.language === "cpp") return buildCppRepairPrompt(slot, repair, ctx); - if (slot.language === "sql") return buildSqlRepairPrompt(slot, repair, ctx); - return buildJavaRepairPrompt(slot, repair, ctx); -} - -/** - * Generate a single problem for the given slot via one Codex LLM call. - * - * Returns GeneratedProblemDraft (includes reference_solution). - * Validates JSON shape and test suite structure. - * Does NOT validate reference solution via Docker (that's the next step). - * Does NOT retry (caller handles retries). - * - * Throws on any validation failure. - */ -export async function generateSingleProblem( - slot: ProblemSlot, - opts?: { repair?: RepairContext; promptContext?: SlotPromptContext } -): Promise { - // Validation-time repair: if Docker/JUnit rejected the reference artifact, don't regenerate the entire - // problem JSON. Repair the reference artifact directly (smaller search space => fewer non-healing loops). - if (slot.language === "java" && opts?.repair?.previousDraft && "reference_solution" in opts.repair.previousDraft) { - const prev = opts.repair.previousDraft as Extract< - GeneratedProblemDraft, - { language: "java"; reference_solution: string } - >; - const { reference_solution, llmOutputHash, llm } = await repairJavaReferenceSolution({ - slot, - draft: prev, - errorMessage: opts.repair.errorMessage ?? "reference solution failed Docker validation", - ...(typeof opts.repair.judgeStdout === "string" ? { judgeStdout: opts.repair.judgeStdout } : {}), - ...(typeof opts.repair.judgeStderr === "string" ? { judgeStderr: opts.repair.judgeStderr } : {}), - ...(typeof opts.promptContext !== "undefined" ? { ctx: opts.promptContext } : {}), - }); - if (reference_solution.trim() === prev.reference_solution.trim()) { - throw new Error("Java reference_solution repair made no changes."); - } - const nextDraft: GeneratedProblemDraft = { ...prev, reference_solution }; - const result = GeneratedProblemDraftSchema.safeParse(nextDraft); - if (!result.success) { - const first = result.error.issues[0]; - throw new Error(`Java reference_solution repair produced invalid draft: ${first?.message ?? "unknown error"}`); - } - // Ensure repaired draft still satisfies deterministic Java invariants before running Docker again. - assertJavaLegacyDraftInvariants(slot, result.data as any); - return { draft: result.data, meta: { llmOutputHash, ...(llm ? { llm } : {}) } }; - } - - const prompt = opts?.repair - ? buildRepairPrompt(slot, opts.repair, opts.promptContext) - : buildSlotPromptWithContext(slot, opts?.promptContext); - trace("generation.slot.start", { slotIndex: slot.index, difficulty: slot.difficulty, repair: Boolean(opts?.repair) }); - traceText("generation.prompt", prompt, { extra: { slotIndex: slot.index, repair: Boolean(opts?.repair) } }); - - let llmMeta: CompletionMeta | undefined; - const completion = await createCodemmCompletion({ - system: getSystemPromptForSlot(slot), - user: prompt, - temperature: opts?.repair ? REPAIR_TEMPERATURE : TEMPERATURE, - maxTokens: MAX_TOKENS, - }); - llmMeta = completion.meta; - - const text = completion.content - .map((block) => (block.type === "text" ? block.text : "")) - .join("\n"); - const llmOutputHash = sha256(text); - traceText("generation.llm.raw", text, { extra: { slotIndex: slot.index } }); - - try { - // Parse JSON (reuse legacy robust parser) - const parsed = tryParseJson(text); - - if (!parsed || typeof parsed !== "object") { - throw new Error("LLM response is not a valid JSON object."); - } - - // Normalize fields (defensive, same pattern as legacy agent) - const raw = parsed as any; - - if (slot.language === "python") { - if (raw.workspace || raw.reference_workspace) { - throw new Error("Python generation does not support workspace problems yet."); - } - - const baseId = - typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : crypto.randomUUID(); - - const title = - typeof raw.title === "string" && raw.title.trim() - ? raw.title.trim() - : `Problem for ${slot.topics[0] ?? "Python"}`; - - const description = - typeof raw.description === "string" && raw.description.trim() - ? raw.description.trim() - : `Problem description for ${title}.`; - - let starterCode = - typeof raw.starter_code === "string" && raw.starter_code.trim() ? raw.starter_code.trim() : ""; - if (!starterCode.trim()) { - starterCode = "def solve(x):\n # TODO: implement\n raise NotImplementedError\n"; - } - - let testSuite = - typeof raw.test_suite === "string" && raw.test_suite.trim() ? raw.test_suite.trim() : ""; - // Note: if the LLM omitted test_suite (or returned an invalid one), we attempt a one-shot - // repair later after schema validation. - - const referenceSolution = - typeof raw.reference_solution === "string" && raw.reference_solution.trim() - ? raw.reference_solution.trim() - : ""; - if (!referenceSolution.trim()) { - throw new Error(`Missing reference_solution for slot ${slot.index}.`); - } - - const rawConstraints = typeof raw.constraints === "string" ? raw.constraints.trim() : ""; - if (rawConstraints && rawConstraints !== slot.constraints) { - throw new Error(`Invalid constraints for slot ${slot.index}: must match slot.constraints exactly.`); - } - const constraints = slot.constraints; - - const samples = coerceNonEmptySamplePairs(raw, "example input"); - const sampleInputs = samples.sampleInputs; - const sampleOutputs = samples.sampleOutputs; - - const difficulty = slot.difficulty; - const topicTag = slot.topics[0] ?? "oop"; - - const draft: GeneratedProblemDraft = { - language: "python", - id: baseId, - title, - description, - starter_code: starterCode, - test_suite: testSuite, - reference_solution: referenceSolution, - constraints, - sample_inputs: sampleInputs, - sample_outputs: sampleOutputs, - difficulty, - topic_tag: topicTag, - }; - - let result = GeneratedProblemDraftSchema.safeParse(draft); - if (!result.success) { - const testSuiteIssue = result.error.issues.some((i) => i.path?.[0] === "test_suite"); - const otherIssues = result.error.issues.some((i) => i.path?.[0] !== "test_suite"); - - // Deterministic self-heal pass: if only test_suite is invalid, ask the LLM to repair it. - if (testSuiteIssue && !otherIssues) { - const msg = - result.error.issues - .slice(0, 6) - .map((i) => `${i.path?.length ? i.path.join(".") : "root"}: ${i.message}`) - .join(" | ") || "unknown error"; - - const repairedTestSuite = await repairPythonTestSuite({ - slot, - title, - description, - constraints, - starterCode, - referenceSolution, - previousTestSuite: testSuite, - errorMessage: msg, - }); - const repairedDraft: GeneratedProblemDraft = { ...draft, test_suite: repairedTestSuite }; - result = GeneratedProblemDraftSchema.safeParse(repairedDraft); - if (result.success) { - trace("generation.python.testSuite.repaired", { slotIndex: slot.index, title }); - } else { - const firstError = result.error.issues[0]; - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation after Python test_suite repair: ${firstError?.message ?? "unknown error"}` - ); - } - } else { - const firstError = result.error.issues[0]; - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation: ${firstError?.message ?? "unknown error"}` - ); - } - } - - const style = normalizeProblemStyle(slot.problem_style); - const parsed = result.data; - if (!("reference_solution" in parsed)) { - throw new Error("Internal error: expected Python draft to include reference_solution."); - } - - if (!isValidPytestTestSuiteForStyle(parsed.test_suite, style, 8)) { - throw new Error( - `Invalid test_suite for slot ${slot.index}: does not match problem_style=${style} requirements.` - ); - } - if (style === "return") { - if (hasPythonStdoutWrites(parsed.reference_solution)) { - throw new Error( - `Invalid reference_solution for slot ${slot.index}: problem_style=return must not write to stdout (no print/sys.stdout).` - ); - } - } else { - if (!hasPythonStdoutWrites(parsed.reference_solution)) { - throw new Error( - `Invalid reference_solution for slot ${slot.index}: problem_style=${style} must write the final answer to stdout (print/sys.stdout).` - ); - } - } - - trace("generation.draft.meta", { slotIndex: slot.index, title, language: "python", difficulty, topicTag }); - return { draft: parsed, meta: { llmOutputHash, ...(llmMeta ? { llm: llmMeta } : {}) } }; - } - - if (slot.language === "cpp") { - if (raw.workspace || raw.reference_workspace) { - throw new Error("C++ generation does not support workspace problems yet."); - } - - const baseId = - typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : crypto.randomUUID(); - - const title = - typeof raw.title === "string" && raw.title.trim() - ? raw.title.trim() - : `Problem for ${slot.topics[0] ?? "C++"}`; - - const description = - typeof raw.description === "string" && raw.description.trim() - ? raw.description.trim() - : `Problem description for ${title}.`; - - let starterCode = - typeof raw.starter_code === "string" && raw.starter_code.trim() ? raw.starter_code.trim() : ""; - if (!starterCode.trim()) { - starterCode = - '#include \\n\\n// Implement solve(...) below.\\n// Avoid I/O in solve().\\nauto solve(auto x) { (void)x; return 0; }\\n'; - } - - let testSuite = - typeof raw.test_suite === "string" && raw.test_suite.trim() ? raw.test_suite.trim() : ""; - if (!testSuite.trim()) { - throw new Error(`Invalid test_suite for slot ${slot.index}: missing.`); - } - - const referenceSolution = - typeof raw.reference_solution === "string" && raw.reference_solution.trim() - ? raw.reference_solution.trim() - : ""; - if (!referenceSolution.trim()) { - throw new Error(`Missing reference_solution for slot ${slot.index}.`); - } - - // Starter code must include a real solve(...) definition (comments don't count). - // If the model only returned includes + a comment, deterministically synthesize a minimal - // starter implementation based on the reference_solution signature (without leaking the solution body). - if (!/\bsolve\s*\(/.test(stripCppComments(starterCode))) { - const synthesized = synthesizeCppStarterCodeFromReference({ - referenceSolution, - fallbackTopic: slot.topics[0] ?? "cpp", - }); - if (synthesized) { - starterCode = synthesized.trim(); - } - } - - const rawConstraints = typeof raw.constraints === "string" ? raw.constraints.trim() : ""; - if (rawConstraints && rawConstraints !== slot.constraints) { - throw new Error(`Invalid constraints for slot ${slot.index}: must match slot.constraints exactly.`); - } - const constraints = slot.constraints; - - const samples = coerceNonEmptySamplePairs(raw, "example input"); - const sampleInputs = samples.sampleInputs; - const sampleOutputs = samples.sampleOutputs; - - const difficulty = slot.difficulty; - const topicTag = slot.topics[0] ?? "oop"; - - const draft: GeneratedProblemDraft = { - language: "cpp", - id: baseId, - title, - description, - starter_code: starterCode, - test_suite: testSuite, - reference_solution: referenceSolution, - constraints, - sample_inputs: sampleInputs, - sample_outputs: sampleOutputs, - difficulty, - topic_tag: topicTag, - }; - - let result = GeneratedProblemDraftSchema.safeParse(draft); - if (!result.success) { - const testSuiteIssue = result.error.issues.some((i) => i.path?.[0] === "test_suite"); - const diagnostics = testSuiteIssue ? diagnoseCppTestSuite(draft.test_suite) : undefined; - if (diagnostics) { - const maybeIncludeSnippet = process.env.CODEMM_TRACE_TEST_SUITES === "1"; - trace("generation.cpp.testSuite.invalid", { - slotIndex: slot.index, - checks: diagnostics, - ...(maybeIncludeSnippet ? { testSuiteSnippet: draft.test_suite.slice(0, 2000) } : {}), - }); - } - - const msg = - result.error.issues - .slice(0, 6) - .map((i) => `${i.path?.length ? i.path.join(".") : "root"}: ${i.message}`) - .join(" | ") || "unknown error"; - const msgWithDiagnostics = - diagnostics ? `${msg} | cpp_test_suite_checks=${JSON.stringify(diagnostics)}` : msg; - - // One deterministic self-heal pass: if only test_suite is invalid, ask the LLM to repair the test suite - // (keeps the overall problem stable while enforcing the strict harness contract). - const failedTestSuite = testSuiteIssue; - if (failedTestSuite) { - const repairedTestSuite = await repairCppTestSuite({ - slot, - title, - description, - constraints, - starterCode, - referenceSolution, - previousTestSuite: testSuite, - errorMessage: msgWithDiagnostics, - }); - const repairedDraft: GeneratedProblemDraft = { ...draft, test_suite: repairedTestSuite }; - result = GeneratedProblemDraftSchema.safeParse(repairedDraft); - if (result.success) { - trace("generation.cpp.testSuite.repaired", { slotIndex: slot.index, title }); - } else { - const repairedDiagnostics = diagnoseCppTestSuite(repairedTestSuite); - const maybeIncludeSnippet = process.env.CODEMM_TRACE_TEST_SUITES === "1"; - trace("generation.cpp.testSuite.repair_invalid", { - slotIndex: slot.index, - checks: repairedDiagnostics, - ...(maybeIncludeSnippet ? { testSuiteSnippet: repairedTestSuite.slice(0, 2000) } : {}), - }); - - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation after C++ test_suite repair: ${msgWithDiagnostics} | repaired_cpp_test_suite_checks=${JSON.stringify(repairedDiagnostics)}` - ); - } - } - - if (!result.success) { - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation: ${msgWithDiagnostics}` - ); - } - } - - trace("generation.draft.meta", { slotIndex: slot.index, title, language: "cpp", difficulty, topicTag }); - const style = normalizeProblemStyle(slot.problem_style); - const parsed = result.data; - if (!("reference_solution" in parsed)) { - throw new Error("Internal error: expected C++ draft to include reference_solution."); - } - if (style === "return") { - if (hasCppStdoutWrites(parsed.reference_solution)) { - throw new Error( - `Invalid reference_solution for slot ${slot.index}: problem_style=return must not write to stdout/stderr (no cout/cerr/printf).` - ); - } - if (looksLikeCppTestSuiteCapturesStdout(parsed.test_suite)) { - throw new Error( - `Invalid test_suite for slot ${slot.index}: problem_style=return should not capture stdout; compare returned values instead.` - ); - } - } else { - if (!hasCppStdoutWrites(parsed.reference_solution)) { - throw new Error( - `Invalid reference_solution for slot ${slot.index}: problem_style=${style} must write the final answer to stdout (use std::cout).` - ); - } - if (!looksLikeCppTestSuiteCapturesStdout(parsed.test_suite)) { - throw new Error( - `Invalid test_suite for slot ${slot.index}: problem_style=${style} must capture std::cout output and assert on it (redirect rdbuf).` - ); - } - } - return { draft: parsed, meta: { llmOutputHash, ...(llmMeta ? { llm: llmMeta } : {}) } }; - } - - if (slot.language === "sql") { - if (raw.workspace || raw.reference_workspace) { - throw new Error("SQL generation does not support workspace problems."); - } - - const baseId = - typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : crypto.randomUUID(); - - const title = - typeof raw.title === "string" && raw.title.trim() - ? raw.title.trim() - : `Problem for ${slot.topics[0] ?? "SQL"}`; - - const description = - typeof raw.description === "string" && raw.description.trim() - ? raw.description.trim() - : `Problem description for ${title}.`; - - let starterCode = - typeof raw.starter_code === "string" && raw.starter_code.trim() ? raw.starter_code.trim() : ""; - if (!starterCode.trim()) starterCode = "SELECT 1;"; - - const testSuite = coerceSqlTestSuiteToJsonString((raw as any).test_suite, 8); - if (!testSuite.trim()) { - throw new Error(`Invalid test_suite for slot ${slot.index}: missing.`); - } - - const referenceSolution = - typeof raw.reference_solution === "string" && raw.reference_solution.trim() - ? raw.reference_solution.trim() - : ""; - if (!referenceSolution.trim()) { - throw new Error(`Missing reference_solution for slot ${slot.index}.`); - } - - const rawConstraints = typeof raw.constraints === "string" ? raw.constraints.trim() : ""; - if (rawConstraints && rawConstraints !== slot.constraints) { - throw new Error(`Invalid constraints for slot ${slot.index}: must match slot.constraints exactly.`); - } - const constraints = slot.constraints; - - const samples = coerceNonEmptySamplePairs(raw, "example input"); - const sampleInputs = samples.sampleInputs; - const sampleOutputs = samples.sampleOutputs; - - const difficulty = slot.difficulty; - const topicTag = slot.topics[0] ?? "oop"; - - const draft: GeneratedProblemDraft = { - language: "sql", - id: baseId, - title, - description, - starter_code: starterCode, - test_suite: testSuite, - reference_solution: referenceSolution, - constraints, - sample_inputs: sampleInputs, - sample_outputs: sampleOutputs, - difficulty, - topic_tag: topicTag, - }; - - const result = GeneratedProblemDraftSchema.safeParse(draft); - if (!result.success) { - const firstError = result.error.issues[0]; - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation: ${firstError?.message ?? "unknown error"}` - ); - } - - trace("generation.draft.meta", { slotIndex: slot.index, title, language: "sql", difficulty, topicTag }); - return { draft: result.data, meta: { llmOutputHash, ...(llmMeta ? { llm: llmMeta } : {}) } }; - } - - // Workspace variant (Phase B): accept workspace + reference_workspace. - if (raw.workspace && raw.reference_workspace) { - const rewrites: Array<{ id: string; applied: boolean; detail?: string }> = []; - const title = - typeof raw.title === "string" && raw.title.trim() - ? raw.title.trim() - : `Problem for ${slot.topics[0] ?? "Java"}`; - - const description = - typeof raw.description === "string" && raw.description.trim() - ? raw.description.trim() - : `Problem description for ${title}.`; - - let testSuite = - typeof raw.test_suite === "string" && raw.test_suite.trim() ? raw.test_suite.trim() : ""; - if (!isValidJUnit5TestSuite(testSuite, 8)) { - throw new Error( - `Invalid test_suite for slot ${slot.index}: must have exactly 8 @Test methods, JUnit 5 imports, no package, and non-trivial assertions.` - ); - } - if (hasBrittleWhitespaceStringExpectations(testSuite)) { - const sanitized = sanitizeJavaStringLiteralsBoundaryWhitespace(testSuite); - if (sanitized.changed && !hasBrittleWhitespaceStringExpectations(sanitized.testSuite)) { - // Deterministically de-brittle common whitespace patterns rather than wasting an LLM retry. - // This is safe because (by policy) whitespace behavior is not intended to be the core of the task. - // If a whitespace-specific problem is ever desired, it must be explicitly specified in the prompt/template. - testSuite = sanitized.testSuite; - } else { - throw new Error( - `Invalid test_suite for slot ${slot.index}: avoid assertEquals() against string literals with leading/trailing whitespace (brittle).` - ); - } - } - - const target = getWorkspaceTargetFile(raw); - if (!target || typeof target.path !== "string") { - throw new Error("workspace must include at least one file."); - } - - const targetClassName = target.path.replace(/\.java$/i, ""); - const expectedTestClassName = `${targetClassName}Test`; - { - const renamed = rewriteJavaTopLevelPublicClassName({ source: testSuite, expectedName: expectedTestClassName }); - if (renamed.changed) { - testSuite = renamed.source; - rewrites.push({ - id: "java.rename_test_class", - applied: true, - detail: `Renamed public test class "${renamed.previousName}" -> "${expectedTestClassName}".`, - }); - } - const actualTestClassName = inferPrimaryClassName(testSuite, expectedTestClassName); - if (actualTestClassName !== expectedTestClassName) { - throw new ObligationViolationError( - `Test suite class name "${actualTestClassName}" must match "${expectedTestClassName}".`, - { obligationId: "java.test_class_matches_target" } - ); - } - } - - // Do not require tests to explicitly reference the target type at contract-time. - // The real guardrail is Docker validation of the reference workspace against the test suite. - // Overly strict string matching here causes repeated retries without improving correctness. - - // Enforce structural topic requirements for selected Java OOP topics (deterministic, narrow). - try { - const refCombined = (raw.reference_workspace.files as any[]) - .filter((f) => f && typeof f.content === "string") - .map((f) => String(f.content)) - .join("\n\n"); - if (/\bwhile\s*\(\s*false\s*\)\s*\{?/.test(refCombined)) { - throw new ObligationViolationError('reference workspace must not include "while(false)" (unreachable).', { - obligationId: "java.no_while_false", - }); - } - assertJavaStructuralTopicRequirements({ - topics: slot.topics, - referenceSource: refCombined, - testSuite, - }); - } catch (e: any) { - if (e instanceof ObligationViolationError) throw e; - const innerMsg = e?.message ?? String(e); - const mapped = mapJavaStructuralTopicErrorToObligationId(innerMsg); - const fallback: ObligationId = - (slot.topics.some((t) => String(t).toLowerCase().includes("inheritance")) && "java.structural_topic.inheritance") || - (slot.topics.some((t) => String(t).toLowerCase().includes("abstraction")) && "java.structural_topic.abstraction") || - (slot.topics.some((t) => String(t).toLowerCase().includes("encapsulation")) && "java.structural_topic.encapsulation") || - (slot.topics.some((t) => String(t).toLowerCase().includes("composition")) && "java.structural_topic.composition") || - "java.structural_topic.polymorphism"; - throw new ObligationViolationError( - `Java topic structure validation failed for slot ${slot.index}: ${innerMsg}`, - { obligationId: mapped ?? fallback } - ); - } - - // stdout-only enforcement: reference + tests must be output-driven (not return-only). - if (!javaUsesStdout((raw.reference_workspace.files as any[]).map((f: any) => String(f?.content ?? "")).join("\n\n"))) { - throw new ObligationViolationError( - "For stdout-style Java problems, reference solution must write the final answer to stdout (System.out.print/println/printf).", - { obligationId: "java.stdout_solution_prints" } - ); - } - if (!javaTestSuiteCapturesStdout(testSuite)) { - throw new ObligationViolationError( - "For stdout-style Java problems, test_suite must capture stdout and assert on the printed output.", - { obligationId: "java.stdout_tests_capture" } - ); - } - - // Ensure file constraints: at most one public class per file + filename matches public class. - for (const file of raw.workspace.files as any[]) { - if (!file || typeof file.path !== "string" || typeof file.content !== "string") continue; - const keep = String(file.path).replace(/\.java$/i, ""); - const rewritten = demoteExtraTopLevelPublicTypes(file.content, { keepName: keep }); - if (rewritten.changed) { - file.content = rewritten.source; - rewrites.push({ - id: "java.demote_extra_public_types", - applied: true, - detail: `Demoted extra public types in "${file.path}".`, - }); - } - if (getTopLevelPublicTypeNames(file.content).length > 1) { - throw new ObligationViolationError(`File "${file.path}" must not declare more than one top-level public type.`, { - obligationId: "java.single_public_type_per_unit", - }); - } - assertJavaFilenameMatchesPublicClass(file.path, file.content); - } - - for (const file of raw.reference_workspace.files as any[]) { - if (!file || typeof file.path !== "string" || typeof file.content !== "string") continue; - const keep = String(file.path).replace(/\.java$/i, ""); - const rewritten = demoteExtraTopLevelPublicTypes(file.content, { keepName: keep }); - if (rewritten.changed) { - file.content = rewritten.source; - rewrites.push({ - id: "java.demote_extra_public_types", - applied: true, - detail: `Demoted extra public types in "${file.path}".`, - }); - } - if (getTopLevelPublicTypeNames(file.content).length > 1) { - throw new ObligationViolationError(`File "${file.path}" must not declare more than one top-level public type.`, { - obligationId: "java.single_public_type_per_unit", - }); - } - assertJavaFilenameMatchesPublicClass(file.path, file.content); - } - - // Ensure reference workspace has same file paths. - const studentPaths = new Set((raw.workspace.files as any[]).map((f) => String(f.path))); - const refPaths = new Set((raw.reference_workspace.files as any[]).map((f) => String(f.path))); - if (studentPaths.size !== refPaths.size) { - throw new Error("reference_workspace must include the same file paths as workspace."); - } - for (const p of studentPaths) { - if (!refPaths.has(p)) { - throw new Error("reference_workspace must include the same file paths as workspace."); - } - } - - const rawConstraints = typeof raw.constraints === "string" ? raw.constraints.trim() : ""; - if (rawConstraints && rawConstraints !== slot.constraints) { - throw new Error(`Invalid constraints for slot ${slot.index}: must match slot.constraints exactly.`); - } - const constraints = slot.constraints; - - const samples = coerceNonEmptySamplePairs(raw, "stdin"); - const sampleInputs = samples.sampleInputs; - const sampleOutputs = samples.sampleOutputs; - if (samples.changed) { - rewrites.push({ id: "samples.autofill", applied: true, detail: "Filled missing/mismatched sample_inputs/sample_outputs." }); - } - - const difficulty = slot.difficulty; - const topicTag = slot.topics[0] ?? "oop"; - - const draft: GeneratedProblemDraft = { - language: "java", - id: - typeof raw.id === "string" && raw.id.trim() - ? raw.id.trim() - : crypto.randomUUID(), - title, - description, - workspace: raw.workspace, - reference_workspace: raw.reference_workspace, - test_suite: testSuite, - constraints, - sample_inputs: sampleInputs, - sample_outputs: sampleOutputs, - difficulty, - topic_tag: topicTag, - }; - - const result = GeneratedProblemDraftSchema.safeParse(draft); - if (!result.success) { - const firstError = result.error.issues[0]; - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation: ${firstError?.message ?? "unknown error"}` - ); - } - - trace("generation.draft.meta", { slotIndex: slot.index, title, className: targetClassName, difficulty, topicTag }); - return { draft: result.data, meta: { llmOutputHash, ...(llmMeta ? { llm: llmMeta } : {}), rewrites } }; - } - - const baseId = - typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : crypto.randomUUID(); - - const title = - typeof raw.title === "string" && raw.title.trim() - ? raw.title.trim() - : `Problem for ${slot.topics[0] ?? "Java"}`; - - const description = - typeof raw.description === "string" && raw.description.trim() - ? raw.description.trim() - : `Problem description for ${title}.`; - - const rewrites: Array<{ id: string; applied: boolean; detail?: string }> = []; - - let starterCode = - typeof raw.starter_code === "string" && raw.starter_code.trim() ? raw.starter_code.trim() : ""; - - { - const rewritten = demoteExtraTopLevelPublicTypes(starterCode); - if (rewritten.changed) { - starterCode = rewritten.source; - rewrites.push({ - id: "java.demote_extra_public_types", - applied: true, - detail: "Demoted extra top-level public types in starter_code.", - }); - } - } - - // If starter_code missing or has package, synthesize - let className = inferPrimaryClassName(starterCode, `Problem${slot.index + 1}`); - if (!starterCode.trim() || /^\s*package\s+/m.test(starterCode)) { - starterCode = buildDefaultClassSkeleton(className); - className = inferPrimaryClassName(starterCode, `Problem${slot.index + 1}`); - } - - let starterPublicTypes = getTopLevelPublicTypeNames(starterCode); - if (starterPublicTypes.length > 1) { - throw new ObligationViolationError("starter_code must not declare more than one top-level public type.", { - obligationId: "java.single_public_type_per_unit", - }); - } - if (starterPublicTypes.length === 0) { - const promoted = promoteOneTopLevelTypeToPublic(starterCode, { keepName: className }); - if (promoted.changed) { - starterCode = promoted.source; - rewrites.push({ - id: "java.promote_public_type", - applied: true, - detail: `Promoted top-level type "${promoted.promotedName ?? className}" to public.`, - }); - starterPublicTypes = getTopLevelPublicTypeNames(starterCode); - } - } - if (starterPublicTypes.length === 0) { - throw new ObligationViolationError("starter_code must declare exactly one top-level public type.", { - obligationId: "java.single_public_type_per_unit", - }); - } - - let testSuite = - typeof raw.test_suite === "string" && raw.test_suite.trim() ? raw.test_suite.trim() : ""; - - let referenceSolution = - typeof raw.reference_solution === "string" && raw.reference_solution.trim() - ? raw.reference_solution.trim() - : ""; - - if (!referenceSolution.trim()) { - throw new Error(`Missing reference_solution for slot ${slot.index}.`); - } - - { - const rewritten = demoteExtraTopLevelPublicTypes(referenceSolution, { keepName: className }); - if (rewritten.changed) { - referenceSolution = rewritten.source; - rewrites.push({ - id: "java.demote_extra_public_types", - applied: true, - detail: "Demoted extra top-level public types in reference_solution.", - }); - } - } - - let refPublicTypes = getTopLevelPublicTypeNames(referenceSolution); - if (refPublicTypes.length > 1) { - throw new ObligationViolationError("reference_solution must not declare more than one top-level public type.", { - obligationId: "java.single_public_type_per_unit", - }); - } - if (refPublicTypes.length === 0) { - const promoted = promoteOneTopLevelTypeToPublic(referenceSolution, { keepName: className }); - if (promoted.changed) { - referenceSolution = promoted.source; - rewrites.push({ - id: "java.promote_public_type", - applied: true, - detail: `Promoted top-level type "${promoted.promotedName ?? className}" to public in reference_solution.`, - }); - refPublicTypes = getTopLevelPublicTypeNames(referenceSolution); - } - } - if (refPublicTypes.length === 0) { - throw new ObligationViolationError("reference_solution must declare exactly one top-level public type.", { - obligationId: "java.single_public_type_per_unit", - }); - } - - // Ensure reference solution has no package - if (/^\s*package\s+/m.test(referenceSolution)) { - throw new Error(`reference_solution for slot ${slot.index} contains package declaration.`); - } - - // Avoid pathological patterns that are guaranteed to fail compilation. - if (/\bwhile\s*\(\s*false\s*\)\s*\{?/.test(referenceSolution)) { - throw new ObligationViolationError('reference_solution must not include "while(false)" (unreachable statement).', { - obligationId: "java.no_while_false", - }); - } - - // Ensure reference solution matches class name (prefer public class too) - const refClassName = inferPrimaryClassName(referenceSolution, ""); - if (refClassName !== className) { - throw new ObligationViolationError( - `reference_solution class name "${refClassName}" does not match starter_code class name "${className}".`, - { obligationId: "java.primary_type_matches_target" } - ); - } - - // Ensure test class name matches starter_code class name + "Test" - const expectedTestClassName = `${className}Test`; - - // If the reference solution reads stdin, we either enforce a stdin-aware test suite, - // or (when no structural topics are required) deterministically derive tests from sample I/O. - const usesStdin = javaUsesStdin(referenceSolution); - const requiresStructuralTopics = hasJavaStructuralTopics(slot.topics); - if (usesStdin && requiresStructuralTopics) { - throw new ObligationViolationError( - "stdin reads (Scanner/System.in) are not allowed for Java structural-topic slots (encapsulation/inheritance/polymorphism/etc). Use pure methods and deterministic unit tests instead.", - { obligationId: "java.stdin_disallowed_for_structural_topics" } - ); - } - - if (usesStdin && !hasJavaMainMethod(referenceSolution)) { - throw new ObligationViolationError( - "stdin-driven Java problems must provide a public static void main(String[] args) entrypoint so tests can execute deterministically.", - { obligationId: "java.stdin_requires_main" } - ); - } - - let sampleInputs: string[] | null = null; - let sampleOutputs: string[] | null = null; - - if (usesStdin) { - const stdinSamples = Array.isArray((raw as any).sample_inputs) - ? (raw as any).sample_inputs - .map((x: any) => String(x ?? "").replace(/\r\n/g, "\n")) - .filter((x: string) => x.trim().length > 0) - : []; - if (stdinSamples.length < 8) { - throw new ObligationViolationError( - `stdin-driven Java problems must include at least 8 non-empty sample_inputs (each is a full stdin transcript). Got ${stdinSamples.length}.`, - { obligationId: "java.stdin_tests_provide" } - ); - } - - const { stdoutSamples } = await computeJavaStdoutSamplesByExecutingReference({ - referenceSolution, - stdinSamples, - maxSamples: 8, - }); - - testSuite = buildJavaStdinSampleDrivenJUnitTestSuite({ - testClassName: expectedTestClassName, - mainClassName: className, - cases: Array.from({ length: 8 }, (_, i) => ({ - stdin: stdinSamples[i] ?? "", - expectedStdout: stdoutSamples[i] ?? "", - })), - }); - - rewrites.push({ - id: "java.tests.from_samples", - applied: true, - detail: "Rebuilt test_suite deterministically from sample_inputs by executing the reference_solution in Docker.", - }); - sampleInputs = stdinSamples.slice(0, 8); - sampleOutputs = stdoutSamples.slice(0, 8); - } - - // Validate test suite structure strictly (after potential deterministic rebuild). - if (!isValidJUnit5TestSuite(testSuite, 8)) { - throw new Error( - `Invalid test_suite for slot ${slot.index}: must have exactly 8 @Test methods, JUnit 5 imports, no package, and non-trivial assertions.` - ); - } - if (hasBrittleWhitespaceStringExpectations(testSuite)) { - const sanitized = sanitizeJavaStringLiteralsBoundaryWhitespace(testSuite); - if (sanitized.changed && !hasBrittleWhitespaceStringExpectations(sanitized.testSuite)) { - testSuite = sanitized.testSuite; - } else { - throw new Error( - `Invalid test_suite for slot ${slot.index}: avoid assertEquals() against string literals with leading/trailing whitespace (brittle).` - ); - } - } - - { - const renamed = rewriteJavaTopLevelPublicClassName({ source: testSuite, expectedName: expectedTestClassName }); - if (renamed.changed) { - testSuite = renamed.source; - rewrites.push({ - id: "java.rename_test_class", - applied: true, - detail: `Renamed public test class "${renamed.previousName}" -> "${expectedTestClassName}".`, - }); - } - const actualTestClassName = inferPrimaryClassName(testSuite, expectedTestClassName); - if (actualTestClassName !== expectedTestClassName) { - throw new ObligationViolationError( - `Test suite class name "${actualTestClassName}" must match "${expectedTestClassName}".`, - { obligationId: "java.test_class_matches_target" } - ); - } - } - - // Enforce structural topic requirements for selected Java OOP topics (deterministic, narrow). - try { - assertJavaStructuralTopicRequirements({ - topics: slot.topics, - referenceSource: referenceSolution, - testSuite, - }); - } catch (e: any) { - if (e instanceof ObligationViolationError) throw e; - const innerMsg = e?.message ?? String(e); - const mapped = mapJavaStructuralTopicErrorToObligationId(innerMsg); - const fallback: ObligationId = - (slot.topics.some((t) => String(t).toLowerCase().includes("inheritance")) && "java.structural_topic.inheritance") || - (slot.topics.some((t) => String(t).toLowerCase().includes("abstraction")) && "java.structural_topic.abstraction") || - (slot.topics.some((t) => String(t).toLowerCase().includes("encapsulation")) && "java.structural_topic.encapsulation") || - (slot.topics.some((t) => String(t).toLowerCase().includes("composition")) && "java.structural_topic.composition") || - "java.structural_topic.polymorphism"; - throw new ObligationViolationError( - `Java topic structure validation failed for slot ${slot.index}: ${innerMsg}`, - { obligationId: mapped ?? fallback } - ); - } - - // stdout-only enforcement: reference + tests must be output-driven (not return-only). - if (!javaUsesStdout(referenceSolution)) { - throw new ObligationViolationError( - "For stdout-style Java problems, reference_solution must write the final answer to stdout (System.out.print/println/printf).", - { obligationId: "java.stdout_solution_prints" } - ); - } - if (!javaTestSuiteCapturesStdout(testSuite)) { - throw new ObligationViolationError( - "For stdout-style Java problems, test_suite must capture stdout and assert on the printed output.", - { obligationId: "java.stdout_tests_capture" } - ); - } - if (usesStdin && !javaTestSuiteSetsStdin(testSuite)) { - throw new ObligationViolationError( - "For stdin-driven Java problems, test_suite must set deterministic stdin (System.setIn / ByteArrayInputStream) before executing the code.", - { obligationId: "java.stdin_tests_provide" } - ); - } - - const rawConstraints = typeof raw.constraints === "string" ? raw.constraints.trim() : ""; - if (rawConstraints && rawConstraints !== slot.constraints) { - throw new Error(`Invalid constraints for slot ${slot.index}: must match slot.constraints exactly.`); - } - const constraints = slot.constraints; - - if (!sampleInputs || !sampleOutputs) { - const samples = coerceNonEmptySamplePairs(raw, "stdin"); - sampleInputs = samples.sampleInputs; - sampleOutputs = samples.sampleOutputs; - if (samples.changed) { - rewrites.push({ id: "samples.autofill", applied: true, detail: "Filled missing/mismatched sample_inputs/sample_outputs." }); - } - } - - const difficulty = slot.difficulty; - const topicTag = slot.topics[0] ?? "oop"; - - const draft: GeneratedProblemDraft = { - language: "java", - id: baseId, - title, - description, - starter_code: starterCode, - test_suite: testSuite, - reference_solution: referenceSolution, - constraints, - sample_inputs: sampleInputs, - sample_outputs: sampleOutputs, - difficulty, - topic_tag: topicTag, - }; - trace("generation.draft.meta", { slotIndex: slot.index, title, className, difficulty, topicTag }); - - // Validate against GeneratedProblemDraftSchema - const result = GeneratedProblemDraftSchema.safeParse(draft); - if (!result.success) { - const firstError = result.error.issues[0]; - throw new Error( - `Generated problem for slot ${slot.index} failed schema validation: ${firstError?.message ?? "unknown error"}` - ); - } - - return { draft: result.data, meta: { llmOutputHash, ...(llmMeta ? { llm: llmMeta } : {}), rewrites } }; - } catch (err: any) { - const msg = err?.message ?? String(err); - const obligationId = err instanceof ObligationViolationError ? err.obligationId : undefined; - throw new GenerationContractError(msg, { - slotIndex: slot.index, - llmOutputHash, - rawSnippet: text.slice(0, 2400), - ...(llmMeta ? { llm: llmMeta } : {}), - ...(obligationId ? { obligationId } : {}), - }); - } -} diff --git a/apps/backend/src/generation/progressBus.ts b/apps/backend/src/generation/progressBus.ts index 93da38a..0443177 100644 --- a/apps/backend/src/generation/progressBus.ts +++ b/apps/backend/src/generation/progressBus.ts @@ -9,28 +9,28 @@ type Channel = { cleanupTimer: NodeJS.Timeout | null; }; -const channelsBySessionId = new Map(); +const channelsByRunId = new Map(); -function getOrCreateChannel(sessionId: string): Channel { - const existing = channelsBySessionId.get(sessionId); +function getOrCreateChannel(runId: string): Channel { + const existing = channelsByRunId.get(runId); if (existing) return existing; const next: Channel = { listeners: new Set(), buffer: [], terminal: false, cleanupTimer: null }; - channelsBySessionId.set(sessionId, next); + channelsByRunId.set(runId, next); return next; } -function scheduleCleanup(sessionId: string, channel: Channel): void { +function scheduleCleanup(runId: string, channel: Channel): void { if (channel.cleanupTimer) return; channel.cleanupTimer = setTimeout(() => { - channelsBySessionId.delete(sessionId); + channelsByRunId.delete(runId); }, 5 * 60 * 1000); // Allow the process to exit naturally (important for tests/CLI). channel.cleanupTimer.unref?.(); } -export function publishGenerationProgress(sessionId: string, event: GenerationProgressEvent): void { - if (!sessionId) return; - const channel = getOrCreateChannel(sessionId); +export function publishGenerationProgress(runId: string, event: GenerationProgressEvent): void { + if (!runId) return; + const channel = getOrCreateChannel(runId); // Don't buffer heartbeats; they are only to keep UI responsive. if (event.type !== "heartbeat") { @@ -46,7 +46,7 @@ export function publishGenerationProgress(sessionId: string, event: GenerationPr event.type === "generation_failed" ) { channel.terminal = true; - scheduleCleanup(sessionId, channel); + scheduleCleanup(runId, channel); } if (channel.listeners.size === 0) return; @@ -59,21 +59,21 @@ export function publishGenerationProgress(sessionId: string, event: GenerationPr } } -export function getGenerationProgressBuffer(sessionId: string): GenerationProgressEvent[] { - const channel = channelsBySessionId.get(sessionId); +export function getGenerationProgressBuffer(runId: string): GenerationProgressEvent[] { + const channel = channelsByRunId.get(runId); return channel ? [...channel.buffer] : []; } -export function subscribeGenerationProgress(sessionId: string, listener: Listener): () => void { - const channel = getOrCreateChannel(sessionId); +export function subscribeGenerationProgress(runId: string, listener: Listener): () => void { + const channel = getOrCreateChannel(runId); channel.listeners.add(listener); return () => { - const c = channelsBySessionId.get(sessionId); + const c = channelsByRunId.get(runId); if (!c) return; c.listeners.delete(listener); if (c.listeners.size === 0 && c.terminal) { - scheduleCleanup(sessionId, c); + scheduleCleanup(runId, c); } }; } diff --git a/apps/backend/src/generation/referenceSolutionValidator.ts b/apps/backend/src/generation/referenceSolutionValidator.ts index 69887a4..7bf7df3 100644 --- a/apps/backend/src/generation/referenceSolutionValidator.ts +++ b/apps/backend/src/generation/referenceSolutionValidator.ts @@ -1,4 +1,5 @@ import type { GeneratedProblemDraft } from "../contracts/problem"; +import type { JudgeResult } from "../types"; import { traceText } from "../utils/trace"; import type { GenerationFailureKind } from "./errors"; import { getLanguageProfile } from "../languages/profiles"; @@ -8,10 +9,27 @@ export class ReferenceSolutionValidationError extends Error { judgeStderr: string; exitCode: number | undefined; kind: GenerationFailureKind; + failureCategory: string | undefined; + timeoutStage: "compile" | "execute" | "overall" | undefined; + watchdogSource: "inner" | "outer" | "unknown" | undefined; + parsedFailures: Record | undefined; + budgetProfile: Record | undefined; + judgeResult: JudgeResult | undefined; constructor( message: string, - opts: { stdout: string; stderr: string; exitCode?: number; kind: GenerationFailureKind } + opts: { + stdout: string; + stderr: string; + exitCode?: number; + kind: GenerationFailureKind; + failureCategory?: string; + timeoutStage?: "compile" | "execute" | "overall"; + watchdogSource?: "inner" | "outer" | "unknown"; + parsedFailures?: Record; + budgetProfile?: Record; + judgeResult?: JudgeResult; + } ) { super(message); this.name = "ReferenceSolutionValidationError"; @@ -19,6 +37,12 @@ export class ReferenceSolutionValidationError extends Error { this.judgeStderr = opts.stderr; this.exitCode = opts.exitCode; this.kind = opts.kind; + this.failureCategory = opts.failureCategory; + this.timeoutStage = opts.timeoutStage; + this.watchdogSource = opts.watchdogSource; + this.parsedFailures = opts.parsedFailures; + this.budgetProfile = opts.budgetProfile; + this.judgeResult = opts.judgeResult; } } @@ -34,7 +58,7 @@ export class ReferenceSolutionValidationError extends Error { * After this validation passes, the caller MUST discard reference_solution * before persisting the problem. */ -export async function validateReferenceSolution(draft: GeneratedProblemDraft): Promise { +export async function validateReferenceSolution(draft: GeneratedProblemDraft): Promise { const profile = getLanguageProfile(draft.language); if (!profile.judgeAdapter) { throw new Error(`No judge adapter configured for "${draft.language}".`); @@ -57,7 +81,7 @@ export async function validateReferenceSolution(draft: GeneratedProblemDraft): P const stderrLower = (result.stderr || "").toLowerCase(); const combinedLower = `${stdoutLower}\n${stderrLower}`; - if (result.timedOut) { + if (result.failureCategory === "TIME_BUDGET_EXCEEDED" || result.failureCategory === "EXEC_TIMEOUT" || result.timedOut) { throw new ReferenceSolutionValidationError( `Reference solution timed out for "${draft.title}".`, { @@ -65,6 +89,44 @@ export async function validateReferenceSolution(draft: GeneratedProblemDraft): P stderr: result.stderr, ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), kind: "timeout", + ...(result.failureCategory ? { failureCategory: result.failureCategory } : {}), + ...(result.timeoutStage ? { timeoutStage: result.timeoutStage } : {}), + ...(result.watchdogSource ? { watchdogSource: result.watchdogSource } : {}), + ...(result.parsedFailures ? { parsedFailures: result.parsedFailures } : {}), + ...(result.budgetProfile ? { budgetProfile: result.budgetProfile } : {}), + judgeResult: result, + } + ); + } + + if (result.failureCategory === "OUTPUT_LIMIT_EXCEEDED" || result.failureCategory === "OUTPUT_LIMIT") { + throw new ReferenceSolutionValidationError( + `Reference solution exceeded output limits for "${draft.title}".`, + { + stdout: result.stdout, + stderr: result.stderr, + ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), + kind: "infra", + ...(result.failureCategory ? { failureCategory: result.failureCategory } : {}), + ...(result.parsedFailures ? { parsedFailures: result.parsedFailures } : {}), + ...(result.budgetProfile ? { budgetProfile: result.budgetProfile } : {}), + judgeResult: result, + } + ); + } + + if (result.failureCategory === "JUDGE_INFRA_FAILURE" || result.failureCategory === "INFRA_ERROR") { + throw new ReferenceSolutionValidationError( + `Reference solution validation hit judge infrastructure failure for "${draft.title}".`, + { + stdout: result.stdout, + stderr: result.stderr, + ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), + kind: "infra", + ...(result.failureCategory ? { failureCategory: result.failureCategory } : {}), + ...(result.parsedFailures ? { parsedFailures: result.parsedFailures } : {}), + ...(result.budgetProfile ? { budgetProfile: result.budgetProfile } : {}), + judgeResult: result, } ); } @@ -82,7 +144,7 @@ export async function validateReferenceSolution(draft: GeneratedProblemDraft): P ? /\b(operationalerror|syntax error|no such table|no such column)\b/.test(combinedLower) : false; - if (hasCompileError || hasSqlError) { + if (result.failureCategory === "COMPILE_FAILURE" || result.failureCategory === "COMPILE_ERROR" || hasCompileError || hasSqlError) { const snippet = `${result.stderr || result.stdout || ""}`.slice(0, 1200); const fallback = snippet || `No compiler output captured (exitCode=${result.exitCode ?? "unknown"}).`; throw new ReferenceSolutionValidationError( @@ -92,12 +154,16 @@ export async function validateReferenceSolution(draft: GeneratedProblemDraft): P stderr: result.stderr, ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), kind: "compile", + ...(result.failureCategory ? { failureCategory: result.failureCategory } : {}), + ...(result.parsedFailures ? { parsedFailures: result.parsedFailures } : {}), + ...(result.budgetProfile ? { budgetProfile: result.budgetProfile } : {}), + judgeResult: result, } ); } // Check that tests pass - if (!result.success) { + if (result.failureCategory === "TEST_FAILURE" || !result.success) { const stdout = result.stdout || ""; const stderr = result.stderr || ""; const likelyJUnitFailure = @@ -112,10 +178,15 @@ export async function validateReferenceSolution(draft: GeneratedProblemDraft): P stderr: result.stderr, ...(result.exitCode === undefined ? {} : { exitCode: result.exitCode }), kind: "tests", + ...(result.failureCategory ? { failureCategory: result.failureCategory } : {}), + ...(result.parsedFailures ? { parsedFailures: result.parsedFailures } : {}), + ...(result.budgetProfile ? { budgetProfile: result.budgetProfile } : {}), + judgeResult: result, } ); } // Success: reference solution compiles and passes all tests. // Caller must discard reference_solution before persistence. + return result; } diff --git a/apps/backend/src/generation/services/executionBundle.ts b/apps/backend/src/generation/services/executionBundle.ts new file mode 100644 index 0000000..2bf6642 --- /dev/null +++ b/apps/backend/src/generation/services/executionBundle.ts @@ -0,0 +1,300 @@ +import crypto from "crypto"; +import { GeneratedProblemDraftSchema, type GeneratedProblemDraft } from "../../contracts/problem"; +import type { ProblemSlot } from "../../planner/types"; +import { JavaSourceNoPackageSchema, isValidJUnit5TestSuiteCountRange, javaTestSuiteCapturesStdout, javaTestSuiteSetsStdin } from "../../languages/java/rules"; +import { hasJavaStructuralTopics } from "../../languages/java/structuralTopics"; +import { PythonSourceSchema, isValidPytestTestSuiteForStyle } from "../../languages/python/rules"; +import { CppSourceSchema, isValidCppTestSuite } from "../../languages/cpp/rules"; +import { SqlQuerySchema, isValidSqlTestSuite } from "../../languages/sql/rules"; +import { getJudgeCompileTimeoutMs, getJudgeExecutionTimeoutMs, getJudgeTimeoutMs } from "../../judge/docker"; +import type { RepairStrategy } from "@codemm/shared-contracts"; + +type FindingSeverity = "info" | "warn" | "error"; + +export type ValidationFinding = { + code: string; + severity: FindingSeverity; + message: string; +}; + +export type ValidatedExecutionBundle = { + language: GeneratedProblemDraft["language"]; + normalizedStarterArtifact: string; + normalizedReferenceArtifact: string; + normalizedTestArtifact: string; + artifactHashes: { + starter?: string; + reference?: string; + tests?: string; + description?: string; + }; + staticFindings: ValidationFinding[]; + riskScore: number; + executionBudgetProfile: Record; + repairStrategy?: RepairStrategy | null; + slotPromptLineage: { + slotIndex: number; + language: string; + topicSignature: string; + problemStyle: string; + llmOutputHash?: string | null; + }; + draft: GeneratedProblemDraft; + bundleHash: string; +}; + +export class ExecutionBundleValidationError extends Error { + kind: + | "generation_schema_error" + | "static_rule_violation" + | "api_shape_mismatch" + | "complexity_risk_exceeded"; + findings: ValidationFinding[]; + riskScore: number; + + constructor( + message: string, + opts: { + kind: + | "generation_schema_error" + | "static_rule_violation" + | "api_shape_mismatch" + | "complexity_risk_exceeded"; + findings: ValidationFinding[]; + riskScore: number; + } + ) { + super(message); + this.name = "ExecutionBundleValidationError"; + this.kind = opts.kind; + this.findings = opts.findings; + this.riskScore = opts.riskScore; + } +} + +function sha256(text: string): string { + return crypto.createHash("sha256").update(text).digest("hex"); +} + +function topicSignature(slot: ProblemSlot): string { + return [...slot.topics].sort().join("|"); +} + +function normalizeStyle(raw: string): "stdout" | "return" | "mixed" { + const value = String(raw ?? "").trim().toLowerCase(); + if (value === "stdout" || value === "mixed") return value; + return "return"; +} + +function detectHighRiskNonTermination(source: string): string[] { + const text = String(source ?? ""); + const findings: string[] = []; + if (/\bwhile\s*\(\s*true\s*\)/.test(text) || /\bfor\s*\(\s*;\s*;\s*\)/.test(text)) findings.push("unbounded_loop"); + if (/\bwhile\s+True\s*:/.test(text)) findings.push("unbounded_loop"); + if (/\bloop\s*\{/.test(text)) findings.push("unbounded_loop"); + if (/\bThread\s*\.\s*sleep\s*\(/.test(text) || /\btime\s*\.\s*sleep\s*\(/.test(text)) findings.push("sleep_call"); + return findings; +} + +function buildBudgetProfile(language: GeneratedProblemDraft["language"]) { + const base: Record = { + overallTimeoutMs: getJudgeTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; + if (language === "java" || language === "cpp") { + base.compileTimeoutMs = getJudgeCompileTimeoutMs(); + } + return base; +} + +function getReferenceArtifact(draft: GeneratedProblemDraft): string { + if ("reference_solution" in draft && typeof draft.reference_solution === "string") { + return draft.reference_solution; + } + if ("reference_workspace" in draft) { + return draft.reference_workspace.files + .map((file: { path: string; content: string }) => `// ${file.path}\n${file.content}`) + .join("\n\n"); + } + return ""; +} + +function getStarterArtifact(draft: GeneratedProblemDraft): string { + if ("starter_code" in draft && typeof draft.starter_code === "string") return draft.starter_code; + if ("workspace" in draft) { + return draft.workspace.files + .map((file: { path: string; content: string }) => `// ${file.path}\n${file.content}`) + .join("\n\n"); + } + return ""; +} + +function pushSchemaIssues(findings: ValidationFinding[], issues: { message: string }[], code: string) { + for (const issue of issues) { + findings.push({ code, severity: "error", message: issue.message }); + } +} + +export function buildValidatedExecutionBundle(args: { + slot: ProblemSlot; + draft: GeneratedProblemDraft; + repairStrategy?: RepairStrategy | null; + llmOutputHash?: string | null; +}): ValidatedExecutionBundle { + const findings: ValidationFinding[] = []; + const parsed = GeneratedProblemDraftSchema.safeParse(args.draft); + if (!parsed.success) { + pushSchemaIssues(findings, parsed.error.issues, "generation_schema"); + const firstMessage = parsed.error.issues[0]?.message ?? "Draft failed generation schema validation."; + const lower = firstMessage.toLowerCase(); + const kind = + lower.includes("stdin") || lower.includes("must not") || lower.includes("read-only select") + ? "static_rule_violation" + : lower.includes("test_suite") || lower.includes("must define") || lower.includes("entrypoint") + ? "api_shape_mismatch" + : "generation_schema_error"; + throw new ExecutionBundleValidationError( + firstMessage, + { + kind, + findings, + riskScore: 100, + } + ); + } + + const draft = parsed.data; + const style = normalizeStyle(args.slot.problem_style); + const starterArtifact = getStarterArtifact(draft); + const referenceSource = getReferenceArtifact(draft); + + const highRisk = detectHighRiskNonTermination(referenceSource); + let riskScore = highRisk.length * 35; + if (style === "stdout") riskScore += 5; + if (draft.language === "java" && hasJavaStructuralTopics(args.slot.topics)) riskScore += 5; + + if (draft.language === "java") { + const sourceResult = JavaSourceNoPackageSchema.safeParse(referenceSource); + if (!sourceResult.success) pushSchemaIssues(findings, sourceResult.error.issues, "java_source"); + if (!isValidJUnit5TestSuiteCountRange(draft.test_suite, 1, args.slot.test_case_count)) { + findings.push({ + code: "java_test_suite_shape", + severity: "error", + message: `Java test suite must have 1-${args.slot.test_case_count} JUnit 5 tests with non-trivial assertions.`, + }); + } + if (style === "stdout" && !javaTestSuiteCapturesStdout(draft.test_suite)) { + findings.push({ + code: "java_stdout_capture_missing", + severity: "error", + message: "Java stdout-style slots must capture and assert stdout deterministically.", + }); + } + if (/\bScanner\s*\(|\bSystem\s*\.\s*in\b/.test(referenceSource) && !javaTestSuiteSetsStdin(draft.test_suite)) { + findings.push({ + code: "java_stdin_api_mismatch", + severity: "error", + message: "Java reference reads stdin but tests do not provide deterministic stdin.", + }); + } + } else if (draft.language === "python") { + const sourceResult = PythonSourceSchema.safeParse(referenceSource); + if (!sourceResult.success) pushSchemaIssues(findings, sourceResult.error.issues, "python_source"); + if (!isValidPytestTestSuiteForStyle(draft.test_suite, style, args.slot.test_case_count)) { + findings.push({ + code: "python_test_suite_shape", + severity: "error", + message: `Python pytest suite does not match the ${style} contract for ${args.slot.test_case_count} tests.`, + }); + } + } else if (draft.language === "cpp") { + const sourceResult = CppSourceSchema.safeParse(referenceSource); + if (!sourceResult.success) pushSchemaIssues(findings, sourceResult.error.issues, "cpp_source"); + if (!isValidCppTestSuite(draft.test_suite, args.slot.test_case_count)) { + findings.push({ + code: "cpp_test_suite_shape", + severity: "error", + message: `C++ test suite must define exactly ${args.slot.test_case_count} deterministic tests and include solution.cpp.`, + }); + } + } else if (draft.language === "sql") { + const sourceResult = SqlQuerySchema.safeParse(referenceSource); + if (!sourceResult.success) pushSchemaIssues(findings, sourceResult.error.issues, "sql_source"); + if (!isValidSqlTestSuite(draft.test_suite, args.slot.test_case_count)) { + findings.push({ + code: "sql_test_suite_shape", + severity: "error", + message: `SQL test suite JSON must define exactly ${args.slot.test_case_count} deterministic cases.`, + }); + } + } + + for (const code of highRisk) { + findings.push({ + code, + severity: code === "unbounded_loop" ? "error" : "warn", + message: + code === "unbounded_loop" + ? "Detected an unbounded loop pattern before execution." + : "Detected a sleep call that increases timeout risk.", + }); + } + + const blocking = findings.filter((finding) => finding.severity === "error"); + if (blocking.length > 0) { + const first = blocking[0]!; + const kind = + first.code.includes("test_suite") || first.code.includes("api_mismatch") + ? "api_shape_mismatch" + : first.code === "unbounded_loop" + ? "complexity_risk_exceeded" + : first.code.includes("schema") + ? "generation_schema_error" + : "static_rule_violation"; + throw new ExecutionBundleValidationError(first.message, { + kind, + findings, + riskScore: Math.min(100, Math.max(20, riskScore)), + }); + } + + const bundleHash = sha256( + JSON.stringify({ + language: draft.language, + starter: starterArtifact, + reference: referenceSource, + tests: draft.test_suite, + title: draft.title, + constraints: draft.constraints, + topics: args.slot.topics, + style, + repairStrategy: args.repairStrategy ?? null, + }) + ); + + return { + language: draft.language, + normalizedStarterArtifact: starterArtifact, + normalizedReferenceArtifact: referenceSource, + normalizedTestArtifact: draft.test_suite, + artifactHashes: { + starter: sha256(starterArtifact), + reference: sha256(referenceSource), + tests: sha256(draft.test_suite), + description: sha256(draft.description), + }, + staticFindings: findings, + riskScore: Math.min(100, Math.max(0, riskScore)), + executionBudgetProfile: buildBudgetProfile(draft.language), + repairStrategy: args.repairStrategy ?? null, + slotPromptLineage: { + slotIndex: args.slot.index, + language: args.slot.language, + topicSignature: topicSignature(args.slot), + problemStyle: style, + ...(args.llmOutputHash ? { llmOutputHash: args.llmOutputHash } : {}), + }, + draft, + bundleHash, + }; +} diff --git a/apps/backend/src/generation/services/validationService.ts b/apps/backend/src/generation/services/validationService.ts index 160c19b..afea506 100644 --- a/apps/backend/src/generation/services/validationService.ts +++ b/apps/backend/src/generation/services/validationService.ts @@ -1,3 +1,4 @@ +import crypto from "crypto"; import type { ProblemPlan } from "../../planner/types"; import type { GeneratedProblemDraft } from "../../contracts/problem"; import type { AttemptDiagnostic, SlotIntent } from "../../contracts/generationDiagnostics"; @@ -6,6 +7,13 @@ import { GenerationContractError, type GenerationFailureKind, } from "../errors"; +import { getTraceContext } from "../../utils/traceContext"; +import { + generationExecutionAttemptRepository, + generationRunFailureCacheRepository, + generationSlotDiagnosisRepository, +} from "../../database/repositories/generationRunRepository"; +import type { RepairStrategy } from "@codemm/shared-contracts"; import { ReferenceSolutionValidationError, validateReferenceSolution, @@ -13,8 +21,37 @@ import { import { TestStrengthGateError } from "../testStrengthGate"; import { isValidJUnit5TestSuiteCountRange, pruneJUnitTestMethods } from "../../languages/java/rules"; import { assertJavaStructuralTopicRequirements } from "../../languages/java/structuralTopics"; +import { + buildValidatedExecutionBundle, + ExecutionBundleValidationError, + type ValidatedExecutionBundle, +} from "./executionBundle"; export function inferFailureKind(err: unknown): GenerationFailureKind { + const explicitKind = (err as { kind?: unknown } | null)?.kind; + if ( + explicitKind === "generation_schema_error" || + explicitKind === "static_rule_violation" || + explicitKind === "api_shape_mismatch" || + explicitKind === "complexity_risk_exceeded" || + explicitKind === "compile_failure" || + explicitKind === "test_failure" || + explicitKind === "time_budget_exceeded" || + explicitKind === "output_limit_exceeded" || + explicitKind === "judge_infra_failure" || + explicitKind === "repair_no_progress" || + explicitKind === "run_policy_failure" || + explicitKind === "compile" || + explicitKind === "tests" || + explicitKind === "timeout" || + explicitKind === "contract" || + explicitKind === "quality" || + explicitKind === "llm" || + explicitKind === "unknown" + ) { + return explicitKind; + } + if (err instanceof ExecutionBundleValidationError) return err.kind; if (err instanceof ReferenceSolutionValidationError) return err.kind; if (err instanceof GenerationContractError) return "contract"; if (err instanceof TestStrengthGateError) return "quality"; @@ -25,6 +62,17 @@ export function inferFailureKind(err: unknown): GenerationFailureKind { } export function recommendedRemediation(kind: GenerationFailureKind): string[] { + if (kind === "generation_schema_error") return ["Regenerate this slot", "Tighten the generation contract"]; + if (kind === "static_rule_violation") return ["Regenerate this slot", "Inject stricter deterministic guardrails"]; + if (kind === "api_shape_mismatch") return ["Regenerate reference shape", "Regenerate test shape"]; + if (kind === "complexity_risk_exceeded") return ["Regenerate simpler logic", "Tighten constraints"]; + if (kind === "compile_failure") return ["Regenerate this slot", "Repair the reference implementation"]; + if (kind === "test_failure") return ["Repair the reference logic", "Regenerate this slot"]; + if (kind === "time_budget_exceeded") return ["Regenerate simpler logic", "Inject bounded-loop guardrails"]; + if (kind === "output_limit_exceeded") return ["Regenerate this slot", "Reduce extraneous output"]; + if (kind === "judge_infra_failure") return ["Retry this slot", "Re-run after judge health recovers"]; + if (kind === "repair_no_progress") return ["Change repair strategy", "Quarantine this slot"]; + if (kind === "run_policy_failure") return ["Adjust run policy", "Retry with a simpler request"]; if (kind === "compile") return ["Regenerate this slot", "Reduce difficulty for this slot"]; if (kind === "tests") return ["Regenerate this slot", "Narrow topic scope"]; if (kind === "timeout") return ["Regenerate this slot", "Reduce constraints and complexity"]; @@ -58,6 +106,183 @@ export async function validateDraftArtifacts(draft: GeneratedProblemDraft): Prom await validateReferenceSolution(draft); } +function sha256(text: string): string { + return crypto.createHash("sha256").update(text).digest("hex"); +} + +function trimSnippet(text: string | undefined, limit: number = 1200): string | null { + const clean = String(text ?? ""); + if (!clean.trim()) return null; + return clean.slice(0, limit); +} + +export function buildFailureDiagnosis(args: { + kind: GenerationFailureKind; + err: unknown; +}): { + diagnosisClass: string; + recoverability: "recoverable" | "fatal" | "quarantine"; + normalizedSymptom: string; + recommendedRepairStrategy: string | null; +} { + const message = String((args.err as any)?.message ?? "unknown failure"); + const lower = message.toLowerCase(); + + if (args.kind === "time_budget_exceeded" || args.kind === "timeout") { + return { + diagnosisClass: "timeout_or_nontermination", + recoverability: "recoverable", + normalizedSymptom: lower.includes("stdin") ? "stdin_timeout" : "execution_timeout", + recommendedRepairStrategy: "tighten_constraints", + }; + } + if (args.kind === "api_shape_mismatch" || args.kind === "contract") { + return { + diagnosisClass: "api_mismatch", + recoverability: "recoverable", + normalizedSymptom: lower.includes("stdin") ? "stdin_api_mismatch" : "shape_mismatch", + recommendedRepairStrategy: "regenerate_reference_shape", + }; + } + if (args.kind === "compile_failure" || args.kind === "compile") { + return { + diagnosisClass: "compile_breakage", + recoverability: "recoverable", + normalizedSymptom: "compile_failure", + recommendedRepairStrategy: "regenerate_reference_shape", + }; + } + if (args.kind === "test_failure" || args.kind === "tests") { + return { + diagnosisClass: "logic_bug", + recoverability: "recoverable", + normalizedSymptom: lower.includes("baseline") ? "quality_gate_failed" : "reference_failed_tests", + recommendedRepairStrategy: lower.includes("baseline") ? "regenerate_tests_shape" : "regenerate_reference_logic", + }; + } + if (args.kind === "quality") { + return { + diagnosisClass: "quality_gate_weak_tests", + recoverability: "recoverable", + normalizedSymptom: "weak_test_suite", + recommendedRepairStrategy: "regenerate_tests_shape", + }; + } + if (args.kind === "repair_no_progress") { + return { + diagnosisClass: "repair_no_progress", + recoverability: "quarantine", + normalizedSymptom: "repair_no_progress", + recommendedRepairStrategy: "quarantine_slot", + }; + } + if (args.kind === "judge_infra_failure" || args.kind === "infra") { + return { + diagnosisClass: "judge_infra_failure", + recoverability: "recoverable", + normalizedSymptom: "judge_infra_failure", + recommendedRepairStrategy: "inject_guardrails", + }; + } + return { + diagnosisClass: "logic_bug", + recoverability: "recoverable", + normalizedSymptom: "generic_generation_failure", + recommendedRepairStrategy: "regenerate_reference_logic", + }; +} + +export function persistExecutionAttempt(args: { + slotIndex: number; + attempt: number; + executionPhase: "compile" | "test_exec" | "quality_gate"; + bundle: Pick; + strategy?: string | null; + result?: { + startedAt: string; + finishedAt?: string | null; + exitCode?: number | null; + timeoutStage?: "compile" | "execute" | "overall" | null; + watchdogSource?: "inner" | "outer" | "unknown" | null; + failureCategory?: string | null; + stdout?: string; + stderr?: string; + parsedFailures?: unknown; + trace?: unknown; + }; +}): number | null { + const ctx = getTraceContext(); + if (!ctx?.runId) return null; + return generationExecutionAttemptRepository.create({ + runId: ctx.runId, + slotIndex: args.slotIndex, + attempt: args.attempt, + executionPhase: args.executionPhase, + bundleHash: args.bundle.bundleHash, + strategy: args.strategy ?? null, + budgetProfile: args.bundle.executionBudgetProfile, + startedAt: args.result?.startedAt ?? new Date().toISOString(), + finishedAt: args.result?.finishedAt ?? new Date().toISOString(), + exitCode: args.result?.exitCode ?? null, + timeoutStage: args.result?.timeoutStage ?? null, + watchdogSource: args.result?.watchdogSource ?? null, + failureCategory: args.result?.failureCategory ?? null, + stdoutHash: args.result?.stdout ? sha256(args.result.stdout) : null, + stderrHash: args.result?.stderr ? sha256(args.result.stderr) : null, + stdoutSnippet: trimSnippet(args.result?.stdout), + stderrSnippet: trimSnippet(args.result?.stderr), + parsedFailures: args.result?.parsedFailures, + trace: args.result?.trace, + }); +} + +export function persistFailureDiagnosis(args: { + slot: ProblemPlan[number]; + attempt: number; + kind: GenerationFailureKind; + err: unknown; + sourceExecutionAttemptId?: number | null; +}): void { + const ctx = getTraceContext(); + if (!ctx?.runId) return; + const diagnosis = buildFailureDiagnosis({ kind: args.kind, err: args.err }); + generationSlotDiagnosisRepository.create({ + runId: ctx.runId, + slotIndex: args.slot.index, + attempt: args.attempt, + diagnosisClass: diagnosis.diagnosisClass, + recoverability: diagnosis.recoverability, + normalizedSymptom: diagnosis.normalizedSymptom, + recommendedRepairStrategy: diagnosis.recommendedRepairStrategy, + sourceExecutionAttemptId: args.sourceExecutionAttemptId ?? null, + }); + generationRunFailureCacheRepository.create({ + runId: ctx.runId, + language: args.slot.language, + topicSignature: [...args.slot.topics].sort().join("|"), + failureClass: args.kind, + normalizedSymptom: diagnosis.normalizedSymptom, + guardrailPatch: + diagnosis.normalizedSymptom === "stdin_timeout" || diagnosis.normalizedSymptom === "stdin_api_mismatch" + ? { injectGuardrails: ["No stdin reads", "Use pure deterministic methods", "Bound loops and recursion"] } + : { injectGuardrails: ["Keep solutions deterministic", "Avoid brittle edge-case assumptions"] }, + }); +} + +export function prepareValidatedExecutionBundle(args: { + slot: ProblemPlan[number]; + draft: GeneratedProblemDraft; + repairStrategy?: RepairStrategy | null; + llmOutputHash?: string | null; +}): ValidatedExecutionBundle { + return buildValidatedExecutionBundle({ + slot: args.slot, + draft: args.draft, + repairStrategy: args.repairStrategy ?? null, + llmOutputHash: args.llmOutputHash ?? null, + }); +} + export function progressSummaryForFailure(args: { slotIndex: number; attempt: number; diff --git a/apps/backend/src/ipc/activities.ts b/apps/backend/src/ipc/activities.ts index 74d278d..5fbf01d 100644 --- a/apps/backend/src/ipc/activities.ts +++ b/apps/backend/src/ipc/activities.ts @@ -1,7 +1,9 @@ import { z } from "zod"; import type { ActivityDetailDto, ActivityListResponseDto, ActivityResponseDto, PublishActivityResponseDto } from "@codemm/shared-contracts"; import { activityRepository } from "../database/repositories/activityRepository"; +import { threadRepository } from "../database/repositories/threadRepository"; import { editDraftProblemWithAi } from "../services/activityProblemEditService"; +import { parseGenerationOutcomes } from "../services/threads/shared"; import { getString, requireParams } from "./common"; import type { RpcHandlerDef } from "./types"; @@ -14,6 +16,9 @@ function toActivityDetailDto(dbActivity: { time_limit_seconds?: number | null; created_at: string; }): ActivityDetailDto { + const thread = threadRepository.findByActivityId(dbActivity.id); + const generationOutcomes = thread ? parseGenerationOutcomes(thread.generation_outcomes_json) : []; + const failedSlotIndexes = generationOutcomes.filter((outcome) => !outcome.success).map((outcome) => outcome.slotIndex); return { id: dbActivity.id, title: dbActivity.title, @@ -22,6 +27,9 @@ function toActivityDetailDto(dbActivity: { status: (dbActivity.status as ActivityDetailDto["status"]) ?? "DRAFT", timeLimitSeconds: typeof dbActivity.time_limit_seconds === "number" ? dbActivity.time_limit_seconds : null, createdAt: dbActivity.created_at, + threadId: thread?.id ?? null, + failedSlotIndexes, + failedSlotCount: failedSlotIndexes.length, }; } @@ -65,7 +73,9 @@ export function createActivityHandlers(): Record { if (!id) throw new Error("id is required."); const dbActivity = activityRepository.findById(id); if (!dbActivity) throw new Error("Activity not found."); - if ((dbActivity.status ?? "DRAFT") !== "DRAFT") throw new Error("This activity has already been published."); + if (!["DRAFT", "INCOMPLETE"].includes(dbActivity.status ?? "DRAFT")) { + throw new Error("This activity has already been published."); + } const title = typeof params.title === "string" ? params.title.trim() : undefined; const timeLimitSeconds = @@ -94,6 +104,9 @@ export function createActivityHandlers(): Record { const dbActivity = activityRepository.findById(id); if (!dbActivity) throw new Error("Activity not found."); if ((dbActivity.status ?? "DRAFT") === "PUBLISHED") return { ok: true } satisfies PublishActivityResponseDto; + if ((dbActivity.status ?? "DRAFT") === "INCOMPLETE") { + throw new Error("Incomplete activities cannot be published until all failed slots are repaired."); + } activityRepository.update(id, { status: "PUBLISHED" }); return { ok: true } satisfies PublishActivityResponseDto; }, @@ -118,7 +131,9 @@ export function createActivityHandlers(): Record { const dbActivity = activityRepository.findById(id); if (!dbActivity) throw new Error("Activity not found."); - if ((dbActivity.status ?? "DRAFT") !== "DRAFT") throw new Error("This activity has already been published."); + if (!["DRAFT", "INCOMPLETE"].includes(dbActivity.status ?? "DRAFT")) { + throw new Error("This activity has already been published."); + } let problems: any[] = []; try { diff --git a/apps/backend/src/ipc/threads.ts b/apps/backend/src/ipc/threads.ts index 9d81f77..1417912 100644 --- a/apps/backend/src/ipc/threads.ts +++ b/apps/backend/src/ipc/threads.ts @@ -8,6 +8,7 @@ import { generateFromThread, getThread, processThreadMessage, + repairFailedSlotsFromThread, regenerateSlotFromThread, setThreadInstructions, } from "../services/sessionService"; @@ -27,15 +28,16 @@ import { collectAttemptDiagnostics } from "../generation/diagnostics"; import { defaultAssistantPrompt, getNumber, getString, makeSubId, requireParams, safeJsonStringify } from "./common"; import type { RpcHandlerDef } from "./types"; -type Subscription = { threadId: string; unsubscribe: () => void }; +type Subscription = { threadId: string; runId: string; unsubscribe: () => void }; const generationSubs = new Map(); async function runGenerationWithRunTracking(args: { threadId: string; + runId?: string; meta: Record; - execute: () => Promise<{ activityId: string; problems: unknown[] }>; + execute: (runId: string) => Promise<{ activityId: string; problems: unknown[] }>; }): Promise { - const runId = crypto.randomUUID(); + const runId = typeof args.runId === "string" && args.runId.trim() ? args.runId : crypto.randomUUID(); const routePlan = getResolvedLlmSnapshot(); runRepository.create(runId, "generation", { threadId: args.threadId, @@ -46,7 +48,7 @@ async function runGenerationWithRunTracking(args: { }); let seq = 0; - const unsubPersist = subscribeGenerationProgress(args.threadId, (ev: GenerationProgressEvent) => { + const unsubPersist = subscribeGenerationProgress(runId, (ev: GenerationProgressEvent) => { seq += 1; try { runEventRepository.append(runId, seq, "progress", safeJsonStringify(ev)); @@ -56,7 +58,7 @@ async function runGenerationWithRunTracking(args: { }); try { - const { activityId, problems } = await args.execute(); + const { activityId, problems } = await args.execute(runId); runRepository.finish(runId, "succeeded"); return { activityId, problemCount: Array.isArray(problems) ? problems.length : 0, runId }; } catch (err) { @@ -137,6 +139,8 @@ export function createThreadHandlers(deps: { commitments: s.commitments, generationOutcomes: s.generationOutcomes, intentTrace: s.intentTrace, + latestGenerationRunId: s.latestGenerationRunId ?? null, + latestGenerationRunStatus: s.latestGenerationRunStatus ?? null, }; return response; }, @@ -181,19 +185,39 @@ export function createThreadHandlers(deps: { }, "threads.subscribeGeneration": { - schema: z.object({ threadId: z.string().min(1).max(128) }).passthrough(), + schema: z + .object({ + threadId: z.string().min(1).max(128), + runId: z.string().min(1).max(128).optional(), + }) + .passthrough(), handler: async (paramsRaw) => { const params = requireParams(paramsRaw); const threadId = getString(params.threadId); + const requestedRunId = getString(params.runId); if (!threadId) throw new Error("threadId is required."); getThread(threadId); + if (requestedRunId) { + const run = runRepository.findById(requestedRunId); + // Allow pre-subscribing to a caller-generated future runId before the run row exists. + // If the run already exists, it must still be a generation run owned by this thread. + if (run) { + if (run.kind !== "generation") { + throw new Error("runId does not reference a generation run."); + } + if (String(run.thread_id ?? "") !== threadId) { + throw new Error("runId does not belong to the provided threadId."); + } + } + } const subId = makeSubId(); const latest = runRepository.latestByThread(threadId, "generation"); + const effectiveRunId = requestedRunId ?? (latest && typeof latest.id === "string" ? latest.id : null); const buffered = (() => { - if (latest && typeof latest.id === "string" && latest.id) { - const rows = runEventRepository.listByRun(latest.id, 1500); + if (effectiveRunId) { + const rows = runEventRepository.listByRun(effectiveRunId, 1500); const events: GenerationProgressEvent[] = []; for (const r of rows) { if (r.type !== "progress") continue; @@ -206,14 +230,18 @@ export function createThreadHandlers(deps: { } if (events.length > 0) return events; } - return getGenerationProgressBuffer(threadId); + return effectiveRunId ? getGenerationProgressBuffer(effectiveRunId) : []; })(); - const unsubscribe = subscribeGenerationProgress(threadId, (ev: GenerationProgressEvent) => { + const subscribeRunId = effectiveRunId ?? requestedRunId; + if (!subscribeRunId) { + throw new Error("runId is required to subscribe before a generation run has been persisted."); + } + const unsubscribe = subscribeGenerationProgress(subscribeRunId, (ev: GenerationProgressEvent) => { deps.sendEvent("threads.generation", { subId, event: ev }); }); - generationSubs.set(subId, { threadId, unsubscribe }); + generationSubs.set(subId, { threadId, runId: subscribeRunId, unsubscribe }); - return { subId, buffered, ...(latest && typeof latest.id === "string" ? { runId: latest.id } : {}) }; + return { subId, buffered, runId: subscribeRunId }; }, }, @@ -236,29 +264,43 @@ export function createThreadHandlers(deps: { }, "threads.generate": { - schema: z.object({ threadId: z.string().min(1).max(128) }).passthrough(), + schema: z + .object({ + threadId: z.string().min(1).max(128), + runId: z.string().min(1).max(128).optional(), + }) + .passthrough(), handler: async (paramsRaw) => { const params = requireParams(paramsRaw); const threadId = getString(params.threadId); + const runId = getString(params.runId); if (!threadId) throw new Error("threadId is required."); return runGenerationWithRunTracking({ threadId, + ...(runId ? { runId } : {}), meta: { threadId, mode: "v1", operation: "generate" }, - execute: async () => generateFromThread(threadId), + execute: async (effectiveRunId) => generateFromThread(threadId, { runId: effectiveRunId }), }); }, }, "threads.generateV2": { - schema: z.object({ threadId: z.string().min(1).max(128) }).passthrough(), + schema: z + .object({ + threadId: z.string().min(1).max(128), + runId: z.string().min(1).max(128).optional(), + }) + .passthrough(), handler: async (paramsRaw) => { const params = requireParams(paramsRaw); const threadId = getString(params.threadId); + const runId = getString(params.runId); if (!threadId) throw new Error("threadId is required."); return runGenerationWithRunTracking({ threadId, + ...(runId ? { runId } : {}), meta: { threadId, mode: "v2", operation: "generate" }, - execute: async () => generateFromThread(threadId), + execute: async (effectiveRunId) => generateFromThread(threadId, { runId: effectiveRunId }), }); }, }, @@ -305,6 +347,28 @@ export function createThreadHandlers(deps: { }, }, + "threads.repairFailedSlots": { + schema: z + .object({ + threadId: z.string().min(1).max(128), + runId: z.string().min(1).max(128).optional(), + }) + .passthrough(), + handler: async (paramsRaw) => { + const params = requireParams(paramsRaw); + const threadId = getString(params.threadId); + const runId = getString(params.runId); + if (!threadId) throw new Error("threadId is required."); + + return runGenerationWithRunTracking({ + threadId, + ...(runId ? { runId } : {}), + meta: { threadId, mode: "v2", operation: "repair_failed_slots" }, + execute: async (effectiveRunId) => repairFailedSlotsFromThread(threadId, { runId: effectiveRunId }), + }); + }, + }, + "threads.getGenerationDiagnostics": { schema: z .object({ diff --git a/apps/backend/src/ipcServer.ts b/apps/backend/src/ipcServer.ts index 2be0918..73c183c 100644 --- a/apps/backend/src/ipcServer.ts +++ b/apps/backend/src/ipcServer.ts @@ -8,6 +8,7 @@ import { isObject, replyErr, replyOk, send, validateOrThrow } from "./ipc/server import { createJudgeHandlers } from "./ipc/judge"; import { createThreadHandlers, shutdownThreadHandlers } from "./ipc/threads"; import type { JsonObject, RpcHandlerDef } from "./ipc/types"; +import { reconcileInterruptedGenerationState } from "./services/threads/generationRecoveryService"; type RpcRequest = { id: string; @@ -74,6 +75,7 @@ function shutdown() { } initializeDatabase(); +reconcileInterruptedGenerationState(); process.on("message", onMessage); process.on("disconnect", () => process.exit(0)); diff --git a/apps/backend/src/judge/docker.ts b/apps/backend/src/judge/docker.ts index 3b020a2..89df16b 100644 --- a/apps/backend/src/judge/docker.ts +++ b/apps/backend/src/judge/docker.ts @@ -1,9 +1,11 @@ +import crypto from "crypto"; import { spawn } from "child_process"; export type SpawnCaptureResult = { stdout: string; stderr: string; exitCode: number; timedOut: boolean; + outputLimitExceeded: boolean; }; export function stripAnsi(text: string): string { @@ -18,6 +20,24 @@ export function getJudgeTimeoutMs(): number { return Math.min(Math.floor(n), 30_000); } +export function getJudgeCompileTimeoutMs(): number { + const overall = getJudgeTimeoutMs(); + const raw = process.env.JUDGE_COMPILE_TIMEOUT_MS; + if (!raw) return Math.max(1000, Math.min(overall - 1000, 8000)); + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return Math.max(1000, Math.min(overall - 1000, 8000)); + return Math.max(1000, Math.min(Math.floor(n), overall)); +} + +export function getJudgeExecutionTimeoutMs(): number { + const overall = getJudgeTimeoutMs(); + const raw = process.env.JUDGE_EXEC_TIMEOUT_MS; + if (!raw) return Math.max(1000, Math.min(overall - 1000, 6000)); + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) return Math.max(1000, Math.min(overall - 1000, 6000)); + return Math.max(1000, Math.min(Math.floor(n), overall)); +} + function resolveDockerBin(): string { const env = typeof process.env.DOCKER_PATH === "string" ? process.env.DOCKER_PATH.trim() : ""; if (env) return env; @@ -37,11 +57,24 @@ function killProcessBestEffort(proc: ReturnType) { } } +function cleanupContainerBestEffort(containerName: string) { + try { + const cleanup = spawn(resolveDockerBin(), ["rm", "-f", containerName], { + stdio: "ignore", + windowsHide: true, + }); + cleanup.unref(); + } catch { + // ignore + } +} + export async function spawnCapture(opts: { cmd: string; args: string[]; cwd: string; timeoutMs: number; + containerName?: string; maxBufferBytes?: number; env?: NodeJS.ProcessEnv; }): Promise { @@ -60,10 +93,12 @@ export async function spawnCapture(opts: { const stdoutChunks: Buffer[] = []; const stderrChunks: Buffer[] = []; let timedOut = false; + let outputLimitExceeded = false; const t = setTimeout(() => { timedOut = true; killProcessBestEffort(child); + if (opts.containerName) cleanupContainerBestEffort(opts.containerName); }, Math.max(1000, Math.floor(opts.timeoutMs))); const onData = (buf: Buffer, which: "stdout" | "stderr") => { @@ -77,8 +112,9 @@ export async function spawnCapture(opts: { } // If output is exploding, kill to prevent memory blowups. if (stdoutBytes + stderrBytes > maxBufferBytes * 4) { - timedOut = true; + outputLimitExceeded = true; killProcessBestEffort(child); + if (opts.containerName) cleanupContainerBestEffort(opts.containerName); } }; @@ -89,14 +125,14 @@ export async function spawnCapture(opts: { clearTimeout(t); const stdout = Buffer.concat(stdoutChunks).toString("utf8"); const stderr = Buffer.concat(stderrChunks).toString("utf8"); - resolve({ stdout, stderr, exitCode, timedOut }); + resolve({ stdout, stderr, exitCode, timedOut, outputLimitExceeded }); }; child.on("close", (code) => done(typeof code === "number" ? code : 1)); child.on("error", (e: any) => { clearTimeout(t); const msg = e && typeof e.message === "string" ? e.message : String(e); - resolve({ stdout: "", stderr: msg, exitCode: 1, timedOut: false }); + resolve({ stdout: "", stderr: msg, exitCode: 1, timedOut: false, outputLimitExceeded: false }); }); }); } @@ -109,5 +145,8 @@ export async function runDocker(opts: { }): Promise { const cmd = resolveDockerBin(); const extra = typeof opts.env === "object" && opts.env ? { env: opts.env } : {}; - return spawnCapture({ cmd, args: opts.args, cwd: opts.cwd, timeoutMs: opts.timeoutMs, ...extra }); + const containerName = `codemm-${crypto.randomUUID()}`; + const args = + opts.args[0] === "run" ? ["run", "--name", containerName, ...opts.args.slice(1)] : [...opts.args]; + return spawnCapture({ cmd, args, cwd: opts.cwd, timeoutMs: opts.timeoutMs, containerName, ...extra }); } diff --git a/apps/backend/src/judge/outcome.ts b/apps/backend/src/judge/outcome.ts new file mode 100644 index 0000000..ddedabf --- /dev/null +++ b/apps/backend/src/judge/outcome.ts @@ -0,0 +1,98 @@ +import type { JudgeFailureCategoryDto } from "@codemm/shared-contracts"; +import type { SpawnCaptureResult } from "./docker"; +import type { JudgeResult } from "../types"; + +export const COMPILE_TIMEOUT_MARKER = "__CODEMM_COMPILE_TIMEOUT__"; +export const EXEC_TIMEOUT_MARKER = "__CODEMM_EXEC_TIMEOUT__"; + +function stripMarkers(text: string): string { + return text + .split(/\r?\n/) + .filter((line) => line.trim() !== COMPILE_TIMEOUT_MARKER && line.trim() !== EXEC_TIMEOUT_MARKER) + .join("\n") + .trim(); +} + +function inferFailureCategory(args: { + timedOut: boolean; + outputLimitExceeded: boolean; + stdout: string; + stderr: string; + timeoutStage?: "compile" | "execute" | "overall"; + passedTests: string[]; + failedTests: string[]; +}): JudgeFailureCategoryDto | undefined { + if (args.outputLimitExceeded) return "OUTPUT_LIMIT_EXCEEDED"; + if (args.timedOut || args.timeoutStage) return "TIME_BUDGET_EXCEEDED"; + + const combined = `${args.stdout}\n${args.stderr}`.toLowerCase(); + if (/cannot connect to the docker daemon|docker[^a-z]+not found|permission denied|no such image|pull access denied/.test(combined)) { + return "JUDGE_INFRA_FAILURE"; + } + + const hasStructuredTestFailure = + args.failedTests.length > 0 || + args.passedTests.length > 0 || + /assertionfailederror|failures\s*\(\d+\):|\[x\]|===+\s*failures?\s*===+|failed\s+.*::test_/i.test(combined); + if (hasStructuredTestFailure) return "TEST_FAILURE"; + + return "COMPILE_FAILURE"; +} + +export function buildJudgeResult(args: { + success: boolean; + passedTests: string[]; + failedTests: string[]; + executionTimeMs: number; + capture: SpawnCaptureResult; + budgetProfile?: Record; +}): JudgeResult { + const timeoutStage = + args.capture.stderr.includes(COMPILE_TIMEOUT_MARKER) + ? "compile" + : args.capture.stderr.includes(EXEC_TIMEOUT_MARKER) + ? "execute" + : args.capture.timedOut + ? "overall" + : undefined; + const watchdogSource = + timeoutStage === "overall" ? "outer" : timeoutStage === "compile" || timeoutStage === "execute" ? "inner" : undefined; + + const stdout = stripMarkers(args.capture.stdout); + const stderr = stripMarkers(args.capture.stderr); + const failureCategory = args.success + ? undefined + : inferFailureCategory({ + timedOut: args.capture.timedOut, + outputLimitExceeded: args.capture.outputLimitExceeded, + stdout, + stderr, + ...(timeoutStage ? { timeoutStage } : {}), + passedTests: args.passedTests, + failedTests: args.failedTests, + }); + + return { + success: args.success, + passedTests: args.passedTests, + failedTests: args.failedTests, + stdout, + stderr, + executionTimeMs: args.executionTimeMs, + ...(typeof args.capture.exitCode === "number" ? { exitCode: args.capture.exitCode } : {}), + ...(args.capture.timedOut ? { timedOut: true } : {}), + ...(failureCategory ? { failureCategory } : {}), + ...(timeoutStage ? { timeoutStage } : {}), + ...(watchdogSource ? { watchdogSource } : {}), + ...(args.capture.outputLimitExceeded ? { outputLimitExceeded: true } : {}), + ...(!args.success + ? { + parsedFailures: { + passedTests: args.passedTests, + failedTests: args.failedTests, + }, + } + : {}), + ...(args.budgetProfile ? { budgetProfile: args.budgetProfile } : {}), + }; +} diff --git a/apps/backend/src/judge/tmp.ts b/apps/backend/src/judge/tmp.ts index a092ddf..c4c1fe7 100644 --- a/apps/backend/src/judge/tmp.ts +++ b/apps/backend/src/judge/tmp.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync } from "fs"; +import { chmodSync, mkdirSync, mkdtempSync } from "fs"; import { tmpdir } from "os"; import { join, resolve } from "path"; @@ -13,6 +13,9 @@ export function mkCodemTmpDir(prefix: string): string { const override = process.env.CODEMM_JUDGE_TMPDIR?.trim(); const root = override ? resolve(override) : tmpdir(); if (override) mkdirSync(root, { recursive: true }); - return mkdtempSync(join(root, prefix)); + const dir = mkdtempSync(join(root, prefix)); + // Judge containers now run as an unprivileged user, so the mounted temp + // directory must be traversable/readable from inside the container. + chmodSync(dir, 0o755); + return dir; } - diff --git a/apps/backend/src/languages/cpp/judge.ts b/apps/backend/src/languages/cpp/judge.ts index 578fdcb..15e2adc 100644 --- a/apps/backend/src/languages/cpp/judge.ts +++ b/apps/backend/src/languages/cpp/judge.ts @@ -2,9 +2,10 @@ import { rmSync, writeFileSync } from "fs"; import { join } from "path"; import type { JudgeResult } from "../../types"; import { trace } from "../../utils/trace"; -import { getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; +import { getJudgeCompileTimeoutMs, getJudgeExecutionTimeoutMs, getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; import { writeUserFiles } from "../../judge/files"; import { mkCodemTmpDir } from "../../judge/tmp"; +import { buildJudgeResult, COMPILE_TIMEOUT_MARKER, EXEC_TIMEOUT_MARKER } from "../../judge/outcome"; function parseCppRunner(stdout: string): { passed: string[]; failed: string[] } { const clean = stripAnsi(stdout); @@ -24,6 +25,10 @@ function parseCppRunner(stdout: string): { passed: string[]; failed: string[] } export type CppFiles = Record; +function secondsFromMs(ms: number): number { + return Math.max(1, Math.ceil(ms / 1000)); +} + export async function runCppJudge(userCode: string, testSuite: string): Promise { return runCppJudgeFiles({ "solution.cpp": userCode }, testSuite); } @@ -31,6 +36,11 @@ export async function runCppJudge(userCode: string, testSuite: string): Promise< export async function runCppJudgeFiles(userFiles: CppFiles, testSuite: string): Promise { const start = Date.now(); const tmp = mkCodemTmpDir("codem-cpp-judge-"); + const budgetProfile = { + overallTimeoutMs: getJudgeTimeoutMs(), + compileTimeoutMs: getJudgeCompileTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; try { writeUserFiles(tmp, userFiles); @@ -51,15 +61,28 @@ export async function runCppJudgeFiles(userFiles: CppFiles, testSuite: string): writeFileSync(join(tmp, testFilename), testSuite, "utf8"); const compileCmd = - "g++ -std=c++20 -O2 -pipe -Wall -Wextra -Wno-unused-parameter -o /tmp/test /workspace/test.cpp"; - const runCmd = "/tmp/test"; + `timeout -k 1s ${secondsFromMs(getJudgeCompileTimeoutMs())}s sh -lc 'g++ -std=c++20 -O2 -pipe -Wall -Wextra -Wno-unused-parameter -o /tmp/test /workspace/test.cpp' || { status=$?; if [ "$status" -eq 124 ]; then echo '${COMPILE_TIMEOUT_MARKER}' >&2; exit 124; fi; exit "$status"; }`; + const runCmd = + `timeout -k 1s ${secondsFromMs(getJudgeExecutionTimeoutMs())}s sh -lc '/tmp/test' || { status=$?; if [ "$status" -eq 124 ]; then echo '${EXEC_TIMEOUT_MARKER}' >&2; exit 124; fi; exit "$status"; }`; const args = [ "run", "--rm", "--network", "none", - "--read-only", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", "/tmp:rw,exec", "-v", @@ -73,21 +96,25 @@ export async function runCppJudgeFiles(userFiles: CppFiles, testSuite: string): `${compileCmd} && ${runCmd}`, ]; - const { stdout, stderr, exitCode, timedOut } = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); - trace("judge.result", { exitCode, timedOut, stdoutLen: stdout.length, stderrLen: stderr.length }); + const capture = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); + trace("judge.result", { + exitCode: capture.exitCode, + timedOut: capture.timedOut, + outputLimitExceeded: capture.outputLimitExceeded, + stdoutLen: capture.stdout.length, + stderrLen: capture.stderr.length, + }); const executionTimeMs = Date.now() - start; - const { passed, failed } = parseCppRunner(stdout); - return { - success: exitCode === 0, + const { passed, failed } = parseCppRunner(capture.stdout); + return buildJudgeResult({ + success: capture.exitCode === 0 && !capture.timedOut && !capture.outputLimitExceeded, passedTests: passed, failedTests: failed, - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } catch (e: any) { const executionTimeMs = Date.now() - start; return { diff --git a/apps/backend/src/languages/cpp/run.ts b/apps/backend/src/languages/cpp/run.ts index 03bbc60..5e101b7 100644 --- a/apps/backend/src/languages/cpp/run.ts +++ b/apps/backend/src/languages/cpp/run.ts @@ -44,11 +44,23 @@ export async function runCppFiles(opts: { files: CppFiles; stdin?: string }): Pr const args = [ "run", "--rm", - "--network", - "none", - "--read-only", - "--tmpfs", - "/tmp:rw,exec", + "--network", + "none", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", + "--tmpfs", + "/tmp:rw,exec", "-v", `${tmp}:/workspace:ro`, "--workdir", diff --git a/apps/backend/src/languages/java/judge.ts b/apps/backend/src/languages/java/judge.ts index abe2330..f7bc62c 100644 --- a/apps/backend/src/languages/java/judge.ts +++ b/apps/backend/src/languages/java/judge.ts @@ -2,10 +2,11 @@ import { rmSync, writeFileSync } from "fs"; import { join } from "path"; import type { JudgeResult } from "../../types"; import { trace } from "../../utils/trace"; -import { getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; +import { getJudgeCompileTimeoutMs, getJudgeExecutionTimeoutMs, getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; import { writeUserFiles } from "../../judge/files"; import { mkCodemTmpDir } from "../../judge/tmp"; import { inferClassName } from "../../utils/javaCodegen"; +import { buildJudgeResult, COMPILE_TIMEOUT_MARKER, EXEC_TIMEOUT_MARKER } from "../../judge/outcome"; function parseJUnitTree(stdout: string): { passed: string[]; failed: string[] } { const clean = stripAnsi(stdout); @@ -32,9 +33,18 @@ function parseJUnitTree(stdout: string): { passed: string[]; failed: string[] } export type JavaFiles = Record; +function secondsFromMs(ms: number): number { + return Math.max(1, Math.ceil(ms / 1000)); +} + export async function runJavaJudge(userCode: string, testSuite: string): Promise { const start = Date.now(); const tmp = mkCodemTmpDir("codem-judge-"); + const budgetProfile = { + overallTimeoutMs: getJudgeTimeoutMs(), + compileTimeoutMs: getJudgeCompileTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; try { const userClassName = inferClassName(userCode, "Solution"); @@ -48,31 +58,53 @@ export async function runJavaJudge(userCode: string, testSuite: string): Promise "--rm", "--network", "none", - "--read-only", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", - "/tmp:rw", + "/tmp:rw,exec,size=256m", "-v", - `${tmp}:/workspace`, - "--workdir", - "/workspace", + `${tmp}:/workspace:ro`, + "--entrypoint", + "/bin/bash", "codem-java-judge", + "-lc", + [ + "mkdir -p /tmp/classes", + `timeout -k 1s ${secondsFromMs(getJudgeCompileTimeoutMs())}s sh -lc 'javac -cp \"$JUNIT_JAR:/workspace\" -d /tmp/classes /workspace/*.java' || { status=$?; if [ \"$status\" -eq 124 ]; then echo '${COMPILE_TIMEOUT_MARKER}' >&2; exit 124; fi; exit \"$status\"; }`, + `timeout -k 1s ${secondsFromMs(getJudgeExecutionTimeoutMs())}s sh -lc 'java -jar \"$JUNIT_JAR\" --class-path /tmp/classes --scan-classpath --details-theme ascii' || { status=$?; if [ \"$status\" -eq 124 ]; then echo '${EXEC_TIMEOUT_MARKER}' >&2; exit 124; fi; exit \"$status\"; }`, + ].join(" && "), ]; - const { stdout, stderr, exitCode, timedOut } = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); - trace("judge.result", { exitCode, timedOut, stdoutLen: stdout.length, stderrLen: stderr.length }); + const capture = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); + trace("judge.result", { + exitCode: capture.exitCode, + timedOut: capture.timedOut, + outputLimitExceeded: capture.outputLimitExceeded, + stdoutLen: capture.stdout.length, + stderrLen: capture.stderr.length, + }); const executionTimeMs = Date.now() - start; - const { passed, failed } = parseJUnitTree(stdout); - return { - success: exitCode === 0, + const { passed, failed } = parseJUnitTree(capture.stdout); + return buildJudgeResult({ + success: capture.exitCode === 0 && !capture.timedOut && !capture.outputLimitExceeded, passedTests: passed, failedTests: failed, - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } catch (e: any) { const executionTimeMs = Date.now() - start; return { @@ -91,6 +123,11 @@ export async function runJavaJudge(userCode: string, testSuite: string): Promise export async function runJavaJudgeFiles(userFiles: JavaFiles, testSuite: string): Promise { const start = Date.now(); const tmp = mkCodemTmpDir("codem-judge-"); + const budgetProfile = { + overallTimeoutMs: getJudgeTimeoutMs(), + compileTimeoutMs: getJudgeCompileTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; try { writeUserFiles(tmp, userFiles); @@ -116,31 +153,53 @@ export async function runJavaJudgeFiles(userFiles: JavaFiles, testSuite: string) "--rm", "--network", "none", - "--read-only", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", - "/tmp:rw", + "/tmp:rw,exec,size=256m", "-v", - `${tmp}:/workspace`, - "--workdir", - "/workspace", + `${tmp}:/workspace:ro`, + "--entrypoint", + "/bin/bash", "codem-java-judge", + "-lc", + [ + "mkdir -p /tmp/classes", + `timeout -k 1s ${secondsFromMs(getJudgeCompileTimeoutMs())}s sh -lc 'javac -cp \"$JUNIT_JAR:/workspace\" -d /tmp/classes /workspace/*.java' || { status=$?; if [ \"$status\" -eq 124 ]; then echo '${COMPILE_TIMEOUT_MARKER}' >&2; exit 124; fi; exit \"$status\"; }`, + `timeout -k 1s ${secondsFromMs(getJudgeExecutionTimeoutMs())}s sh -lc 'java -jar \"$JUNIT_JAR\" --class-path /tmp/classes --scan-classpath --details-theme ascii' || { status=$?; if [ \"$status\" -eq 124 ]; then echo '${EXEC_TIMEOUT_MARKER}' >&2; exit 124; fi; exit \"$status\"; }`, + ].join(" && "), ]; - const { stdout, stderr, exitCode, timedOut } = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); - trace("judge.result", { exitCode, timedOut, stdoutLen: stdout.length, stderrLen: stderr.length }); + const capture = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); + trace("judge.result", { + exitCode: capture.exitCode, + timedOut: capture.timedOut, + outputLimitExceeded: capture.outputLimitExceeded, + stdoutLen: capture.stdout.length, + stderrLen: capture.stderr.length, + }); const executionTimeMs = Date.now() - start; - const { passed, failed } = parseJUnitTree(stdout); - return { - success: exitCode === 0, + const { passed, failed } = parseJUnitTree(capture.stdout); + return buildJudgeResult({ + success: capture.exitCode === 0 && !capture.timedOut && !capture.outputLimitExceeded, passedTests: passed, failedTests: failed, - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } catch (e: any) { const executionTimeMs = Date.now() - start; return { diff --git a/apps/backend/src/languages/java/prompts.ts b/apps/backend/src/languages/java/prompts.ts index 28c2521..05791cb 100644 --- a/apps/backend/src/languages/java/prompts.ts +++ b/apps/backend/src/languages/java/prompts.ts @@ -15,7 +15,7 @@ Hard requirements: - Java 17, no package declarations anywhere. - Return JSON for a SINGLE problem (not an array). - You MUST follow the exact output shape requested in the user prompt: - - EITHER the legacy single-file shape (starter_code + reference_solution) + - EITHER the single-file shape (starter_code + reference_solution) - OR the workspace shape (workspace + reference_workspace). Product checking mode (stdout-only): @@ -37,7 +37,7 @@ Test suite requirements: - Use assertEquals/assertTrue/assertFalse/assertThrows with meaningful expectations - Avoid brittle whitespace expectations (do not assertEquals against string literals with leading/trailing spaces). -Reference solution requirements (legacy): +Reference solution requirements (single-file): - reference_solution must compile and pass all tests - starter_code and reference_solution must each declare at most ONE top-level public type (helper types should be non-public). - JSON formatting: represent newlines as "\n" (single backslash). Do NOT use "\\n" (double backslash). diff --git a/apps/backend/src/languages/java/run.ts b/apps/backend/src/languages/java/run.ts index 452469c..94a69b1 100644 --- a/apps/backend/src/languages/java/run.ts +++ b/apps/backend/src/languages/java/run.ts @@ -71,26 +71,36 @@ export async function runJavaFiles(opts: { writeFileSync(join(tmp, "stdin.txt"), opts.stdin ?? "", "utf8"); } - const runCmd = hasStdin ? `java ${mainClass} < stdin.txt` : `java ${mainClass}`; + const runCmd = hasStdin ? `java -cp /tmp/classes ${mainClass} < /workspace/stdin.txt` : `java -cp /tmp/classes ${mainClass}`; // Reuse the existing judge image, but override ENTRYPOINT so it doesn't run JUnit. const args = [ "run", "--rm", - "--network", - "none", - "--read-only", + "--network", + "none", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", - "/tmp:rw", + "/tmp:rw,exec,size=256m", "-v", - `${tmp}:/workspace`, - "--workdir", - "/workspace", + `${tmp}:/workspace:ro`, "--entrypoint", "/bin/bash", "codem-java-judge", "-lc", - `javac *.java && ${runCmd}`, + `mkdir -p /tmp/classes && javac -d /tmp/classes /workspace/*.java && ${runCmd}`, ]; const { stdout, stderr } = await runDocker({ args, cwd: tmp, timeoutMs: getRunTimeoutMs() }); diff --git a/apps/backend/src/languages/java/structuralTopics.ts b/apps/backend/src/languages/java/structuralTopics.ts index a965b5d..d0ee024 100644 --- a/apps/backend/src/languages/java/structuralTopics.ts +++ b/apps/backend/src/languages/java/structuralTopics.ts @@ -1,5 +1,13 @@ type StructuralTopic = "polymorphism" | "inheritance" | "abstraction" | "encapsulation" | "composition"; +export const JAVA_STRUCTURAL_TOPICS: readonly StructuralTopic[] = [ + "polymorphism", + "inheritance", + "abstraction", + "encapsulation", + "composition", +]; + function normalizeTopic(raw: string): string { return String(raw ?? "") .trim() @@ -12,6 +20,10 @@ function hasStructuralTopic(topics: string[], topic: StructuralTopic): boolean { return topics.some((t) => normalizeTopic(t).includes(key)); } +export function hasJavaStructuralTopics(topics: string[]): boolean { + return JAVA_STRUCTURAL_TOPICS.some((topic) => hasStructuralTopic(topics, topic)); +} + type JavaTypeIndex = { publicClassName: string | null; interfaces: string[]; diff --git a/apps/backend/src/languages/python/judge.ts b/apps/backend/src/languages/python/judge.ts index 6f76198..8fc7783 100644 --- a/apps/backend/src/languages/python/judge.ts +++ b/apps/backend/src/languages/python/judge.ts @@ -2,9 +2,10 @@ import { rmSync, writeFileSync } from "fs"; import { join } from "path"; import type { JudgeResult } from "../../types"; import { trace } from "../../utils/trace"; -import { getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; +import { getJudgeExecutionTimeoutMs, getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; import { writeUserFiles } from "../../judge/files"; import { mkCodemTmpDir } from "../../judge/tmp"; +import { buildJudgeResult, EXEC_TIMEOUT_MARKER } from "../../judge/outcome"; function parsePytestFailures(output: string): { failed: string[]; errored: string[] } { const failed = new Set(); @@ -31,6 +32,10 @@ function inferPytestTestNames(testSuite: string): string[] { export type PythonFiles = Record; +function secondsFromMs(ms: number): number { + return Math.max(1, Math.ceil(ms / 1000)); +} + export async function runPythonJudge(userCode: string, testSuite: string): Promise { return runPythonJudgeFiles({ "solution.py": userCode }, testSuite); } @@ -38,6 +43,10 @@ export async function runPythonJudge(userCode: string, testSuite: string): Promi export async function runPythonJudgeFiles(userFiles: PythonFiles, testSuite: string): Promise { const start = Date.now(); const tmp = mkCodemTmpDir("codem-py-judge-"); + const budgetProfile = { + overallTimeoutMs: getJudgeTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; try { writeUserFiles(tmp, userFiles); @@ -62,7 +71,19 @@ export async function runPythonJudgeFiles(userFiles: PythonFiles, testSuite: str "--rm", "--network", "none", - "--read-only", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", "/tmp:rw", "-e", @@ -77,41 +98,47 @@ export async function runPythonJudgeFiles(userFiles: PythonFiles, testSuite: str `${tmp}:/workspace:ro`, "--workdir", "/workspace", + "--entrypoint", + "/bin/bash", "codem-python-judge", + "-lc", + `timeout -k 1s ${secondsFromMs(getJudgeExecutionTimeoutMs())}s sh -lc 'pytest -q -p no:cacheprovider' || { status=$?; if [ "$status" -eq 124 ]; then echo '${EXEC_TIMEOUT_MARKER}' >&2; exit 124; fi; exit "$status"; }`, ]; - const { stdout, stderr, exitCode, timedOut } = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); - trace("judge.result", { exitCode, timedOut, stdoutLen: stdout.length, stderrLen: stderr.length }); + const capture = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); + trace("judge.result", { + exitCode: capture.exitCode, + timedOut: capture.timedOut, + outputLimitExceeded: capture.outputLimitExceeded, + stdoutLen: capture.stdout.length, + stderrLen: capture.stderr.length, + }); const executionTimeMs = Date.now() - start; - if (exitCode === 0) { + if (capture.exitCode === 0 && !capture.timedOut && !capture.outputLimitExceeded) { const inferred = inferPytestTestNames(testSuite); - return { + return buildJudgeResult({ success: true, passedTests: inferred, failedTests: [], - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } - const { failed, errored } = parsePytestFailures(stdout + "\n" + stderr); + const { failed, errored } = parsePytestFailures(capture.stdout + "\n" + capture.stderr); const inferred = inferPytestTestNames(testSuite); const failing = Array.from(new Set([...failed, ...errored])); const passedTests = inferred.filter((t) => !failing.includes(t)); - return { + return buildJudgeResult({ success: false, passedTests, failedTests: failing, - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } catch (e: any) { const executionTimeMs = Date.now() - start; return { diff --git a/apps/backend/src/languages/python/run.ts b/apps/backend/src/languages/python/run.ts index b6b2fe6..27cb34f 100644 --- a/apps/backend/src/languages/python/run.ts +++ b/apps/backend/src/languages/python/run.ts @@ -42,11 +42,23 @@ export async function runPythonFiles(opts: { files: PythonFiles; stdin?: string const args = [ "run", "--rm", - "--network", - "none", - "--read-only", - "--tmpfs", - "/tmp:rw", + "--network", + "none", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", + "--tmpfs", + "/tmp:rw", "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", diff --git a/apps/backend/src/languages/sql/judge.ts b/apps/backend/src/languages/sql/judge.ts index c78e11b..4bef61c 100644 --- a/apps/backend/src/languages/sql/judge.ts +++ b/apps/backend/src/languages/sql/judge.ts @@ -2,8 +2,9 @@ import { rmSync, writeFileSync } from "fs"; import { join } from "path"; import type { JudgeResult } from "../../types"; import { trace } from "../../utils/trace"; -import { getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; +import { getJudgeExecutionTimeoutMs, getJudgeTimeoutMs, runDocker, stripAnsi } from "../../judge/docker"; import { mkCodemTmpDir } from "../../judge/tmp"; +import { buildJudgeResult, EXEC_TIMEOUT_MARKER } from "../../judge/outcome"; function parseSqlRunner(stdout: string): { passed: string[]; failed: string[] } { const clean = stripAnsi(stdout); @@ -21,9 +22,17 @@ function parseSqlRunner(stdout: string): { passed: string[]; failed: string[] } return { passed: Array.from(passed), failed: Array.from(failed) }; } +function secondsFromMs(ms: number): number { + return Math.max(1, Math.ceil(ms / 1000)); +} + export async function runSqlJudge(userSql: string, testSuiteJson: string): Promise { const start = Date.now(); const tmp = mkCodemTmpDir("codem-sql-judge-"); + const budgetProfile = { + overallTimeoutMs: getJudgeTimeoutMs(), + executeTimeoutMs: getJudgeExecutionTimeoutMs(), + }; try { writeFileSync(join(tmp, "solution.sql"), userSql, "utf8"); @@ -34,31 +43,51 @@ export async function runSqlJudge(userSql: string, testSuiteJson: string): Promi "--rm", "--network", "none", - "--read-only", + "--read-only", + "--user", + "65534:65534", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + "256", + "--memory", + "512m", + "--cpus", + "1.0", "--tmpfs", "/tmp:rw", "-v", `${tmp}:/workspace:ro`, "--workdir", "/workspace", + "--entrypoint", + "/bin/bash", "codem-sql-judge", + "-lc", + `timeout -k 1s ${secondsFromMs(getJudgeExecutionTimeoutMs())}s sh -lc 'python /opt/codem/sql_judge.py' || { status=$?; if [ "$status" -eq 124 ]; then echo '${EXEC_TIMEOUT_MARKER}' >&2; exit 124; fi; exit "$status"; }`, ]; - const { stdout, stderr, exitCode, timedOut } = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); - trace("judge.result", { exitCode, timedOut, stdoutLen: stdout.length, stderrLen: stderr.length }); + const capture = await runDocker({ args, cwd: tmp, timeoutMs: getJudgeTimeoutMs() }); + trace("judge.result", { + exitCode: capture.exitCode, + timedOut: capture.timedOut, + outputLimitExceeded: capture.outputLimitExceeded, + stdoutLen: capture.stdout.length, + stderrLen: capture.stderr.length, + }); const executionTimeMs = Date.now() - start; - const { passed, failed } = parseSqlRunner(stdout); - return { - success: exitCode === 0, + const { passed, failed } = parseSqlRunner(capture.stdout); + return buildJudgeResult({ + success: capture.exitCode === 0 && !capture.timedOut && !capture.outputLimitExceeded, passedTests: passed, failedTests: failed, - stdout, - stderr, executionTimeMs, - exitCode, - timedOut, - }; + capture, + budgetProfile, + }); } catch (e: any) { const executionTimeMs = Date.now() - start; return { diff --git a/apps/backend/src/pipeline/slotStages.ts b/apps/backend/src/pipeline/slotStages.ts index 151d320..0feec6e 100644 --- a/apps/backend/src/pipeline/slotStages.ts +++ b/apps/backend/src/pipeline/slotStages.ts @@ -16,6 +16,7 @@ import { getResolvedSnapshotOrNull, getRouteForRole, summarizeRoutePlan } from " import type { ProblemSlot } from "../planner/types"; import type { SlotPromptContext } from "../languages/types"; import { buildDefaultClassSkeleton, inferClassName } from "../utils/javaCodegen"; +import { javaUsesStdin } from "../utils/javaSource"; import { ReferenceSolutionValidationError, validateReferenceSolution, @@ -27,6 +28,12 @@ import { generateReference, REFERENCE_PROMPT_TEMPLATE_ID, REPAIR_PROMPT_TEMPLATE import { generateSkeleton, SKELETON_PROMPT_TEMPLATE_ID } from "./stages/skeleton"; import { generateTests, TESTS_PROMPT_TEMPLATE_ID } from "./stages/tests"; import type { GenerationFailureKind } from "../generation/errors"; +import { assertJavaStructuralTopicRequirements, hasJavaStructuralTopics } from "../languages/java/structuralTopics"; +import { + persistExecutionAttempt, + persistFailureDiagnosis, + prepareValidatedExecutionBundle, +} from "../generation/services/validationService"; export class SlotPipelineTerminalError extends Error { stage: Exclude; @@ -93,6 +100,18 @@ function buildSlotIntent(slot: ProblemSlot): SlotIntent { } function inferFailureKind(err: unknown): GenerationFailureKind { + const explicitKind = (err as { kind?: unknown } | null)?.kind; + if ( + explicitKind === "compile" || + explicitKind === "tests" || + explicitKind === "timeout" || + explicitKind === "contract" || + explicitKind === "quality" || + explicitKind === "llm" || + explicitKind === "unknown" + ) { + return explicitKind; + } if (err instanceof ReferenceSolutionValidationError) return err.kind; if (err instanceof TestStrengthGateError) return "quality"; if (/timed out|timeout/i.test(String((err as any)?.message ?? ""))) return "timeout"; @@ -100,6 +119,69 @@ function inferFailureKind(err: unknown): GenerationFailureKind { return "llm"; } +function preflightValidateDraft(args: { + slot: ProblemSlot; + draft: GeneratedProblemDraft; + stage: "reference" | "repair"; + llm?: CompletionMeta; + llmOutputHash?: string; +}): void { + if ( + args.draft.language !== "java" || + !("reference_solution" in args.draft) || + typeof args.draft.reference_solution !== "string" + ) { + return; + } + + if (hasJavaStructuralTopics(args.slot.topics) && javaUsesStdin(args.draft.reference_solution)) { + throw new SlotPipelineTerminalError( + "stdin reads (Scanner/System.in) are not allowed for Java structural-topic slots (encapsulation/inheritance/polymorphism/etc). Use pure methods and deterministic unit tests instead.", + { + stage: args.stage, + kind: "contract", + ...(args.llm ? { llm: args.llm } : {}), + ...(args.llmOutputHash ? { llmOutputHash: args.llmOutputHash } : {}), + routeRole: args.stage === "repair" ? "repair" : "reference", + title: args.draft.title, + } + ); + } + + try { + assertJavaStructuralTopicRequirements({ + topics: args.slot.topics, + referenceSource: args.draft.reference_solution, + testSuite: args.draft.test_suite, + }); + } catch (error) { + throw new SlotPipelineTerminalError(error instanceof Error ? error.message : String(error), { + stage: args.stage, + kind: "contract", + ...(args.llm ? { llm: args.llm } : {}), + ...(args.llmOutputHash ? { llmOutputHash: args.llmOutputHash } : {}), + routeRole: args.stage === "repair" ? "repair" : "reference", + title: args.draft.title, + }); + } +} + +function isNoOpReferenceRepair(args: { + previousReferenceSource: string; + previousReferenceHash: string | undefined; + nextReferenceSource: string; + nextReferenceHash: string | undefined; +}): boolean { + if ( + typeof args.previousReferenceHash === "string" && + typeof args.nextReferenceHash === "string" && + args.previousReferenceHash === args.nextReferenceHash + ) { + return true; + } + return args.previousReferenceSource.trim() === args.nextReferenceSource.trim(); +} + function maybeEmitRouteSelection(args: { slot: ProblemSlot; routeRole: LlmRole; @@ -361,6 +443,8 @@ async function validateDraftWithTelemetry(args: { ctx: StageContext; draft: GeneratedProblemDraft; attempt: number; + repairStrategy?: "regenerate_reference_logic" | null; + llmOutputHash?: string | null; }): Promise { const startedAt = new Date().toISOString(); args.ctx.onProgress?.({ @@ -373,8 +457,58 @@ async function validateDraftWithTelemetry(args: { startedAt, }); try { - await validateReferenceSolution(args.draft); - await runTestStrengthGate(args.draft, args.ctx.slot); + const bundle = prepareValidatedExecutionBundle({ + slot: args.ctx.slot, + draft: args.draft, + repairStrategy: args.repairStrategy ?? null, + llmOutputHash: args.llmOutputHash ?? null, + }); + + const execStartedAt = new Date().toISOString(); + const judgeResult = await validateReferenceSolution(bundle.draft); + persistExecutionAttempt({ + slotIndex: args.ctx.slot.index, + attempt: args.attempt, + executionPhase: + judgeResult.timeoutStage === "compile" || judgeResult.failureCategory === "COMPILE_FAILURE" || judgeResult.failureCategory === "COMPILE_ERROR" + ? "compile" + : "test_exec", + bundle, + strategy: args.repairStrategy ?? null, + result: { + startedAt: execStartedAt, + finishedAt: new Date().toISOString(), + exitCode: judgeResult.exitCode ?? null, + timeoutStage: judgeResult.timeoutStage ?? null, + watchdogSource: judgeResult.watchdogSource ?? null, + failureCategory: judgeResult.failureCategory ?? null, + stdout: judgeResult.stdout, + stderr: judgeResult.stderr, + parsedFailures: judgeResult.parsedFailures, + trace: { + passedTests: judgeResult.passedTests, + failedTests: judgeResult.failedTests, + executionTimeMs: judgeResult.executionTimeMs, + }, + }, + }); + + const qualityStartedAt = new Date().toISOString(); + await runTestStrengthGate(bundle.draft, args.ctx.slot); + persistExecutionAttempt({ + slotIndex: args.ctx.slot.index, + attempt: args.attempt, + executionPhase: "quality_gate", + bundle, + strategy: args.repairStrategy ?? null, + result: { + startedAt: qualityStartedAt, + finishedAt: new Date().toISOString(), + trace: { + qualityGate: "passed", + }, + }, + }); const endedAt = new Date().toISOString(); args.ctx.onProgress?.({ type: "slot_stage_finished", @@ -391,6 +525,88 @@ async function validateDraftWithTelemetry(args: { } catch (error) { const endedAt = new Date().toISOString(); const kind = inferFailureKind(error); + const bundle = + (() => { + try { + return prepareValidatedExecutionBundle({ + slot: args.ctx.slot, + draft: args.draft, + repairStrategy: args.repairStrategy ?? null, + llmOutputHash: args.llmOutputHash ?? null, + }); + } catch { + return null; + } + })(); + + if (bundle && error instanceof ReferenceSolutionValidationError) { + const executionAttemptId = persistExecutionAttempt({ + slotIndex: args.ctx.slot.index, + attempt: args.attempt, + executionPhase: + error.timeoutStage === "compile" || error.failureCategory === "COMPILE_FAILURE" || error.failureCategory === "COMPILE_ERROR" + ? "compile" + : "test_exec", + bundle, + strategy: args.repairStrategy ?? null, + result: { + startedAt, + finishedAt: endedAt, + exitCode: error.exitCode ?? null, + timeoutStage: error.timeoutStage ?? null, + watchdogSource: error.watchdogSource ?? null, + failureCategory: error.failureCategory ?? null, + stdout: error.judgeStdout, + stderr: error.judgeStderr, + parsedFailures: error.parsedFailures, + trace: { + budgetProfile: error.budgetProfile ?? null, + }, + }, + }); + persistFailureDiagnosis({ + slot: args.ctx.slot, + attempt: args.attempt, + kind, + err: error, + sourceExecutionAttemptId: executionAttemptId, + }); + } else if (bundle && error instanceof TestStrengthGateError) { + const executionAttemptId = persistExecutionAttempt({ + slotIndex: args.ctx.slot.index, + attempt: args.attempt, + executionPhase: "quality_gate", + bundle, + strategy: args.repairStrategy ?? null, + result: { + startedAt, + finishedAt: endedAt, + failureCategory: "TEST_FAILURE", + parsedFailures: { + baselineId: error.baselineId, + }, + trace: { + qualityGate: "failed", + baselineId: error.baselineId, + }, + }, + }); + persistFailureDiagnosis({ + slot: args.ctx.slot, + attempt: args.attempt, + kind, + err: error, + sourceExecutionAttemptId: executionAttemptId, + }); + } else { + persistFailureDiagnosis({ + slot: args.ctx.slot, + attempt: args.attempt, + kind, + err: error, + }); + } + args.ctx.onProgress?.({ type: "slot_stage_finished", slotIndex: args.ctx.slot.index, @@ -462,12 +678,6 @@ export async function runSlotPipeline(args: { }); envelope.tests = tests.value; - maybeEmitRouteSelection({ - slot: args.slot, - routeRole: "reference", - promptTemplateId: REFERENCE_PROMPT_TEMPLATE_ID, - ...(args.onProgress ? { onProgress: args.onProgress } : {}), - }); let reference = await executeStage({ stage: "reference", routeRole: "reference", @@ -504,9 +714,21 @@ export async function runSlotPipeline(args: { ); } draft = parsedDraft.data; + preflightValidateDraft({ + slot: args.slot, + draft, + stage: "reference", + ...(reference.llm ? { llm: reference.llm } : {}), + ...(reference.llmOutputHash ? { llmOutputHash: reference.llmOutputHash } : {}), + }); try { - await validateDraftWithTelemetry({ ctx, draft, attempt: 1 }); + await validateDraftWithTelemetry({ + ctx, + draft, + attempt: 1, + llmOutputHash: reference.llmOutputHash ?? null, + }); } catch (error) { if (!(error instanceof ReferenceSolutionValidationError)) { throw new SlotPipelineTerminalError(error instanceof Error ? error.message : String(error), { @@ -519,12 +741,8 @@ export async function runSlotPipeline(args: { }); } - maybeEmitRouteSelection({ - slot: args.slot, - routeRole: "repair", - promptTemplateId: REPAIR_PROMPT_TEMPLATE_ID, - ...(args.onProgress ? { onProgress: args.onProgress } : {}), - }); + const previousReferenceSource = (envelope.reference as SlotReference).reference_solution; + const previousReferenceHash = reference.artifactHash; const repair = await executeStage({ stage: "repair", routeRole: "repair", @@ -545,6 +763,26 @@ export async function runSlotPipeline(args: { }); envelope.reference = repair.value; reference = repair; + if ( + isNoOpReferenceRepair({ + previousReferenceSource, + previousReferenceHash, + nextReferenceSource: repair.value.reference_solution, + nextReferenceHash: repair.artifactHash, + }) + ) { + throw new SlotPipelineTerminalError( + "Repair regenerated the same reference artifact after a validation failure.", + { + stage: "repair", + kind: "repair_no_progress", + ...(repair.llm ? { llm: repair.llm } : {}), + ...(repair.llmOutputHash ? { llmOutputHash: repair.llmOutputHash } : {}), + routeRole: "repair", + title: draft.title, + } + ); + } draft = buildDraft({ slot: args.slot, skeleton: envelope.skeleton as SlotSkeleton, @@ -566,8 +804,21 @@ export async function runSlotPipeline(args: { ); } draft = repairedDraft.data; + preflightValidateDraft({ + slot: args.slot, + draft, + stage: "repair", + ...(repair.llm ? { llm: repair.llm } : {}), + ...(repair.llmOutputHash ? { llmOutputHash: repair.llmOutputHash } : {}), + }); try { - await validateDraftWithTelemetry({ ctx, draft, attempt: 2 }); + await validateDraftWithTelemetry({ + ctx, + draft, + attempt: 2, + repairStrategy: "regenerate_reference_logic", + llmOutputHash: repair.llmOutputHash ?? null, + }); } catch (repairError) { throw new SlotPipelineTerminalError(repairError instanceof Error ? repairError.message : String(repairError), { stage: "repair", @@ -606,3 +857,12 @@ export async function runSlotPipeline(args: { }, }; } + +export const __test__ = { + inferFailureKind, + preflightValidateDraft, + isNoOpReferenceRepair, + stripCppComments, + extractCppSolveSignature, + deriveCppStarter, +}; diff --git a/apps/backend/src/services/sessionService.ts b/apps/backend/src/services/sessionService.ts index 053aadd..332a599 100644 --- a/apps/backend/src/services/sessionService.ts +++ b/apps/backend/src/services/sessionService.ts @@ -6,6 +6,7 @@ export { export { generateFromThread, generateFromSession, + repairFailedSlotsFromThread, regenerateSlotFromThread, regenerateSlotFromSession, type GenerateFromSessionResponse, diff --git a/apps/backend/src/services/threads/generationRecoveryService.ts b/apps/backend/src/services/threads/generationRecoveryService.ts new file mode 100644 index 0000000..158aef4 --- /dev/null +++ b/apps/backend/src/services/threads/generationRecoveryService.ts @@ -0,0 +1,85 @@ +import type { GenerationRunStatus, ThreadState } from "@codemm/shared-contracts"; +import { generationRunRepository, generationSlotRunRepository } from "../../database/repositories/generationRunRepository"; +import { threadRepository } from "../../database/repositories/threadRepository"; +import { mapRunStatusToThreadState } from "./generationState"; + +function deriveRecoveredRunStatus(args: { + successfulSlots: number; + failedSlots: number; + completedSlots: number; + totalSlots: number; +}): GenerationRunStatus { + if (args.successfulSlots > 0 && args.failedSlots > 0) return "INCOMPLETE"; + if (args.successfulSlots > 0 && args.completedSlots >= args.totalSlots) return "COMPLETED"; + return "RETRYABLE_FAILURE"; +} + +export function reconcileInterruptedGenerationState(): { + reconciledRunIds: string[]; + updatedThreadIds: string[]; +} { + const reconciledRunIds: string[] = []; + const updatedThreadIds = new Set(); + + for (const run of generationRunRepository.listStaleActiveRuns()) { + generationSlotRunRepository.reconcileIncomplete(run.id); + const slots = generationSlotRunRepository.listByRun(run.id); + const successfulSlots = slots.filter((slot) => slot.status === "SUCCEEDED").length; + const failedSlots = slots.filter( + (slot) => + slot.status === "RETRYABLE_FAILURE" || + slot.status === "RECOVERABLE_FAILED" || + slot.status === "QUARANTINED" || + slot.status === "HARD_FAILURE" || + slot.status === "FATAL_FAILED" + ).length; + const completedSlots = slots.filter( + (slot) => + slot.status === "SUCCEEDED" || + slot.status === "RETRYABLE_FAILURE" || + slot.status === "RECOVERABLE_FAILED" || + slot.status === "QUARANTINED" || + slot.status === "HARD_FAILURE" || + slot.status === "FATAL_FAILED" || + slot.status === "SKIPPED" + ).length; + const recoveredStatus = deriveRecoveredRunStatus({ + successfulSlots, + failedSlots, + completedSlots, + totalSlots: run.total_slots, + }); + + generationRunRepository.finish({ + id: run.id, + status: recoveredStatus, + activityId: run.activity_id ?? null, + completedSlots, + successfulSlots, + failedSlots, + lastFailureCode: run.last_failure_code ?? "ENGINE_RESTART", + lastFailureKind: (run.last_failure_kind as any) ?? "infra", + lastFailureMessage: run.last_failure_message ?? "Generation was interrupted before completion.", + }); + threadRepository.updateState(run.thread_id, mapRunStatusToThreadState(recoveredStatus)); + threadRepository.setLastError( + run.thread_id, + recoveredStatus === "COMPLETED" ? null : run.last_failure_message ?? "Generation was interrupted before completion." + ); + reconciledRunIds.push(run.id); + updatedThreadIds.add(run.thread_id); + } + + const stuckThreads = threadRepository.listByStates(["GENERATE_PENDING", "GENERATING"]); + for (const thread of stuckThreads) { + const latestRun = generationRunRepository.latestByThread(thread.id); + const nextState: ThreadState = latestRun ? mapRunStatusToThreadState(latestRun.status as GenerationRunStatus) : "READY"; + threadRepository.updateState(thread.id, nextState); + if (nextState === "READY") { + threadRepository.setLastError(thread.id, "Recovered thread state after interrupted generation."); + } + updatedThreadIds.add(thread.id); + } + + return { reconciledRunIds, updatedThreadIds: [...updatedThreadIds] }; +} diff --git a/apps/backend/src/services/threads/generationState.ts b/apps/backend/src/services/threads/generationState.ts new file mode 100644 index 0000000..fe06fd4 --- /dev/null +++ b/apps/backend/src/services/threads/generationState.ts @@ -0,0 +1,59 @@ +import type { + GenerationFailureKind, + GenerationRunStatus, + GenerationSlotStage, + GenerationSlotTerminalStatus, + ThreadState, +} from "@codemm/shared-contracts"; +import type { GenerationOutcome } from "../../contracts/generationOutcome"; + +export type SlotExecutionFailure = { + kind: GenerationFailureKind; + code: string; + message: string; + stage: GenerationSlotStage; + title?: string; + llmOutputHash?: string; +}; + +export type SlotExecutionResult = + | { + slotIndex: number; + terminalStatus: "SUCCEEDED"; + retries: number; + problem: any; + outcome: GenerationOutcome; + title?: string; + } + | { + slotIndex: number; + terminalStatus: Exclude; + retries: number; + outcome: GenerationOutcome; + failure: SlotExecutionFailure; + title?: string; + }; + +export function mapRunStatusToThreadState(status: GenerationRunStatus): ThreadState { + if (status === "COMPLETED") return "COMPLETED"; + if (status === "INCOMPLETE" || status === "PARTIAL_SUCCESS") return "INCOMPLETE"; + if (status === "HARD_FAILURE") return "HARD_FAILURE"; + if (status === "RETRYABLE_FAILURE" || status === "ABORTED") return "RETRYABLE_FAILURE"; + if (status === "RUNNING") return "GENERATING"; + return "GENERATE_PENDING"; +} + +export function deriveRunStatus(results: SlotExecutionResult[]): GenerationRunStatus { + const succeeded = results.filter((result) => result.terminalStatus === "SUCCEEDED").length; + const hardFailures = results.filter((result) => result.terminalStatus === "HARD_FAILURE").length; + const retryableFailures = results.filter( + (result) => result.terminalStatus === "RETRYABLE_FAILURE" || result.terminalStatus === "RECOVERABLE_FAILED" || result.terminalStatus === "QUARANTINED" + ).length; + const skipped = results.filter((result) => result.terminalStatus === "SKIPPED").length; + + if (results.length > 0 && succeeded === results.length) return "COMPLETED"; + if (succeeded > 0 && succeeded + hardFailures + retryableFailures + skipped === results.length) return "INCOMPLETE"; + if (hardFailures > 0 && succeeded === 0 && retryableFailures === 0) return "HARD_FAILURE"; + if (retryableFailures > 0 || skipped === results.length) return "RETRYABLE_FAILURE"; + return "HARD_FAILURE"; +} diff --git a/apps/backend/src/services/threads/shared.ts b/apps/backend/src/services/threads/shared.ts index e8821d6..0e8e9dc 100644 --- a/apps/backend/src/services/threads/shared.ts +++ b/apps/backend/src/services/threads/shared.ts @@ -14,6 +14,7 @@ import { DEFAULT_LEARNING_MODE, LearningModeSchema, type LearningMode } from ".. import type { GenerationOutcome } from "../../contracts/generationOutcome"; import type { GeneratedProblem } from "../../contracts/problem"; import type { ConfidenceMap } from "../../agent/readiness"; +import type { GenerationFailureKind } from "@codemm/shared-contracts"; import { USER_EDITABLE_SPEC_KEYS, type UserEditableSpecKey, @@ -39,6 +40,8 @@ export type SessionRecord = { commitments: ReturnType; generationOutcomes: GenerationOutcome[]; intentTrace: unknown[]; + latestGenerationRunId?: string | null; + latestGenerationRunStatus?: string | null; }; export type SessionCollectorState = { @@ -90,15 +93,56 @@ export function parseGenerationOutcomes(json: string | null | undefined): Genera if (!item || typeof item !== "object") continue; const slotIndex = (item as any).slotIndex; const success = (item as any).success; + const status = (item as any).status; const retries = (item as any).retries; const appliedFallback = (item as any).appliedFallback; + const failureKind = (item as any).failureKind; + const failureCode = (item as any).failureCode; + const message = (item as any).message; if (typeof slotIndex !== "number" || !Number.isFinite(slotIndex)) continue; if (typeof success !== "boolean") continue; if (typeof retries !== "number" || !Number.isFinite(retries)) continue; + const normalizedFailureKind = + failureKind === "generation_schema_error" || + failureKind === "static_rule_violation" || + failureKind === "api_shape_mismatch" || + failureKind === "complexity_risk_exceeded" || + failureKind === "compile_failure" || + failureKind === "test_failure" || + failureKind === "time_budget_exceeded" || + failureKind === "output_limit_exceeded" || + failureKind === "judge_infra_failure" || + failureKind === "repair_no_progress" || + failureKind === "run_policy_failure" || + failureKind === "compile" || + failureKind === "tests" || + failureKind === "timeout" || + failureKind === "contract" || + failureKind === "quality" || + failureKind === "llm" || + failureKind === "infra" || + failureKind === "unknown" + ? (failureKind as GenerationFailureKind) + : undefined; outcomes.push({ slotIndex, success, + status: + status === "SUCCEEDED" || + status === "RECOVERABLE_FAILED" || + status === "FATAL_FAILED" || + status === "QUARANTINED" || + status === "RETRYABLE_FAILURE" || + status === "HARD_FAILURE" || + status === "SKIPPED" + ? status + : success + ? "SUCCEEDED" + : "RETRYABLE_FAILURE", retries, + ...(normalizedFailureKind ? { failureKind: normalizedFailureKind } : {}), + ...(typeof failureCode === "string" ? { failureCode } : {}), + ...(typeof message === "string" ? { message } : {}), ...(typeof appliedFallback === "string" && appliedFallback.trim() ? { appliedFallback } : {}), }); } diff --git a/apps/backend/src/services/threads/threadConversationService.ts b/apps/backend/src/services/threads/threadConversationService.ts index 69509d1..c3a150e 100644 --- a/apps/backend/src/services/threads/threadConversationService.ts +++ b/apps/backend/src/services/threads/threadConversationService.ts @@ -184,6 +184,20 @@ export async function processSessionMessage( return ops; }; + const sanitizeTopicTags = (base: SpecDraft, candidate: SpecDraft): SpecDraft => { + const raw = Array.isArray((candidate as any).topic_tags) ? ((candidate as any).topic_tags as unknown[]) : []; + const difficultyLike = new Set(["easy", "medium", "hard", "beginner", "intermediate", "advanced"]); + const sanitized = raw + .map((value) => String(value ?? "").trim()) + .filter(Boolean) + .filter((value) => !difficultyLike.has(value.toLowerCase())); + if (sanitized.length === raw.length) return candidate; + if (sanitized.length > 0) return { ...candidate, topic_tags: sanitized }; + return Array.isArray((base as any).topic_tags) && (base as any).topic_tags.length > 0 + ? { ...candidate, topic_tags: (base as any).topic_tags } + : candidate; + }; + const buildNextQuestion = (spec: SpecDraft): { key: string; prompt: string } | null => { const gaps = analyzeSpecGaps(spec); if (gaps.complete) return null; @@ -277,7 +291,8 @@ export async function processSessionMessage( const applyWithDraftValidation = (ops: JsonPatchOp[]) => { const merged = ops.length > 0 ? (applyJsonPatch(specWithFixed as any, ops) as SpecDraft) : (specWithFixed as SpecDraft); const fixedAfter = ensureFixedFields(merged); - const final = fixedAfter.length > 0 ? (applyJsonPatch(merged as any, fixedAfter) as SpecDraft) : merged; + const fixedMerged = fixedAfter.length > 0 ? (applyJsonPatch(merged as any, fixedAfter) as SpecDraft) : merged; + const final = sanitizeTopicTags(specWithFixed as SpecDraft, fixedMerged); return { final }; }; diff --git a/apps/backend/src/services/threads/threadGenerationService.ts b/apps/backend/src/services/threads/threadGenerationService.ts index ce00f0b..b4cbd45 100644 --- a/apps/backend/src/services/threads/threadGenerationService.ts +++ b/apps/backend/src/services/threads/threadGenerationService.ts @@ -10,7 +10,6 @@ import { generateProblemsFromPlan } from "../../generation"; import type { GeneratedProblem } from "../../contracts/problem"; import type { GenerationOutcome } from "../../contracts/generationOutcome"; import type { GenerationProgressEvent } from "../../contracts/generationProgress"; -import { GenerationSlotFailureError } from "../../generation/errors"; import { publishGenerationProgress } from "../../generation/progressBus"; import { applyJsonPatch } from "../../compiler/jsonPatch"; import { proposeGenerationFallbackWithPolicy } from "../../agent/generationFallback"; @@ -18,6 +17,10 @@ import { trace } from "../../utils/trace"; import { withTraceContext } from "../../utils/traceContext"; import { activityRepository } from "../../database/repositories/activityRepository"; import { threadRepository } from "../../database/repositories/threadRepository"; +import { + generationRunRepository, + generationSlotRunRepository, +} from "../../database/repositories/generationRunRepository"; import { appendIntentTrace, mergeConfidence, @@ -31,6 +34,9 @@ import { transitionOrThrow, } from "./shared"; import { parseCommitmentsJson } from "../../agent/commitments"; +import { deriveRunStatus, mapRunStatusToThreadState, type SlotExecutionResult } from "./generationState"; +import type { GenerationFailureKind, GenerationRunStatus, GenerationSlotStage } from "@codemm/shared-contracts"; +import type { SessionState } from "../../contracts/session"; export type GenerateFromSessionResponse = { activityId: string; @@ -39,10 +45,132 @@ export type GenerateFromSessionResponse = { export type GenerateFromThreadResponse = GenerateFromSessionResponse; -export async function generateFromThread(sessionId: string): Promise { - return withTraceContext({ sessionId, threadId: sessionId }, async () => { +type GenerationResumeRequest = { + problems: GeneratedProblem[]; + outcomes: GenerationOutcome[]; + targetSlotIndexes: number[]; + existingActivityId?: string | null; + mode: "repair_failed_slots" | "resume_interrupted"; +}; + +type ResumePlanState = { + problems: GeneratedProblem[]; + outcomes: GenerationOutcome[]; + targetSlotIndexes: number[]; + keptSuccessfulSlotIndexes: number[]; + existingActivityId: string | null; +}; + +function currentIso() { + return new Date().toISOString(); +} + +function translateEventStage(stage: "skeleton" | "tests" | "reference" | "validate" | "repair"): GenerationSlotStage { + if (stage === "skeleton") return "SKELETON_RUNNING"; + if (stage === "tests") return "TESTS_RUNNING"; + if (stage === "reference") return "REFERENCE_RUNNING"; + if (stage === "repair") return "REPAIRING_REFERENCE"; + return "VALIDATING_REFERENCE"; +} + +function isTerminalRunStatus(status: GenerationRunStatus): boolean { + return ( + status === "COMPLETED" || + status === "INCOMPLETE" || + status === "PARTIAL_SUCCESS" || + status === "RETRYABLE_FAILURE" || + status === "HARD_FAILURE" || + status === "ABORTED" + ); +} + +function getLastFailure(results: SlotExecutionResult[]): { + kind?: GenerationFailureKind | null; + code?: string | null; + message?: string | null; +} { + const lastFailure = [...results].reverse().find((result) => result.terminalStatus !== "SUCCEEDED"); + if (!lastFailure || !("failure" in lastFailure)) return {}; + return { + kind: lastFailure.failure.kind, + code: lastFailure.failure.code, + message: lastFailure.failure.message, + }; +} + +function failureKindToFailureCode(kind: GenerationFailureKind | null | undefined, stage: string): string | null { + if (!kind) return null; + if (kind === "timeout" || kind === "time_budget_exceeded") return "TIME_BUDGET_EXCEEDED"; + if (kind === "compile" || kind === "compile_failure") return "COMPILE_FAILURE"; + if (kind === "tests" || kind === "test_failure") return "TEST_FAILURE"; + if (kind === "infra" || kind === "judge_infra_failure") return "JUDGE_INFRA_FAILURE"; + if (kind === "output_limit_exceeded") return "OUTPUT_LIMIT_EXCEEDED"; + if (kind === "generation_schema_error") return "GENERATION_SCHEMA_ERROR"; + if (kind === "static_rule_violation") return "STATIC_RULE_VIOLATION"; + if (kind === "api_shape_mismatch") return "API_SHAPE_MISMATCH"; + if (kind === "complexity_risk_exceeded") return "COMPLEXITY_RISK_EXCEEDED"; + if (kind === "repair_no_progress") return "REPAIR_NO_PROGRESS"; + if (kind === "run_policy_failure") return "RUN_POLICY_FAILURE"; + if (kind === "spec_error") return "SPEC_ERROR"; + if (kind === "contract") return `STAGE_${stage.toUpperCase()}_CONTRACT`; + if (kind === "quality") return "QUALITY_GATE_FAILED"; + if (kind === "llm") return "LLM_GENERATION_FAILED"; + return `STAGE_${stage.toUpperCase()}`; +} + +function buildProblemMapFromResume(problems: GeneratedProblem[], outcomes: GenerationOutcome[]): Map { + const bySlot = new Map(); + let cursor = 0; + for (const outcome of outcomes) { + if (!outcome.success) continue; + const problem = problems[cursor]; + if (problem) { + bySlot.set(outcome.slotIndex, problem); + cursor += 1; + } + } + return bySlot; +} + +function deriveResumePlanState(args: { + plan: ReturnType; + resume?: GenerationResumeRequest; +}): ResumePlanState | null { + if (!args.resume) return null; + const targetSlotIndexSet = new Set( + args.resume.targetSlotIndexes.filter((value) => Number.isInteger(value) && value >= 0 && value < args.plan.length), + ); + const problemBySlot = buildProblemMapFromResume(args.resume.problems, args.resume.outcomes); + const keptSuccessfulSlotIndexes: number[] = []; + for (const slot of args.plan) { + if (targetSlotIndexSet.has(slot.index)) continue; + const outcome = args.resume.outcomes.find((candidate) => candidate.slotIndex === slot.index); + if (outcome?.success && problemBySlot.has(slot.index)) keptSuccessfulSlotIndexes.push(slot.index); + } + return { + problems: args.resume.problems, + outcomes: args.resume.outcomes, + targetSlotIndexes: [...targetSlotIndexSet].sort((a, b) => a - b), + keptSuccessfulSlotIndexes, + existingActivityId: + typeof args.resume.existingActivityId === "string" && args.resume.existingActivityId.trim() + ? args.resume.existingActivityId + : null, + }; +} + +function activityStatusForRunStatus(status: GenerationRunStatus): "DRAFT" | "INCOMPLETE" { + return status === "INCOMPLETE" || status === "PARTIAL_SUCCESS" ? "INCOMPLETE" : "DRAFT"; +} + +export async function generateFromThread( + sessionId: string, + opts?: { runId?: string; resume?: GenerationResumeRequest } +): Promise { + const runId = typeof opts?.runId === "string" && opts.runId.trim() ? opts.runId : crypto.randomUUID(); + return withTraceContext({ sessionId, threadId: sessionId, runId }, async () => { const session = requireSession(sessionId); - const state = session.state; + const state = session.state as SessionState; const learning_mode = parseLearningMode(session.learning_mode); const instructionsMdRaw = typeof session.instructions_md === "string" ? String(session.instructions_md) : ""; const instructionsMd = instructionsMdRaw.trim() ? instructionsMdRaw : null; @@ -71,12 +199,16 @@ export async function generateFromThread(sessionId: string): Promise = []; - let resumeProblems: GeneratedProblem[] = []; - let resumeOutcomes: GenerationOutcome[] = []; let problems: GeneratedProblem[] | null = null; let outcomes: GenerationOutcome[] | null = null; + let slotResults: SlotExecutionResult[] = []; let usedFallback = false; let appliedFallbackReason: string | null = null; + let runStatus: GenerationRunStatus = "PENDING"; + let currentThreadState: SessionState = state; + let resumeState: ResumePlanState | null = null; + const existingActivityId = + typeof session.activity_id === "string" && session.activity_id.trim() ? session.activity_id : null; const derivePlanForSpec = (currentSpec: ActivitySpec) => { const pedagogyPolicy = @@ -84,10 +216,107 @@ export async function generateFromThread(sessionId: string): Promise { + const runScopedEvent = event.runId ? event : ({ ...event, runId } as GenerationProgressEvent); + if (runScopedEvent.type === "slot_started") { + generationSlotRunRepository.beginSlot({ + runId, + slotIndex: runScopedEvent.slotIndex, + topic: runScopedEvent.topic, + language: runScopedEvent.language, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: "slot_started", + payload: runScopedEvent, + }); + } else if (runScopedEvent.type === "slot_stage_started") { + generationSlotRunRepository.updateStage({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: translateEventStage(runScopedEvent.stage), + currentStage: translateEventStage(runScopedEvent.stage), + attemptCount: runScopedEvent.attempt, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex: runScopedEvent.slotIndex, + attempt: runScopedEvent.attempt, + stage: runScopedEvent.stage, + status: "stage_started", + payload: runScopedEvent, + }); + } else if (runScopedEvent.type === "slot_stage_finished") { + generationSlotRunRepository.updateStage({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: translateEventStage(runScopedEvent.stage), + currentStage: translateEventStage(runScopedEvent.stage), + attemptCount: runScopedEvent.attempt, + lastFailureKind: runScopedEvent.status === "failed" ? runScopedEvent.failureKind ?? null : null, + lastFailureCode: + runScopedEvent.status === "failed" + ? failureKindToFailureCode(runScopedEvent.failureKind, runScopedEvent.stage) + : null, + lastFailureMessage: runScopedEvent.status === "failed" ? runScopedEvent.message ?? null : null, + lastArtifactHash: runScopedEvent.artifactHash ?? null, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex: runScopedEvent.slotIndex, + attempt: runScopedEvent.attempt, + stage: runScopedEvent.stage, + status: runScopedEvent.status === "success" ? "stage_succeeded" : "stage_failed", + payload: runScopedEvent, + }); + } else if (runScopedEvent.type === "slot_completed") { + const slot = generationSlotRunRepository.find(runId, runScopedEvent.slotIndex); + generationSlotRunRepository.markTerminal({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: "SUCCEEDED", + attemptCount: slot?.attempt_count ?? 1, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: "slot_succeeded", + payload: runScopedEvent, + }); + } else if (runScopedEvent.type === "slot_failed_terminal") { + const slot = generationSlotRunRepository.find(runId, runScopedEvent.slotIndex); + generationSlotRunRepository.markTerminal({ + runId, + slotIndex: runScopedEvent.slotIndex, + status: + runScopedEvent.failureKind === "repair_no_progress" + ? "QUARANTINED" + : runScopedEvent.failureKind === "infra" || runScopedEvent.failureKind === "judge_infra_failure" + ? "HARD_FAILURE" + : "RETRYABLE_FAILURE", + attemptCount: slot?.attempt_count ?? 1, + lastFailureKind: runScopedEvent.failureKind, + lastFailureCode: + failureKindToFailureCode(runScopedEvent.failureKind, runScopedEvent.stage), + lastFailureMessage: runScopedEvent.message, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex: runScopedEvent.slotIndex, + stage: runScopedEvent.stage, + status: "slot_failed_terminal", + payload: runScopedEvent, + }); + } + publishGenerationProgress(runId, runScopedEvent); + }; + const workflow = createExecutionContext, { response?: GenerateFromThreadResponse }>({ - workflowId: `thread-generation:${sessionId}`, + workflowId: `thread-generation:${sessionId}:${runId}`, threadId: sessionId, - publishProgress: (event) => publishGenerationProgress(sessionId, event as GenerationProgressEvent), + runId, + publishProgress: (event) => persistRunProgress(event as GenerationProgressEvent), loggerName: "threads.generation", initialState: {}, initialResults: {}, @@ -96,21 +325,33 @@ export async function generateFromThread(sessionId: string): Promise { - if (state !== "READY") { - const err = new Error(`Cannot generate when session state is ${state}. Expected READY.`); + if (!["READY", "INCOMPLETE", "RETRYABLE_FAILURE", "HARD_FAILURE"].includes(state)) { + const err = new Error( + `Cannot generate when session state is ${state}. Expected READY, INCOMPLETE, RETRYABLE_FAILURE, or HARD_FAILURE.` + ); (err as any).status = 409; throw err; } - if (typeof session.activity_id === "string" && session.activity_id.trim()) { + if (existingActivityId && opts?.resume?.existingActivityId !== existingActivityId) { const err = new Error("Session already produced an activity. Cannot re-generate."); (err as any).status = 409; throw err; } + if (existingActivityId && opts?.resume?.existingActivityId === existingActivityId) { + const dbActivity = activityRepository.findById(existingActivityId); + if (!dbActivity || (dbActivity.status ?? "DRAFT") !== "INCOMPLETE") { + const err = new Error("Only incomplete activities can be repaired in place."); + (err as any).status = 409; + throw err; + } + } - transitionOrThrow(state, "GENERATING"); - threadRepository.updateState(sessionId, "GENERATING"); + transitionOrThrow(state as any, "GENERATE_PENDING"); + threadRepository.updateState(sessionId, "GENERATE_PENDING"); + currentThreadState = "GENERATE_PENDING"; + threadRepository.setLastError(sessionId, null); progressHeartbeat = setInterval(() => { - publishGenerationProgress(sessionId, { type: "heartbeat", ts: new Date().toISOString() }); + persistRunProgress({ type: "heartbeat", ts: currentIso() }); }, 1000); const specObj = parseSpecJson(session.spec_json); @@ -123,174 +364,251 @@ export async function generateFromThread(sessionId: string): Promise ({ + slotIndex: slot.index, + topic: slot.topics[0] ?? null, + language: slot.language, + })) + ); + if (resumeState?.keptSuccessfulSlotIndexes.length) { + const resumeProblemBySlot = buildProblemMapFromResume(resumeState.problems, resumeState.outcomes); + for (const slotIndex of resumeState.keptSuccessfulSlotIndexes) { + generationSlotRunRepository.markTerminal({ + runId, + slotIndex, + status: "SUCCEEDED", + attemptCount: 0, + title: resumeProblemBySlot.get(slotIndex)?.title ?? null, + }); + generationSlotRunRepository.appendTransition({ + runId, + slotIndex, + status: "slot_resumed", + payload: { runId, slotIndex, status: "SUCCEEDED" }, + }); + } + } + generationRunRepository.markRunning(runId); + transitionOrThrow("GENERATE_PENDING", "GENERATING"); + threadRepository.updateState(sessionId, "GENERATING"); + currentThreadState = "GENERATING"; threadRepository.setPlanJson(sessionId, JSON.stringify(plan)); - publishGenerationProgress(sessionId, { + persistRunProgress({ type: "generation_started", totalSlots: plan.length, totalProblems: plan.length, run: 1, }); - - if (resumeProblems.length > 0) { - for (let i = 0; i < Math.min(resumeProblems.length, plan.length); i++) { - publishGenerationProgress(sessionId, { type: "slot_completed", slotIndex: i }); - } - } + persistRunProgress({ type: "generation_run_status", status: "RUNNING" }); }, }, { id: "run-generation-pipeline", run: async () => { while (!problems) { - try { - const generated = await generateProblemsFromPlan(plan, { - customInstructionsMd: instructionsMd, - resume: { problems: resumeProblems, outcomes: resumeOutcomes }, - onProgress: (event: GenerationProgressEvent) => publishGenerationProgress(sessionId, event), - onCheckpoint: ({ problems: checkpointProblems, outcomes: checkpointOutcomes }) => { - threadRepository.setProblemsJson(sessionId, JSON.stringify(checkpointProblems)); - threadRepository.updateGenerationOutcomesJson(sessionId, JSON.stringify(checkpointOutcomes)); - }, + const generated = await generateProblemsFromPlan(plan, { + customInstructionsMd: instructionsMd, + ...(resumeState + ? { + resume: { problems: resumeState.problems, outcomes: resumeState.outcomes }, + targetSlotIndexes: resumeState.targetSlotIndexes, + } + : {}), + onProgress: (event: GenerationProgressEvent) => persistRunProgress(event), + onCheckpoint: ({ problems: checkpointProblems, outcomes: checkpointOutcomes }) => { + threadRepository.setProblemsJson(sessionId, JSON.stringify(checkpointProblems)); + threadRepository.updateGenerationOutcomesJson(sessionId, JSON.stringify(checkpointOutcomes)); + }, + }); + problems = generated.problems; + outcomes = generated.outcomes; + slotResults = generated.slotResults; + + const allFailed = slotResults.length > 0 && slotResults.every((result) => result.terminalStatus !== "SUCCEEDED"); + if (allFailed && !usedFallback && spec) { + const explicitDifficultyLocked = commitments?.difficulty_plan?.locked === true; + const explicitTopicsLocked = commitments?.topic_tags?.locked === true; + const decision = proposeGenerationFallbackWithPolicy(spec, { + allowDowngradeDifficulty: !explicitDifficultyLocked, + allowNarrowTopics: !explicitTopicsLocked, }); - problems = generated.problems; - outcomes = generated.outcomes; - } catch (err: any) { - if (err instanceof GenerationSlotFailureError) { - if (Array.isArray(err.problemsSoFar)) { - resumeProblems = err.problemsSoFar; - threadRepository.setProblemsJson(sessionId, JSON.stringify(resumeProblems)); - } - if (Array.isArray(err.outcomesSoFar)) { - resumeOutcomes = err.outcomesSoFar; - threadRepository.updateGenerationOutcomesJson(sessionId, JSON.stringify(resumeOutcomes)); - } + if (decision) { + usedFallback = true; + appliedFallbackReason = decision.reason; + + persistRunProgress({ + type: "generation_soft_fallback_applied", + reason: decision.reason, + patchPaths: decision.patch.map((p) => p.path), + }); persistTraceEvent({ - ts: new Date().toISOString(), - type: "generation_failure", - slotIndex: err.slotIndex, - kind: err.kind, - attempts: err.attempts, - title: err.title ?? null, - llmOutputHash: err.llmOutputHash ?? null, - message: err.message, - outcomes: err.outcomesSoFar ?? null, + ts: currentIso(), + type: "generation_soft_fallback", + reason: decision.reason, + patch: decision.patch, }); - trace("generation.failure.persisted", { - sessionId, - slotIndex: err.slotIndex, - kind: err.kind, - llmOutputHash: err.llmOutputHash, - }); + persistConfidencePatch(decision.patch); - if (!usedFallback && spec) { - const explicitDifficultyLocked = commitments?.difficulty_plan?.locked === true; - const explicitTopicsLocked = commitments?.topic_tags?.locked === true; - const decision = proposeGenerationFallbackWithPolicy(spec, { - allowDowngradeDifficulty: !explicitDifficultyLocked, - allowNarrowTopics: !explicitTopicsLocked, + const adjusted = applyJsonPatch(spec as any, decision.patch) as ActivitySpec; + const adjustedRes = ActivitySpecSchema.safeParse(adjusted); + if (!adjustedRes.success) { + persistTraceEvent({ + ts: currentIso(), + type: "generation_soft_fallback_failed", + reason: "fallback patch produced invalid ActivitySpec", + error: adjustedRes.error.issues[0]?.message ?? "invalid", }); - if (decision) { - usedFallback = true; - appliedFallbackReason = decision.reason; - - publishGenerationProgress(sessionId, { - type: "generation_soft_fallback_applied", - reason: decision.reason, - patchPaths: decision.patch.map((p) => p.path), - }); - - persistTraceEvent({ - ts: new Date().toISOString(), - type: "generation_soft_fallback", - reason: decision.reason, - patch: decision.patch, - }); - - persistConfidencePatch(decision.patch); - - const adjusted = applyJsonPatch(spec as any, decision.patch) as ActivitySpec; - const adjustedRes = ActivitySpecSchema.safeParse(adjusted); - if (!adjustedRes.success) { - persistTraceEvent({ - ts: new Date().toISOString(), - type: "generation_soft_fallback_failed", - reason: "fallback patch produced invalid ActivitySpec", - error: adjustedRes.error.issues[0]?.message ?? "invalid", - }); - throw err; - } - - spec = adjustedRes.data; - threadRepository.updateSpecJson(sessionId, JSON.stringify(spec)); - ({ plan } = derivePlanForSpec(spec)); - threadRepository.setPlanJson(sessionId, JSON.stringify(plan)); - continue; - } + throw new Error("Fallback patch produced an invalid ActivitySpec."); } - } - throw err; + spec = adjustedRes.data; + threadRepository.updateSpecJson(sessionId, JSON.stringify(spec)); + ({ plan } = derivePlanForSpec(spec)); + threadRepository.setPlanJson(sessionId, JSON.stringify(plan)); + generationSlotRunRepository.seed( + runId, + plan.map((slot) => ({ + slotIndex: slot.index, + topic: slot.topics[0] ?? null, + language: slot.language, + })) + ); + problems = null; + outcomes = null; + slotResults = []; + continue; + } } - } - - if (!problems) { - throw new Error("Generation failed: problems were not produced."); + break; } }, }, { id: "persist-generated-activity", run: async (ctx) => { - if (!problems || !spec) { - throw new Error("Generation did not produce finalized problems."); + if (!problems || !outcomes || !spec) { + throw new Error("Generation did not produce finalized results."); } - if (outcomes) { - const finalOutcomes = appliedFallbackReason - ? outcomes.map((outcome) => ({ - ...outcome, - appliedFallback: outcome.appliedFallback ?? appliedFallbackReason, - })) - : outcomes; - threadRepository.updateGenerationOutcomesJson(sessionId, JSON.stringify(finalOutcomes)); - persistTraceEvent({ - ts: new Date().toISOString(), - type: "generation_outcomes", - outcomes: finalOutcomes, - }); - } + const finalOutcomes = appliedFallbackReason + ? outcomes.map((outcome) => ({ + ...outcome, + appliedFallback: outcome.appliedFallback ?? appliedFallbackReason, + })) + : outcomes; + threadRepository.updateGenerationOutcomesJson(sessionId, JSON.stringify(finalOutcomes)); + persistTraceEvent({ + ts: currentIso(), + type: "generation_outcomes", + outcomes: finalOutcomes, + runId, + }); threadRepository.setProblemsJson(sessionId, JSON.stringify(problems)); + runStatus = deriveRunStatus(slotResults); + const successfulSlots = slotResults.filter((result) => result.terminalStatus === "SUCCEEDED").length; + const failedSlots = slotResults.filter((result) => result.terminalStatus !== "SUCCEEDED").length; + const { kind: lastFailureKind, code: lastFailureCode, message: lastFailureMessage } = getLastFailure(slotResults); + + let activityId: string | null = null; + if (problems.length > 0) { + if (resumeState?.existingActivityId) { + const updated = activityRepository.update(resumeState.existingActivityId, { + problems: JSON.stringify(problems), + status: activityStatusForRunStatus(runStatus), + }); + if (!updated) { + throw new Error("Failed to update incomplete activity during repair."); + } + activityId = resumeState.existingActivityId; + } else { + activityId = crypto.randomUUID(); + const activityTitle = `Activity (${problems.length} of ${spec.problem_count} problems)`; + activityRepository.create(activityId, activityTitle, JSON.stringify(problems), undefined, { + status: activityStatusForRunStatus(runStatus), + timeLimitSeconds: null, + }); + threadRepository.setActivityId(sessionId, activityId); + } + } - const activityId = crypto.randomUUID(); - const activityTitle = `Activity (${spec.problem_count} problems)`; - activityRepository.create(activityId, activityTitle, JSON.stringify(problems), undefined, { - status: "DRAFT", - timeLimitSeconds: null, + generationRunRepository.finish({ + id: runId, + status: runStatus, + activityId, + completedSlots: slotResults.length, + successfulSlots, + failedSlots, + lastFailureKind: lastFailureKind ?? null, + lastFailureCode: lastFailureCode ?? null, + lastFailureMessage: lastFailureMessage ?? null, }); - threadRepository.setActivityId(sessionId, activityId); - transitionOrThrow("GENERATING", "SAVED"); - threadRepository.updateState(sessionId, "SAVED"); - publishGenerationProgress(sessionId, { type: "generation_completed", activityId }); - publishGenerationProgress(sessionId, { type: "generation_complete", activityId }); + const terminalThreadState = mapRunStatusToThreadState(runStatus); + transitionOrThrow("GENERATING", terminalThreadState as any); + threadRepository.updateState(sessionId, terminalThreadState); + currentThreadState = terminalThreadState; + threadRepository.setLastError( + sessionId, + runStatus === "COMPLETED" + ? null + : lastFailureMessage ?? + (runStatus === "INCOMPLETE" || runStatus === "PARTIAL_SUCCESS" + ? "Generation completed incompletely. Some slots failed." + : "Generation failed.") + ); + persistRunProgress({ + type: "generation_run_status", + status: runStatus, + ...(activityId ? { activityId } : {}), + ...(lastFailureMessage ? { error: lastFailureMessage } : {}), + }); + if (activityId) { + persistRunProgress({ type: "generation_completed", activityId }); + persistRunProgress({ type: "generation_complete", activityId }); + } else { + persistRunProgress({ + type: "generation_failed", + error: lastFailureMessage ?? "Generation failed.", + }); + } if (usedFallback) { persistTraceEvent({ - ts: new Date().toISOString(), + ts: currentIso(), type: "generation_soft_fallback_succeeded", + runId, }); } + if (!activityId) { + throw new Error(lastFailureMessage ?? "Generation failed without producing any problems."); + } ctx.setResult("response", { activityId, problems }); }, }, @@ -302,19 +620,45 @@ export async function generateFromThread(sessionId: string): Promise result.terminalStatus === "SUCCEEDED").length, + failedSlots: slotResults.filter((result) => result.terminalStatus !== "SUCCEEDED").length, + lastFailureKind: "infra", + lastFailureCode: "THREAD_GENERATION_FAILED", + lastFailureMessage: err.message ?? "Unknown error during generation.", + }); + persistRunProgress({ + type: "generation_run_status", + status: failureRunStatus, + error: err.message ?? "Unknown error during generation.", + }); + persistRunProgress({ + type: "generation_failed", + error: "Generation failed. Please try again.", + }); + } } catch (transitionErr) { - console.error("Failed to transition session to READY:", transitionErr); + console.error("Failed to transition session after generation error:", transitionErr); } - - publishGenerationProgress(sessionId, { - type: "generation_failed", - error: "Generation failed. Please try again.", - ...(err instanceof GenerationSlotFailureError ? { slotIndex: err.slotIndex } : {}), - }); throw err; } finally { if (progressHeartbeat) clearInterval(progressHeartbeat); @@ -324,6 +668,80 @@ export async function generateFromThread(sessionId: string): Promise { + const session = requireSession(sessionId); + const state = session.state as SessionState; + if (state === "GENERATING" || state === "GENERATE_PENDING") { + const err = new Error("Cannot repair failed slots while generation is in progress."); + (err as any).status = 409; + throw err; + } + + if (!["INCOMPLETE", "RETRYABLE_FAILURE", "HARD_FAILURE"].includes(state)) { + const err = new Error( + `Cannot repair failed slots when session state is ${state}. Expected INCOMPLETE, RETRYABLE_FAILURE, or HARD_FAILURE.`, + ); + (err as any).status = 409; + throw err; + } + + const specObj = parseSpecJson(session.spec_json); + const specResult = ActivitySpecSchema.safeParse(specObj); + if (!specResult.success) { + throw new Error(`Invalid ActivitySpec: ${specResult.error.issues[0]?.message ?? "validation failed"}`); + } + const spec = specResult.data; + if (!isLanguageSupportedForGeneration(spec.language)) { + throw new Error(`Language "${spec.language}" is not supported for generation yet.`); + } + + const pedagogyPolicy = + parseLearningMode(session.learning_mode) === "guided" + ? buildGuidedPedagogyPolicy({ spec, learnerProfile: null }) + : undefined; + const plan = deriveProblemPlan(spec, pedagogyPolicy); + const existingProblems = parseGeneratedProblems(session.problems_json); + const existingOutcomes = parseGenerationOutcomes(session.generation_outcomes_json); + const problemBySlot = buildProblemMapFromResume(existingProblems, existingOutcomes); + + const targetSlotIndexes = plan + .filter((slot) => { + const outcome = existingOutcomes.find((candidate) => candidate.slotIndex === slot.index); + return !(outcome?.success && problemBySlot.has(slot.index)); + }) + .map((slot) => slot.index); + + if (targetSlotIndexes.length === 0) { + const err = new Error("There are no failed or interrupted slots left to repair."); + (err as any).status = 409; + throw err; + } + + const existingTrace = parseJsonArray(session.intent_trace_json); + const nextTrace = appendIntentTrace(existingTrace, { + ts: currentIso(), + type: "repair_failed_slots_requested", + targetSlotIndexes, + existingActivityId: session.activity_id ?? null, + }); + threadRepository.updateIntentTraceJson(sessionId, JSON.stringify(nextTrace)); + + const out = await generateFromThread(sessionId, { + ...(typeof opts?.runId === "string" && opts.runId.trim() ? { runId: opts.runId } : {}), + resume: { + problems: existingProblems, + outcomes: existingOutcomes, + targetSlotIndexes, + existingActivityId: session.activity_id ?? null, + mode: "repair_failed_slots", + }, + }); + return { ...out, repairedSlotIndexes: targetSlotIndexes }; +} + export async function regenerateSlotFromThread( sessionId: string, slotIndex: number, @@ -335,10 +753,18 @@ export async function regenerateSlotFromThread( | "narrow_topics" = "retry_full_slot" ): Promise { return withTraceContext({ sessionId }, async () => { + if (strategy !== "retry_full_slot") { + const err = new Error( + `Slot regeneration strategy "${strategy}" is deprecated until stage-targeted slot resume is implemented. Use "retry_full_slot".` + ); + (err as any).status = 400; + throw err; + } + const session = requireSession(sessionId); const state = session.state; - if (state === "GENERATING") { + if (state === "GENERATING" || state === "GENERATE_PENDING") { const err = new Error("Cannot regenerate a slot while generation is in progress."); (err as any).status = 409; throw err; @@ -350,8 +776,10 @@ export async function regenerateSlotFromThread( throw err; } - if (state !== "READY" && state !== "FAILED") { - const err = new Error(`Cannot regenerate slots when session state is ${state}. Expected READY or FAILED.`); + if (!["READY", "RETRYABLE_FAILURE", "HARD_FAILURE"].includes(state)) { + const err = new Error( + `Cannot regenerate slots when session state is ${state}. Expected READY, RETRYABLE_FAILURE, or HARD_FAILURE.` + ); (err as any).status = 409; throw err; } @@ -396,8 +824,8 @@ export async function regenerateSlotFromThread( const nextTrace = appendIntentTrace(existingTrace, traceEntry); threadRepository.updateIntentTraceJson(sessionId, JSON.stringify(nextTrace)); - if (state === "FAILED") { - transitionOrThrow("FAILED", "READY"); + if (state === "RETRYABLE_FAILURE" || state === "HARD_FAILURE") { + transitionOrThrow(state as any, "READY"); threadRepository.updateState(sessionId, "READY"); } diff --git a/apps/backend/src/services/threads/threadReadinessService.ts b/apps/backend/src/services/threads/threadReadinessService.ts index 34e3d06..a4e5006 100644 --- a/apps/backend/src/services/threads/threadReadinessService.ts +++ b/apps/backend/src/services/threads/threadReadinessService.ts @@ -1,5 +1,6 @@ import crypto from "crypto"; import { threadCollectorRepository, threadMessageRepository, threadRepository } from "../../database/repositories/threadRepository"; +import { generationRunRepository } from "../../database/repositories/generationRunRepository"; import type { SessionState } from "../../contracts/session"; import { DEFAULT_LEARNING_MODE, type LearningMode } from "../../contracts/learningMode"; import { listCommitments, parseCommitmentsJson } from "../../agent/commitments"; @@ -43,6 +44,7 @@ export function getSession(id: string): SessionRecord { const intentTrace = parseJsonArray(session.intent_trace_json).slice(-50); const generationOutcomes = parseGenerationOutcomes(session.generation_outcomes_json); const learning_mode = parseLearningMode(session.learning_mode); + const latestGenerationRun = generationRunRepository.latestByThread(id); return { id: session.id, @@ -56,6 +58,8 @@ export function getSession(id: string): SessionRecord { commitments, generationOutcomes, intentTrace, + latestGenerationRunId: latestGenerationRun?.id ?? null, + latestGenerationRunStatus: latestGenerationRun?.status ?? null, }; } diff --git a/apps/backend/src/types.ts b/apps/backend/src/types.ts index 6ed0e06..98855ba 100644 --- a/apps/backend/src/types.ts +++ b/apps/backend/src/types.ts @@ -1,3 +1,5 @@ +import type { JudgeFailureCategoryDto } from "@codemm/shared-contracts"; + /** * Shared types for Codemm v1.0. */ @@ -11,6 +13,12 @@ export interface JudgeResult { executionTimeMs: number; exitCode?: number; timedOut?: boolean; + failureCategory?: JudgeFailureCategoryDto; + timeoutStage?: "compile" | "execute" | "overall"; + watchdogSource?: "inner" | "outer" | "unknown"; + outputLimitExceeded?: boolean; + parsedFailures?: Record; + budgetProfile?: Record; } /** diff --git a/apps/backend/test/README.md b/apps/backend/test/README.md index 366eefc..6b53394 100644 --- a/apps/backend/test/README.md +++ b/apps/backend/test/README.md @@ -20,9 +20,8 @@ Codemm backend uses Node's built-in test runner (`node:test`) with CommonJS test | Test file | Uses DB | Uses Docker judges | Uses real LLM | What it validates | How to scope it down | |---|---:|---:|---:|---|---| | `test/integration/languages/activityGenerationEdgeCases.test.js` | ✅ | ❌ | ❌ (stubs) | Dialogue/spec edge-cases + generation plumbing without Docker/LLM flakiness | Run normally; it’s fast | -| `test/integration/llm/realActivityGenerationE2e.openai.test.js` | ✅ | ✅ | ✅ | Full production flow using OpenAI | Use `CODEMM_E2E_LANGS`, `CODEMM_E2E_STYLES`, `CODEMM_E2E_COUNTS` (comma-separated) | -| `test/integration/llm/realActivityGenerationE2e.anthropic.test.js` | ✅ | ✅ | ✅ | Full production flow using Anthropic | Use `CODEMM_E2E_LANGS`, `CODEMM_E2E_STYLES`, `CODEMM_E2E_COUNTS` (comma-separated) | -| `test/integration/llm/realActivityGenerationE2e.gemini.test.js` | ✅ | ✅ | ✅ | Full production flow using Gemini | Use `CODEMM_E2E_LANGS`, `CODEMM_E2E_STYLES`, `CODEMM_E2E_COUNTS` (comma-separated) | +| `test/integration/languages/activityGenerationMatrix.test.js` | ✅ | ❌ | ❌ (stubs) | Fast language-matrix coverage for java/python/cpp/sql generation plumbing | Run normally; it replaces four duplicated per-language files | +| `test/integration/llm/realActivityGenerationE2e.test.js` | ✅ | ✅ | ✅ | Full production flow across all configured providers plus auto-provider fallback | Use `CODEMM_E2E_PROVIDERS`, `CODEMM_E2E_LANGS`, `CODEMM_E2E_STYLES`, `CODEMM_E2E_COUNTS` | | `test/integration/llm/realProviderIntegrations.test.js` | ❌ | ❌ | ✅ | Tiny smoke tests for the raw provider adapters (token-costing) | Opt-in: `CODEMM_RUN_REAL_PROVIDER_SMOKE=1` | ## Conventions @@ -32,21 +31,21 @@ Codemm backend uses Node's built-in test runner (`node:test`) with CommonJS test ## Real-LLM e2e (required) -- `test/integration/llm/realActivityGenerationE2e.*.test.js` runs the full flow (dialogue + generation + Docker validation) and requires: +- `test/integration/llm/realActivityGenerationE2e.test.js` runs the full flow (dialogue + generation + Docker validation) and requires: - an LLM API key in the environment (one of `CODEX_API_KEY`/`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`/`GOOGLE_API_KEY`) - local Docker running + judge images built (`./run-codem-backend.sh`) -Defaults: this test only runs `CODEMM_E2E_COUNTS=2` unless you override it. +Defaults: this test only runs `CODEMM_E2E_LANGS=java` and `CODEMM_E2E_COUNTS=1` unless you override it. To run a single “cell” in the matrix: -- `CODEMM_E2E_LANGS=java CODEMM_E2E_STYLES=stdout CODEMM_E2E_COUNTS=2 npm run test:integration` +- `CODEMM_RUN_REAL_PROVIDER_SMOKE=1 CODEMM_E2E_PROVIDERS=openai CODEMM_E2E_LANGS=java CODEMM_E2E_STYLES=stdout CODEMM_E2E_COUNTS=1 npm --workspace codem-backend run test:integration -- llm/realActivityGenerationE2e.test.js` ## Debugging generation failures (the errors in your log) -The real-LLM e2e test ultimately calls `generateFromSession()`, which calls `generateProblemsFromPlan()` (per-slot retries). +The real-LLM e2e test ultimately calls `generateFromSession()`, which calls `generateProblemsFromPlan()` and the staged slot pipeline. -When a slot fails after retries, you’ll see a `GenerationSlotFailureError` like: -- `kind="contract"`: LLM output didn’t match the schema/format rules (most commonly `test_suite` shape). +When a slot fails terminally, you’ll see a slot failure classified like: +- `kind="contract"`: staged generation or deterministic preflight rejected the draft before Docker (schema mismatch, structural-topic violation, etc.). - `kind="tests"` / `kind="compile"` / `kind="timeout"`: Docker judge ran the reference solution and it failed. To see the raw per-slot LLM output + judge stdout/stderr in your terminal: @@ -54,7 +53,7 @@ To see the raw per-slot LLM output + judge stdout/stderr in your terminal: | Failure kind | Where it comes from | What it usually means | What to inspect next | |---|---|---|---| -| `contract` | `src/generation/perSlotGenerator.ts` schema/style checks | Bad JSON / missing fields / wrong `test_suite` format | The “Last error:” message + the generated draft’s `test_suite` format rules for that language | +| `contract` | `src/pipeline/slotStages.ts` preflight + staged artifact validation | Bad JSON / missing fields / wrong `test_suite` format / structural-topic rule violation | The slot-stage failure message + the generated artifact shape rules for that language | | `tests` | `src/generation/referenceSolutionValidator.ts` (Docker) | Reference solution and tests disagree (or tests are too strict/brittle) | Judge output showing the first failing test (e.g. JUnit `expected … but was …`) | | `compile` | `src/generation/referenceSolutionValidator.ts` (Docker) | Reference solution didn’t compile | Judge stderr/compile output | | `timeout` | `src/generation/referenceSolutionValidator.ts` (Docker) | Judge timed out (infinite loop / very slow solution/tests) | Judge stdout/stderr + consider tightening generator prompts | diff --git a/apps/backend/test/helpers/loadRealProviderAuth.js b/apps/backend/test/helpers/loadRealProviderAuth.js new file mode 100644 index 0000000..51b198f --- /dev/null +++ b/apps/backend/test/helpers/loadRealProviderAuth.js @@ -0,0 +1,41 @@ +const path = require("node:path"); + +function loadDotenvFallbacks() { + try { + require("dotenv").config({ + path: path.resolve(__dirname, "../../.env"), + quiet: true, + }); + } catch { + // ignore + } + + try { + require("dotenv").config({ + path: path.resolve(__dirname, "../../../../.env"), + quiet: true, + override: false, + }); + } catch { + // ignore + } +} + +function hasAnyProviderKey() { + return Boolean( + process.env.CODEX_API_KEY || + process.env.OPENAI_API_KEY || + process.env.ANTHROPIC_API_KEY || + process.env.GEMINI_API_KEY || + process.env.GOOGLE_API_KEY + ); +} + +function loadRealProviderAuth() { + if (hasAnyProviderKey()) return { source: "env" }; + loadDotenvFallbacks(); + if (hasAnyProviderKey()) return { source: "dotenv" }; + return { source: null }; +} + +module.exports = { loadRealProviderAuth }; diff --git a/apps/backend/test/integration/database/generationRecovery.test.js b/apps/backend/test/integration/database/generationRecovery.test.js new file mode 100644 index 0000000..5aff56b --- /dev/null +++ b/apps/backend/test/integration/database/generationRecovery.test.js @@ -0,0 +1,73 @@ +require("../../helpers/setupDb"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); + +const { threadRepository } = require("../../../src/database/repositories/threadRepository"); +const { + generationRunRepository, + generationSlotRunRepository, +} = require("../../../src/database/repositories/generationRunRepository"); +const { reconcileInterruptedGenerationState } = require("../../../src/services/threads/generationRecoveryService"); + +test("reconcileInterruptedGenerationState finalizes stale generation runs and repairs stuck thread state", () => { + const threadId = crypto.randomUUID(); + const runId = crypto.randomUUID(); + threadRepository.create( + threadId, + "GENERATING", + "practice", + JSON.stringify({ + language: "java", + topic_tags: ["arrays"], + problem_count: 2, + difficulty_plan: { easy: 2, medium: 0, hard: 0 }, + problem_style: "return", + }) + ); + + generationRunRepository.create({ + id: runId, + threadId, + totalSlots: 2, + metaJson: JSON.stringify({ threadId }), + }); + generationRunRepository.markRunning(runId); + generationSlotRunRepository.seed(runId, [ + { slotIndex: 0, topic: "arrays", language: "java" }, + { slotIndex: 1, topic: "strings", language: "java" }, + ]); + generationSlotRunRepository.markTerminal({ + runId, + slotIndex: 0, + status: "SUCCEEDED", + attemptCount: 1, + }); + generationSlotRunRepository.updateStage({ + runId, + slotIndex: 1, + status: "REFERENCE_RUNNING", + currentStage: "REFERENCE_RUNNING", + attemptCount: 1, + }); + + const result = reconcileInterruptedGenerationState(); + + assert.deepEqual(result.reconciledRunIds, [runId]); + assert.ok(result.updatedThreadIds.includes(threadId)); + + const recoveredRun = generationRunRepository.findById(runId); + assert.equal(recoveredRun.status, "INCOMPLETE"); + assert.equal(recoveredRun.successful_slots, 1); + assert.equal(recoveredRun.failed_slots, 1); + assert.equal(recoveredRun.last_failure_code, "ENGINE_RESTART"); + + const slots = generationSlotRunRepository.listByRun(runId); + assert.equal(slots[0].status, "SUCCEEDED"); + assert.equal(slots[1].status, "RETRYABLE_FAILURE"); + + const thread = threadRepository.findById(threadId); + assert.equal(thread.state, "INCOMPLETE"); + assert.match(String(thread.last_error ?? ""), /interrupted before completion/i); +}); diff --git a/apps/backend/test/integration/languages/activityGenerationEdgeCases.test.js b/apps/backend/test/integration/languages/activityGenerationEdgeCases.test.js index 6c08076..8543f06 100644 --- a/apps/backend/test/integration/languages/activityGenerationEdgeCases.test.js +++ b/apps/backend/test/integration/languages/activityGenerationEdgeCases.test.js @@ -89,7 +89,8 @@ test("e2e edge: missing difficulty requires confirmation, then 'yes' applies pen assert.equal(gen.problems.length, 4); const s = getSession(sessionId); - assert.equal(s.state, "SAVED"); + assert.equal(s.state, "COMPLETED"); + assert.equal(s.latestGenerationRunStatus, "COMPLETED"); }); test("e2e edge: problem_count > 7 (without difficulty) does not complete the spec", async (t) => { @@ -119,5 +120,6 @@ test("e2e edge: problem_count > 7 with difficulty shorthand clamps to 7", async assert.equal(gen.problems.length, 7); const s = getSession(sessionId); - assert.equal(s.state, "SAVED"); + assert.equal(s.state, "COMPLETED"); + assert.equal(s.latestGenerationRunStatus, "COMPLETED"); }); diff --git a/apps/backend/test/integration/languages/activityGenerationMatrix.test.js b/apps/backend/test/integration/languages/activityGenerationMatrix.test.js new file mode 100644 index 0000000..c22eecc --- /dev/null +++ b/apps/backend/test/integration/languages/activityGenerationMatrix.test.js @@ -0,0 +1,253 @@ +require("../../helpers/setupDb"); +const { installGenerationStub } = require("../../helpers/installGenerationStub"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { activityDb } = require("../../../src/database"); +const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../src/services/sessionService"); + +const LANGUAGE_CASES = { + java: { + topic: "arrays", + promptLanguage: "Java", + buildDraft(slotIndex) { + return { + id: `java-e2e-${slotIndex}`, + title: `Adder ${slotIndex}`, + description: "Print a + b.", + starter_code: ` +public class Adder { + public void solve(int a, int b) { + // TODO + System.out.println(0); + } +} +`.trim(), + reference_solution: ` +public class Adder { + public void solve(int a, int b) { + System.out.println(a + b); + } +} +`.trim(), + test_suite: ` +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; +import java.io.*; + +public class AdderTest { + private String run(int a, int b) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PrintStream prev = System.out; + System.setOut(new PrintStream(out)); + try { new Adder().solve(a, b); } + finally { System.setOut(prev); } + return out.toString().trim(); + } + + @Test void test_case_1(){ assertEquals("3", run(1,2)); } + @Test void test_case_2(){ assertEquals("0", run(0,0)); } + @Test void test_case_3(){ assertEquals("-1", run(-2,1)); } + @Test void test_case_4(){ assertEquals("7", run(10,-3)); } + @Test void test_case_5(){ assertEquals("123", run(100,23)); } + @Test void test_case_6(){ assertEquals("-11", run(-5,-6)); } + @Test void test_case_7(){ assertEquals("15", run(7,8)); } + @Test void test_case_8(){ assertEquals("2147483647", run(2147483640, 7)); } +} +`.trim(), + constraints: "Java 17, JUnit 5, no package declarations.", + sample_inputs: ["a=1, b=2"], + sample_outputs: ["3"], + difficulty: "easy", + topic_tag: "arrays", + }; + }, + }, + python: { + topic: "strings", + promptLanguage: "Python", + buildDraft(slotIndex) { + return { + id: `py-e2e-${slotIndex}`, + title: `Print Len ${slotIndex}`, + description: "Print len(s).", + starter_code: "def solve(s: str) -> None:\n # TODO\n raise NotImplementedError\n", + reference_solution: "def solve(s: str) -> None:\n print(len(s))\n", + test_suite: `import pytest +from solution import solve + +def test_case_1(capsys): solve(""); captured = capsys.readouterr(); assert captured.out.strip() == "0" +def test_case_2(capsys): solve("a"); captured = capsys.readouterr(); assert captured.out.strip() == "1" +def test_case_3(capsys): solve("abc"); captured = capsys.readouterr(); assert captured.out.strip() == "3" +def test_case_4(capsys): solve("hello"); captured = capsys.readouterr(); assert captured.out.strip() == "5" +def test_case_5(capsys): solve(" "); captured = capsys.readouterr(); assert captured.out.strip() == "2" +def test_case_6(capsys): solve("🙂"); captured = capsys.readouterr(); assert captured.out.strip() == "1" +def test_case_7(capsys): solve("line\\nbreak"); captured = capsys.readouterr(); assert captured.out.strip() == "10" +def test_case_8(capsys): solve("x" * 20); captured = capsys.readouterr(); assert captured.out.strip() == "20" +`, + constraints: + "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced.", + sample_inputs: ['s = "abc"'], + sample_outputs: ["3"], + difficulty: "easy", + topic_tag: "strings", + }; + }, + }, + cpp: { + topic: "graphs", + promptLanguage: "C++", + buildDraft(slotIndex) { + return { + id: `cpp-e2e-${slotIndex}`, + title: `Print Adder ${slotIndex}`, + description: "Print a+b.", + starter_code: + '#include \\n\\nvoid solve(int a, int b) {\\n // TODO\\n}\\n', + reference_solution: + '#include \\n\\nvoid solve(int a, int b) {\\n std::cout << (a + b) << "\\\\n";\\n}\\n', + test_suite: `#include +#include "solution.cpp" +static int __codem_failures = 0; +#define RUN_TEST(name, ...) do { \\ + try { __VA_ARGS__; std::cout << "[PASS] " << (name) << "\\n"; } \\ + catch (...) { std::cout << "[FAIL] " << (name) << "\\n"; __codem_failures++; } \\ +} while (0) + +static std::string capture_stdout(std::function fn) { + std::ostringstream oss; + auto* old = std::cout.rdbuf(oss.rdbuf()); + fn(); + std::cout.rdbuf(old); + return oss.str(); +} + +int main() { + RUN_TEST("test_case_1", { auto out = capture_stdout([&]{ solve(1,2); }); if (out != "3\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_2", { auto out = capture_stdout([&]{ solve(0,0); }); if (out != "0\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_3", { auto out = capture_stdout([&]{ solve(-1,2); }); if (out != "1\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_4", { auto out = capture_stdout([&]{ solve(10,-3); }); if (out != "7\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_5", { auto out = capture_stdout([&]{ solve(100,23); }); if (out != "123\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_6", { auto out = capture_stdout([&]{ solve(-5,-6); }); if (out != "-11\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_7", { auto out = capture_stdout([&]{ solve(7,8); }); if (out != "15\\n") throw std::runtime_error("fail"); }); + RUN_TEST("test_case_8", { auto out = capture_stdout([&]{ solve(2147483640,7); }); if (out != "2147483647\\n") throw std::runtime_error("fail"); }); + return __codem_failures ? 1 : 0; +} +`, + constraints: + "C++20, g++ (GNU), standard library only, no filesystem access, no networking, deterministic behavior.", + sample_inputs: ["a=1, b=2"], + sample_outputs: ["3"], + difficulty: "easy", + topic_tag: "graphs", + }; + }, + }, + sql: { + topic: "filtering", + promptLanguage: "SQL", + buildDraft(slotIndex) { + const suite = { + schema_sql: "CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER);", + cases: Array.from({ length: 8 }, (_, i) => ({ + name: `test_case_${i + 1}`, + seed_sql: `INSERT INTO t (id, v) VALUES (${i + 1}, ${i});`, + expected: { columns: ["v"], rows: [[i]] }, + order_matters: true, + })), + }; + + return { + id: `sql-e2e-${slotIndex}`, + title: `Select V ${slotIndex}`, + description: "Return v for id=1.", + starter_code: "SELECT v FROM t WHERE id = 1 ORDER BY v;", + reference_solution: "SELECT v FROM t WHERE id = 1 ORDER BY v;", + test_suite: JSON.stringify(suite), + constraints: "SQLite 3 (SQL dialect), read-only queries only, deterministic results (explicit ORDER BY when needed).", + sample_inputs: ["t rows: (id=1,v=0)"], + sample_outputs: ["v\\n0"], + difficulty: "easy", + topic_tag: "filtering", + }; + }, + }, +}; + +function parseRequestedCountAndTopic(msg, fallbackTopic) { + const text = String(msg || ""); + const lower = text.toLowerCase(); + const countMatch = lower.match(/\b(\d+)\s+(?:problems?|questions?)\b/); + const count = countMatch ? Number(countMatch[1]) : 1; + const topicsMatch = text.match(/\btopics?\s*:\s*([A-Za-z0-9 _-]+)/i); + const topic = topicsMatch?.[1]?.trim().split(/[,\n]/)[0]?.trim() || fallbackTopic; + return { count, topic }; +} + +function installLanguageStubs(t, language) { + const config = LANGUAGE_CASES[language]; + return installGenerationStub(t, { + language, + buildDialogueResponse(latestUserMessage) { + const { count, topic } = parseRequestedCountAndTopic(latestUserMessage, config.topic); + return { + acknowledgement: "OK", + inferred_intent: "Generate an activity.", + proposedPatch: { + language, + problem_count: count, + difficulty_plan: [{ difficulty: "easy", count }], + topic_tags: [topic], + }, + }; + }, + buildDraft: config.buildDraft, + judgeResult: { success: false, passedTests: [], failedTests: ["baseline"] }, + }); +} + +test("e2e activity generation matrix: java/python/cpp/sql (stdout-only)", async (t) => { + const counts = [2, 4, 7]; + + for (const [language, config] of Object.entries(LANGUAGE_CASES)) { + await t.test(language, async (tLang) => { + const { calls } = installLanguageStubs(tLang, language); + + for (const problemCount of counts) { + await tLang.test(`count=${problemCount}`, async () => { + calls.length = 0; + + const { sessionId } = createSession("practice"); + const prompt = `Create ${problemCount} easy problems in ${config.promptLanguage}. Topics: ${config.topic}`; + + const msgRes = await processSessionMessage(sessionId, prompt); + assert.equal(msgRes.accepted, true); + assert.equal(msgRes.done, true); + assert.equal(msgRes.state, "READY"); + assert.equal(msgRes.spec.language, language); + assert.equal(msgRes.spec.problem_count, problemCount); + assert.equal(msgRes.spec.problem_style, "stdout"); + + const genRes = await generateFromSession(sessionId); + assert.ok(genRes.activityId); + assert.equal(genRes.problems.length, problemCount); + for (const problem of genRes.problems) { + assert.equal(problem.language, language); + assert.equal("reference_solution" in problem, false); + assert.equal("reference_workspace" in problem, false); + } + + const stored = activityDb.findById(genRes.activityId); + assert.ok(stored); + const storedProblems = JSON.parse(stored.problems); + assert.equal(storedProblems.length, problemCount); + + const session = getSession(sessionId); + assert.equal(session.state, "COMPLETED"); + assert.equal(session.latestGenerationRunStatus, "COMPLETED"); + }); + } + }); + } +}); diff --git a/apps/backend/test/integration/languages/cpp/activityGenerationE2e.test.js b/apps/backend/test/integration/languages/cpp/activityGenerationE2e.test.js deleted file mode 100644 index c15d15d..0000000 --- a/apps/backend/test/integration/languages/cpp/activityGenerationE2e.test.js +++ /dev/null @@ -1,126 +0,0 @@ -require("../../../helpers/setupDb"); -const { installGenerationStub } = require("../../../helpers/installGenerationStub"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { activityDb } = require("../../../../src/database"); -const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../../src/services/sessionService"); - -function installStubs(t, language) { - function parseRequestedCountAndTopic(msg) { - const m = String(msg || ""); - const lower = m.toLowerCase(); - const countMatch = lower.match(/\b(\d+)\s+(?:problems?|questions?)\b/); - const count = countMatch ? Number(countMatch[1]) : 1; - const topicsMatch = m.match(/\btopics?\s*:\s*([A-Za-z0-9 _-]+)/i); - const topic = topicsMatch?.[1]?.trim().split(/[,\n]/)[0]?.trim() || "graphs"; - return { count, topic }; - } - - function buildDialogueResponse(latestUserMessage) { - const { count, topic } = parseRequestedCountAndTopic(latestUserMessage); - return { - acknowledgement: "OK", - inferred_intent: "Generate an activity.", - proposedPatch: { - language, - problem_count: count, - difficulty_plan: [{ difficulty: "easy", count }], - topic_tags: [topic], - }, - }; - } - - function cppDraft(slotIndex) { - return { - id: `cpp-e2e-${slotIndex}`, - title: `Print Adder ${slotIndex}`, - description: "Print a+b.", - starter_code: - '#include \\n\\nvoid solve(int a, int b) {\\n // TODO\\n}\\n', - reference_solution: - '#include \\n\\nvoid solve(int a, int b) {\\n std::cout << (a + b) << \"\\\\n\";\\n}\\n', - test_suite: `#include -#include "solution.cpp" -static int __codem_failures = 0; -#define RUN_TEST(name, ...) do { \\ - try { __VA_ARGS__; std::cout << "[PASS] " << (name) << "\\n"; } \\ - catch (...) { std::cout << "[FAIL] " << (name) << "\\n"; __codem_failures++; } \\ -} while (0) - -static std::string capture_stdout(std::function fn) { - std::ostringstream oss; - auto* old = std::cout.rdbuf(oss.rdbuf()); - fn(); - std::cout.rdbuf(old); - return oss.str(); -} - -int main() { - RUN_TEST("test_case_1", { auto out = capture_stdout([&]{ solve(1,2); }); if (out != "3\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_2", { auto out = capture_stdout([&]{ solve(0,0); }); if (out != "0\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_3", { auto out = capture_stdout([&]{ solve(-1,2); }); if (out != "1\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_4", { auto out = capture_stdout([&]{ solve(10,-3); }); if (out != "7\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_5", { auto out = capture_stdout([&]{ solve(100,23); }); if (out != "123\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_6", { auto out = capture_stdout([&]{ solve(-5,-6); }); if (out != "-11\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_7", { auto out = capture_stdout([&]{ solve(7,8); }); if (out != "15\\n") throw std::runtime_error("fail"); }); - RUN_TEST("test_case_8", { auto out = capture_stdout([&]{ solve(2147483640,7); }); if (out != "2147483647\\n") throw std::runtime_error("fail"); }); - return __codem_failures ? 1 : 0; -} -`, - constraints: - "C++20, g++ (GNU), standard library only, no filesystem access, no networking, deterministic behavior.", - sample_inputs: ["a=1, b=2"], - sample_outputs: ["3"], - difficulty: "easy", - topic_tag: "graphs", - }; - } - - return installGenerationStub(t, { - language, - buildDialogueResponse, - buildDraft: cppDraft, - judgeResult: { success: false, passedTests: [], failedTests: ["baseline"] }, - }); -} - -test("e2e activity generation (cpp): 2/4/7 problems (stdout-only)", async (t) => { - const { calls } = installStubs(t, "cpp"); - - const counts = [2, 4, 7]; - - for (const problem_count of counts) { - await t.test(`count=${problem_count}`, async () => { - calls.length = 0; - - const { sessionId } = createSession("practice"); - const prompt = `Create ${problem_count} easy problems in C++. Topics: graphs`; - - const msgRes = await processSessionMessage(sessionId, prompt); - assert.equal(msgRes.accepted, true); - assert.equal(msgRes.done, true); - assert.equal(msgRes.state, "READY"); - assert.equal(msgRes.spec.language, "cpp"); - assert.equal(msgRes.spec.problem_count, problem_count); - assert.equal(msgRes.spec.problem_style, "stdout"); - - const genRes = await generateFromSession(sessionId); - assert.ok(genRes.activityId); - assert.equal(genRes.problems.length, problem_count); - for (const p of genRes.problems) { - assert.equal(p.language, "cpp"); - assert.equal("reference_solution" in p, false); - } - - const stored = activityDb.findById(genRes.activityId); - assert.ok(stored); - const storedProblems = JSON.parse(stored.problems); - assert.equal(storedProblems.length, problem_count); - - const session = getSession(sessionId); - assert.equal(session.state, "SAVED"); - }); - } -}); diff --git a/apps/backend/test/integration/languages/hardIntentWeakTestsBlocked.test.js b/apps/backend/test/integration/languages/hardIntentWeakTestsBlocked.test.js index 13668a9..2889562 100644 --- a/apps/backend/test/integration/languages/hardIntentWeakTestsBlocked.test.js +++ b/apps/backend/test/integration/languages/hardIntentWeakTestsBlocked.test.js @@ -86,7 +86,7 @@ public class BillingTest { }); } -test("e2e: hard intent + weak tests (baseline passes) must not reach SAVED or silently downgrade", async (t) => { +test("e2e: hard intent + weak tests (baseline passes) must not reach successful completion or silently downgrade", async (t) => { installStubs(t); const { sessionId } = createSession("practice"); @@ -99,7 +99,8 @@ test("e2e: hard intent + weak tests (baseline passes) must not reach SAVED or si await assert.rejects(() => generateFromSession(sessionId)); const s = getSession(sessionId); - assert.equal(s.state, "READY"); + assert.equal(s.state, "RETRYABLE_FAILURE"); + assert.equal(s.latestGenerationRunStatus, "RETRYABLE_FAILURE"); assert.equal(s.spec.problem_style, "stdout"); assert.ok(Array.isArray(s.spec.difficulty_plan)); assert.ok(s.spec.difficulty_plan.some((x) => x && x.difficulty === "hard")); diff --git a/apps/backend/test/integration/languages/java/activityGenerationE2e.test.js b/apps/backend/test/integration/languages/java/activityGenerationE2e.test.js deleted file mode 100644 index c4514f8..0000000 --- a/apps/backend/test/integration/languages/java/activityGenerationE2e.test.js +++ /dev/null @@ -1,135 +0,0 @@ -require("../../../helpers/setupDb"); -const { installGenerationStub } = require("../../../helpers/installGenerationStub"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { activityDb } = require("../../../../src/database"); -const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../../src/services/sessionService"); - -function installStubs(t, language) { - function parseRequestedCountAndTopic(msg) { - const m = String(msg || ""); - const lower = m.toLowerCase(); - const countMatch = lower.match(/\b(\d+)\s+(?:problems?|questions?)\b/); - const count = countMatch ? Number(countMatch[1]) : 1; - const topicsMatch = m.match(/\btopics?\s*:\s*([A-Za-z0-9 _-]+)/i); - const topic = topicsMatch?.[1]?.trim().split(/[,\n]/)[0]?.trim() || "arrays"; - return { count, topic }; - } - - function buildDialogueResponse(latestUserMessage) { - const { count, topic } = parseRequestedCountAndTopic(latestUserMessage); - return { - acknowledgement: "OK", - inferred_intent: "Generate an activity.", - proposedPatch: { - language, - problem_count: count, - difficulty_plan: [{ difficulty: "easy", count }], - topic_tags: [topic], - }, - }; - } - - function javaDraft(slotIndex) { - return { - id: `java-e2e-${slotIndex}`, - title: `Adder ${slotIndex}`, - description: "Print a + b.", - starter_code: ` -public class Adder { - public void solve(int a, int b) { - // TODO - System.out.println(0); - } -} -`.trim(), - reference_solution: ` -public class Adder { - public void solve(int a, int b) { - System.out.println(a + b); - } -} -`.trim(), - test_suite: ` -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import java.io.*; - -public class AdderTest { - private String run(int a, int b) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - PrintStream prev = System.out; - System.setOut(new PrintStream(out)); - try { new Adder().solve(a, b); } - finally { System.setOut(prev); } - return out.toString().trim(); - } - - @Test void test_case_1(){ assertEquals("3", run(1,2)); } - @Test void test_case_2(){ assertEquals("0", run(0,0)); } - @Test void test_case_3(){ assertEquals("-1", run(-2,1)); } - @Test void test_case_4(){ assertEquals("7", run(10,-3)); } - @Test void test_case_5(){ assertEquals("123", run(100,23)); } - @Test void test_case_6(){ assertEquals("-11", run(-5,-6)); } - @Test void test_case_7(){ assertEquals("15", run(7,8)); } - @Test void test_case_8(){ assertEquals("2147483647", run(2147483640, 7)); } -} -`.trim(), - constraints: "Java 17, JUnit 5, no package declarations.", - sample_inputs: ["a=1, b=2"], - sample_outputs: ["3"], - difficulty: "easy", - topic_tag: "arrays", - }; - } - - return installGenerationStub(t, { - language, - buildDialogueResponse, - buildDraft: javaDraft, - judgeResult: { success: false, passedTests: [], failedTests: ["baseline"] }, - }); -} - -test("e2e activity generation (java): 2/4/7 problems (stdout-only)", async (t) => { - const { calls } = installStubs(t, "java"); - - const counts = [2, 4, 7]; - - for (const problem_count of counts) { - await t.test(`count=${problem_count} style=stdout`, async () => { - calls.length = 0; - - const { sessionId } = createSession("practice"); - const prompt = `Create ${problem_count} easy problems in Java. Topics: arrays`; - - const msgRes = await processSessionMessage(sessionId, prompt); - assert.equal(msgRes.accepted, true); - assert.equal(msgRes.done, true); - assert.equal(msgRes.state, "READY"); - assert.equal(msgRes.spec.language, "java"); - assert.equal(msgRes.spec.problem_count, problem_count); - assert.equal(msgRes.spec.problem_style, "stdout"); - - const genRes = await generateFromSession(sessionId); - assert.ok(genRes.activityId); - assert.equal(genRes.problems.length, problem_count); - for (const p of genRes.problems) { - assert.equal(p.language, "java"); - assert.equal("reference_solution" in p, false); - assert.equal("reference_workspace" in p, false); - } - - // Stored activity has correct problem count. - const stored = activityDb.findById(genRes.activityId); - assert.ok(stored); - const storedProblems = JSON.parse(stored.problems); - assert.equal(storedProblems.length, problem_count); - - const session = getSession(sessionId); - assert.equal(session.state, "SAVED"); - }); - } -}); diff --git a/apps/backend/test/integration/languages/python/activityGenerationE2e.test.js b/apps/backend/test/integration/languages/python/activityGenerationE2e.test.js deleted file mode 100644 index 84126d0..0000000 --- a/apps/backend/test/integration/languages/python/activityGenerationE2e.test.js +++ /dev/null @@ -1,108 +0,0 @@ -require("../../../helpers/setupDb"); -const { installGenerationStub } = require("../../../helpers/installGenerationStub"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { activityDb } = require("../../../../src/database"); -const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../../src/services/sessionService"); - -function installStubs(t, language) { - function parseRequestedCountAndTopic(msg) { - const m = String(msg || ""); - const lower = m.toLowerCase(); - const countMatch = lower.match(/\b(\d+)\s+(?:problems?|questions?)\b/); - const count = countMatch ? Number(countMatch[1]) : 1; - const topicsMatch = m.match(/\btopics?\s*:\s*([A-Za-z0-9 _-]+)/i); - const topic = topicsMatch?.[1]?.trim().split(/[,\n]/)[0]?.trim() || "strings"; - return { count, topic }; - } - - function buildDialogueResponse(latestUserMessage) { - const { count, topic } = parseRequestedCountAndTopic(latestUserMessage); - return { - acknowledgement: "OK", - inferred_intent: "Generate an activity.", - proposedPatch: { - language, - problem_count: count, - difficulty_plan: [{ difficulty: "easy", count }], - topic_tags: [topic], - }, - }; - } - - function pythonDraft(slotIndex) { - return { - id: `py-e2e-${slotIndex}`, - title: `Print Len ${slotIndex}`, - description: "Print len(s).", - starter_code: "def solve(s: str) -> None:\n # TODO\n raise NotImplementedError\n", - reference_solution: "def solve(s: str) -> None:\n print(len(s))\n", - test_suite: `import pytest -from solution import solve - -def test_case_1(capsys): solve(""); captured = capsys.readouterr(); assert captured.out.strip() == "0" -def test_case_2(capsys): solve("a"); captured = capsys.readouterr(); assert captured.out.strip() == "1" -def test_case_3(capsys): solve("abc"); captured = capsys.readouterr(); assert captured.out.strip() == "3" -def test_case_4(capsys): solve("hello"); captured = capsys.readouterr(); assert captured.out.strip() == "5" -def test_case_5(capsys): solve(" "); captured = capsys.readouterr(); assert captured.out.strip() == "2" -def test_case_6(capsys): solve("🙂"); captured = capsys.readouterr(); assert captured.out.strip() == "1" -def test_case_7(capsys): solve("line\\nbreak"); captured = capsys.readouterr(); assert captured.out.strip() == "10" -def test_case_8(capsys): solve("x" * 20); captured = capsys.readouterr(); assert captured.out.strip() == "20" -`, - constraints: - "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced.", - sample_inputs: ['s = "abc"'], - sample_outputs: ["3"], - difficulty: "easy", - topic_tag: "strings", - }; - } - - return installGenerationStub(t, { - language, - buildDialogueResponse, - buildDraft: pythonDraft, - judgeResult: { success: false, passedTests: [], failedTests: ["baseline"] }, - }); -} - -test("e2e activity generation (python): 2/4/7 problems (stdout-only)", async (t) => { - const { calls } = installStubs(t, "python"); - - const counts = [2, 4, 7]; - - for (const problem_count of counts) { - await t.test(`count=${problem_count}`, async () => { - calls.length = 0; - - const { sessionId } = createSession("practice"); - const prompt = `Create ${problem_count} easy problems in Python. Topics: strings`; - - const msgRes = await processSessionMessage(sessionId, prompt); - assert.equal(msgRes.accepted, true); - assert.equal(msgRes.done, true); - assert.equal(msgRes.state, "READY"); - assert.equal(msgRes.spec.language, "python"); - assert.equal(msgRes.spec.problem_count, problem_count); - assert.equal(msgRes.spec.problem_style, "stdout"); - - const genRes = await generateFromSession(sessionId); - assert.ok(genRes.activityId); - assert.equal(genRes.problems.length, problem_count); - for (const p of genRes.problems) { - assert.equal(p.language, "python"); - assert.equal("reference_solution" in p, false); - } - - const stored = activityDb.findById(genRes.activityId); - assert.ok(stored); - const storedProblems = JSON.parse(stored.problems); - assert.equal(storedProblems.length, problem_count); - - const session = getSession(sessionId); - assert.equal(session.state, "SAVED"); - }); - } -}); diff --git a/apps/backend/test/integration/languages/sql/activityGenerationE2e.test.js b/apps/backend/test/integration/languages/sql/activityGenerationE2e.test.js deleted file mode 100644 index 02982a3..0000000 --- a/apps/backend/test/integration/languages/sql/activityGenerationE2e.test.js +++ /dev/null @@ -1,106 +0,0 @@ -require("../../../helpers/setupDb"); -const { installGenerationStub } = require("../../../helpers/installGenerationStub"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { activityDb } = require("../../../../src/database"); -const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../../src/services/sessionService"); - -function installStubs(t, language) { - function parseRequestedCountAndTopic(msg) { - const m = String(msg || ""); - const lower = m.toLowerCase(); - const countMatch = lower.match(/\b(\d+)\s+(?:problems?|questions?)\b/); - const count = countMatch ? Number(countMatch[1]) : 1; - const topicsMatch = m.match(/\btopics?\s*:\s*([A-Za-z0-9 _-]+)/i); - const topic = topicsMatch?.[1]?.trim().split(/[,\n]/)[0]?.trim() || "filtering"; - return { count, topic }; - } - - function buildDialogueResponse(latestUserMessage) { - const { count, topic } = parseRequestedCountAndTopic(latestUserMessage); - return { - acknowledgement: "OK", - inferred_intent: "Generate an activity.", - proposedPatch: { - language, - problem_count: count, - difficulty_plan: [{ difficulty: "easy", count }], - topic_tags: [topic], - }, - }; - } - - function sqlDraft(slotIndex) { - const suite = { - schema_sql: "CREATE TABLE t (id INTEGER PRIMARY KEY, v INTEGER);", - cases: Array.from({ length: 8 }, (_, i) => ({ - name: `test_case_${i + 1}`, - seed_sql: `INSERT INTO t (id, v) VALUES (${i + 1}, ${i});`, - expected: { columns: ["v"], rows: [[i]] }, - order_matters: true, - })), - }; - - return { - id: `sql-e2e-${slotIndex}`, - title: `Select V ${slotIndex}`, - description: "Return v for id=1.", - starter_code: "SELECT v FROM t WHERE id = 1 ORDER BY v;", - reference_solution: "SELECT v FROM t WHERE id = 1 ORDER BY v;", - test_suite: JSON.stringify(suite), - constraints: "SQLite 3 (SQL dialect), read-only queries only, deterministic results (explicit ORDER BY when needed).", - sample_inputs: ["t rows: (id=1,v=0)"], - sample_outputs: ["v\\n0"], - difficulty: "easy", - topic_tag: "filtering", - }; - } - - return installGenerationStub(t, { - language, - buildDialogueResponse, - buildDraft: sqlDraft, - judgeResult: { success: false, passedTests: [], failedTests: ["baseline"] }, - }); -} - -test("e2e activity generation (sql): 2/4/7 problems (stdout-only)", async (t) => { - const { calls } = installStubs(t, "sql"); - - const counts = [2, 4, 7]; - - for (const problem_count of counts) { - await t.test(`count=${problem_count}`, async () => { - calls.length = 0; - - const { sessionId } = createSession("practice"); - const prompt = `Create ${problem_count} easy problems in SQL. Topics: filtering`; - - const msgRes = await processSessionMessage(sessionId, prompt); - assert.equal(msgRes.accepted, true); - assert.equal(msgRes.done, true); - assert.equal(msgRes.state, "READY"); - assert.equal(msgRes.spec.language, "sql"); - assert.equal(msgRes.spec.problem_count, problem_count); - assert.equal(msgRes.spec.problem_style, "stdout"); - - const genRes = await generateFromSession(sessionId); - assert.ok(genRes.activityId); - assert.equal(genRes.problems.length, problem_count); - for (const p of genRes.problems) { - assert.equal(p.language, "sql"); - assert.equal("reference_solution" in p, false); - } - - const stored = activityDb.findById(genRes.activityId); - assert.ok(stored); - const storedProblems = JSON.parse(stored.problems); - assert.equal(storedProblems.length, problem_count); - - const session = getSession(sessionId); - assert.equal(session.state, "SAVED"); - }); - } -}); diff --git a/apps/backend/test/integration/llm/realActivityGenerationE2e.anthropic.test.js b/apps/backend/test/integration/llm/realActivityGenerationE2e.anthropic.test.js deleted file mode 100644 index 06cc6fa..0000000 --- a/apps/backend/test/integration/llm/realActivityGenerationE2e.anthropic.test.js +++ /dev/null @@ -1,4 +0,0 @@ -const { registerRealActivityGenerationE2e } = require("./realActivityGenerationE2e.shared"); - -registerRealActivityGenerationE2e({ provider: "anthropic" }); - diff --git a/apps/backend/test/integration/llm/realActivityGenerationE2e.gemini.test.js b/apps/backend/test/integration/llm/realActivityGenerationE2e.gemini.test.js deleted file mode 100644 index 973700f..0000000 --- a/apps/backend/test/integration/llm/realActivityGenerationE2e.gemini.test.js +++ /dev/null @@ -1,4 +0,0 @@ -const { registerRealActivityGenerationE2e } = require("./realActivityGenerationE2e.shared"); - -registerRealActivityGenerationE2e({ provider: "gemini" }); - diff --git a/apps/backend/test/integration/llm/realActivityGenerationE2e.openai.test.js b/apps/backend/test/integration/llm/realActivityGenerationE2e.openai.test.js deleted file mode 100644 index da5f786..0000000 --- a/apps/backend/test/integration/llm/realActivityGenerationE2e.openai.test.js +++ /dev/null @@ -1,4 +0,0 @@ -const { registerRealActivityGenerationE2e } = require("./realActivityGenerationE2e.shared"); - -registerRealActivityGenerationE2e({ provider: "openai" }); - diff --git a/apps/backend/test/integration/llm/realActivityGenerationE2e.shared.js b/apps/backend/test/integration/llm/realActivityGenerationE2e.shared.js index 9a31221..336df31 100644 --- a/apps/backend/test/integration/llm/realActivityGenerationE2e.shared.js +++ b/apps/backend/test/integration/llm/realActivityGenerationE2e.shared.js @@ -1,19 +1,30 @@ require("../../helpers/setupDb"); +require("../../helpers/loadRealProviderAuth").loadRealProviderAuth(); const assert = require("node:assert/strict"); -const crypto = require("node:crypto"); const { execSync } = require("node:child_process"); const { activityDb } = require("../../../src/database"); +const { + generationRunRepository, + generationSlotRunRepository, + generationSlotTransitionRepository, +} = require("../../../src/database/repositories/generationRunRepository"); const { createSession, processSessionMessage, generateFromSession, getSession } = require("../../../src/services/sessionService"); +const RUN_SMOKE = String(process.env.CODEMM_RUN_REAL_PROVIDER_SMOKE || "").trim() === "1"; +const PROVIDER_ORDER = ["openai", "anthropic", "gemini"]; +const DEFAULT_PROVIDER_LANGS = ["java"]; +const DEFAULT_PROVIDER_COUNTS = ["1"]; +const DEFAULT_PROVIDER_STYLES = ["stdout"]; + /** * Real-LLM + Docker matrix runner. * * Defaults: - * - `CODEMM_E2E_LANGS=java,python,cpp,sql` + * - `CODEMM_E2E_LANGS=java` * - `CODEMM_E2E_STYLES=stdout` - * - `CODEMM_E2E_COUNTS=2` + * - `CODEMM_E2E_COUNTS=1` * * This test prints a terminal summary table at the end (even on failure). */ @@ -27,7 +38,16 @@ function parseCsvEnv(name, fallback) { .filter(Boolean); } -function withEnv(t, patch) { +function parseProviderFilter() { + const configured = parseCsvEnv("CODEMM_E2E_PROVIDERS", []); + if (configured.length === 0) return null; + const normalized = configured + .map((provider) => String(provider).trim().toLowerCase()) + .filter((provider) => PROVIDER_ORDER.includes(provider)); + return normalized.length > 0 ? Array.from(new Set(normalized)) : null; +} + +function withPatchedEnv(patch, fn) { const keys = Object.keys(patch); const prev = {}; for (const k of keys) prev[k] = process.env[k]; @@ -37,13 +57,17 @@ function withEnv(t, patch) { else process.env[k] = String(v); } - t.after(() => { + const restore = () => { for (const k of keys) { const v = prev[k]; if (v == null) delete process.env[k]; else process.env[k] = v; } - }); + }; + + return Promise.resolve() + .then(() => fn()) + .finally(restore); } function hasProviderKey(provider) { @@ -53,6 +77,41 @@ function hasProviderKey(provider) { return false; } +function listAvailableProviders() { + return PROVIDER_ORDER.filter((provider) => hasProviderKey(provider)); +} + +function providerKeyPatch(provider, value) { + if (provider === "openai") { + return { + CODEX_API_KEY: value, + OPENAI_API_KEY: value, + }; + } + if (provider === "anthropic") { + return { + ANTHROPIC_API_KEY: value, + }; + } + if (provider === "gemini") { + return { + GEMINI_API_KEY: value, + GOOGLE_API_KEY: value, + }; + } + return {}; +} + +function buildAutoProviderPatch(expectedProvider) { + const patch = { CODEX_PROVIDER: "auto" }; + for (const provider of PROVIDER_ORDER) { + if (provider === expectedProvider) break; + if (!hasProviderKey(provider)) continue; + Object.assign(patch, providerKeyPatch(provider, null)); + } + return patch; +} + function preflightOrThrow() { // These tests run the full generation pipeline, including Docker validation. const requiredImages = ["codem-java-judge", "codem-python-judge", "codem-cpp-judge", "codem-sql-judge"]; @@ -99,15 +158,138 @@ function printMatrixSummary(rows) { ); } -function registerRealActivityGenerationE2e({ provider }) { +function topicForLanguage(language) { + if (language === "java") return "arrays"; + if (language === "python") return "strings"; + if (language === "cpp") return "graphs"; + if (language === "sql") return "filtering"; + return "basics"; +} + +function parseTransitionPayload(payloadJson) { + if (!payloadJson) return null; + try { + return JSON.parse(payloadJson); + } catch { + return null; + } +} + +function collectProvidersForRun(runId) { + const providers = []; + const transitions = generationSlotTransitionRepository.listByRun(runId); + for (const transition of transitions) { + const payload = parseTransitionPayload(transition.payload_json); + if (payload && typeof payload.provider === "string" && payload.provider.trim()) { + providers.push(payload.provider.trim()); + } + } + return Array.from(new Set(providers)); +} + +async function runGenerationCase({ + providerLabel, + envPatch, + expectedProvider, + language, + style, + count, +}) { + assert.ok(Number.isInteger(count) && count >= 1 && count <= 7, "Counts must be in 1..7"); + assert.equal(style, "stdout"); + + return withPatchedEnv( + { + CODEMM_WORKSPACE_GEN: "0", + ...envPatch, + }, + async () => { + const prompt = `Language: ${language}\nStyle: stdout\nTopics: ${topicForLanguage(language)}\nDifficulty: easy:${count}`; + const { sessionId } = createSession("practice"); + const msg = await processSessionMessage(sessionId, prompt); + assert.equal(msg.accepted, true); + assert.equal(msg.done, true); + assert.equal(msg.state, "READY"); + assert.equal(msg.spec.language, language); + assert.equal(msg.spec.problem_count, count); + assert.equal(msg.spec.problem_style, "stdout"); + + const generated = await generateFromSession(sessionId); + assert.ok(generated.activityId); + assert.equal(generated.problems.length, count); + for (const p of generated.problems) { + assert.equal(p.language, language); + assert.equal("reference_solution" in p, false); + assert.equal("reference_workspace" in p, false); + } + + const stored = activityDb.findById(generated.activityId); + assert.ok(stored); + const storedProblems = JSON.parse(stored.problems); + assert.equal(storedProblems.length, count); + + const session = getSession(sessionId); + assert.ok(session.latestGenerationRunId, `Missing latestGenerationRunId for ${providerLabel}`); + assert.equal(session.latestGenerationRunStatus, "COMPLETED"); + + const run = generationRunRepository.findById(session.latestGenerationRunId); + assert.ok(run, "Missing persisted generation run."); + assert.equal(run.status, "COMPLETED"); + assert.equal(run.activity_id, generated.activityId); + assert.equal(run.total_slots, count); + assert.equal(run.completed_slots, count); + assert.equal(run.successful_slots, count); + assert.equal(run.failed_slots, 0); + + const slotRuns = generationSlotRunRepository.listByRun(run.id); + assert.equal(slotRuns.length, count); + for (const slotRun of slotRuns) { + assert.equal(slotRun.status, "SUCCEEDED"); + assert.equal(slotRun.language, language); + } + + const providersUsed = collectProvidersForRun(run.id); + assert.ok(providersUsed.length > 0, "Expected persisted slot transitions to include provider metadata."); + assert.ok( + providersUsed.includes(expectedProvider), + `Expected provider ${expectedProvider} but saw ${providersUsed.join(", ") || "none"}` + ); + assert.equal(providersUsed.length, 1); + + return { + activityId: generated.activityId, + runId: run.id, + providersUsed, + }; + } + ); +} + +function buildProviderTestMatrix(options = {}) { + const languages = parseCsvEnv("CODEMM_E2E_LANGS", options.defaultLanguages ?? DEFAULT_PROVIDER_LANGS); + const styles = Array.from( + new Set(parseCsvEnv("CODEMM_E2E_STYLES", options.defaultStyles ?? DEFAULT_PROVIDER_STYLES).filter((s) => s === "stdout")) + ); + if (styles.length === 0) styles.push("stdout"); + const counts = parseCsvEnv("CODEMM_E2E_COUNTS", options.defaultCounts ?? DEFAULT_PROVIDER_COUNTS).map((s) => Number(s)); + return { languages, styles, counts }; +} + +function registerRealActivityGenerationE2e({ provider, defaultLanguages, defaultCounts, defaultStyles }) { const test = require("node:test"); + const matrix = buildProviderTestMatrix({ defaultLanguages, defaultCounts, defaultStyles }); test( - `e2e (real LLM:${provider}): prompt → dialogue → READY → generateFromSession → activity persisted (stdout-only × 4 langs)`, + `e2e (real activity:${provider}): prompt → READY → generateFromSession → activity persisted`, // This test exercises real LLM calls + real Docker validation across a matrix. // Keep a generous timeout to avoid parent cancellation cascading into many subtest failures. { timeout: 6 * 60 * 60 * 1000 }, async (t) => { + if (!RUN_SMOKE) { + t.skip("Set CODEMM_RUN_REAL_PROVIDER_SMOKE=1 to run real provider E2E tests."); + return; + } + if (!["openai", "anthropic", "gemini"].includes(provider)) { t.skip(`Unknown provider "${provider}"`); return; @@ -118,24 +300,13 @@ function registerRealActivityGenerationE2e({ provider }) { return; } - // Keep behavior stable (workspace mode adds extra variability). - withEnv(t, { CODEMM_WORKSPACE_GEN: "0" }); - withEnv(t, { CODEX_PROVIDER: provider }); - - const languages = parseCsvEnv("CODEMM_E2E_LANGS", ["java", "python", "cpp", "sql"]); - const styles = Array.from( - new Set(parseCsvEnv("CODEMM_E2E_STYLES", ["stdout"]).filter((s) => s === "stdout")) - ); - if (styles.length === 0) styles.push("stdout"); - const counts = parseCsvEnv("CODEMM_E2E_COUNTS", ["2"]).map((s) => Number(s)); - preflightOrThrow(); const summaryRows = []; try { - for (const language of languages) { - for (const style of styles) { - for (const count of counts) { + for (const language of matrix.languages) { + for (const style of matrix.styles) { + for (const count of matrix.counts) { const label = `${provider} ${language} style=${style} count=${count}`; const row = { provider, @@ -154,47 +325,15 @@ function registerRealActivityGenerationE2e({ provider }) { const startedAt = Date.now(); try { await t.test(label, { timeout: 90 * 60 * 1000 }, async () => { - assert.ok(Number.isInteger(count) && count >= 1 && count <= 7, "Counts must be in 1..7"); - - const topic = - language === "java" - ? "arrays" - : language === "python" - ? "strings" - : language === "cpp" - ? "graphs" - : "filtering"; - - // Make it 1-turn READY by providing explicit problem_count + difficulty plan. - // difficultyPlanParser will deterministically set difficulty_plan and problem_count from "easy:N". - const prompt = `Language: ${language}\nStyle: stdout\nTopics: ${topic}\nDifficulty: easy:${count}`; - - const { sessionId } = createSession("practice"); - const msg = await processSessionMessage(sessionId, prompt); - assert.equal(msg.accepted, true); - assert.equal(msg.done, true); - assert.equal(msg.state, "READY"); - assert.equal(msg.spec.language, language); - assert.equal(msg.spec.problem_count, count); - assert.equal(msg.spec.problem_style, "stdout"); - - const generated = await generateFromSession(sessionId); - row.activityId = generated.activityId; - assert.ok(generated.activityId); - assert.equal(generated.problems.length, count); - for (const p of generated.problems) { - assert.equal(p.language, language); - assert.equal("reference_solution" in p, false); - assert.equal("reference_workspace" in p, false); - } - - const stored = activityDb.findById(generated.activityId); - assert.ok(stored); - const storedProblems = JSON.parse(stored.problems); - assert.equal(storedProblems.length, count); - - const s = getSession(sessionId); - assert.equal(s.state, "SAVED"); + const result = await runGenerationCase({ + providerLabel: provider, + envPatch: { CODEX_PROVIDER: provider }, + expectedProvider: provider, + language, + style, + count, + }); + row.activityId = result.activityId; }); row.status = "PASS"; @@ -218,4 +357,111 @@ function registerRealActivityGenerationE2e({ provider }) { ); } -module.exports = { registerRealActivityGenerationE2e }; +function registerRealActivityGenerationAllProvidersE2e(options = {}) { + const providerFilter = parseProviderFilter(); + const providers = providerFilter ?? PROVIDER_ORDER; + for (const provider of providers) { + registerRealActivityGenerationE2e({ + provider, + defaultLanguages: options.defaultLanguages, + defaultCounts: options.defaultCounts, + defaultStyles: options.defaultStyles, + }); + } +} + +function registerRealActivityGenerationAutoFallbackE2e(options = {}) { + const test = require("node:test"); + const providerFilter = parseProviderFilter(); + const matrix = buildProviderTestMatrix({ + defaultLanguages: options.defaultLanguages ?? ["java"], + defaultCounts: options.defaultCounts ?? ["1"], + defaultStyles: options.defaultStyles ?? ["stdout"], + }); + + test( + "e2e (real activity:auto): generates with the highest-priority available provider and falls back as providers are masked", + { timeout: 4 * 60 * 60 * 1000 }, + async (t) => { + if (!RUN_SMOKE) { + t.skip("Set CODEMM_RUN_REAL_PROVIDER_SMOKE=1 to run real provider E2E tests."); + return; + } + + if (providerFilter && providerFilter.length <= 1) { + t.skip("Auto-provider fallback coverage is skipped when CODEMM_E2E_PROVIDERS narrows execution to one provider."); + return; + } + + const availableProviders = listAvailableProviders(); + if (availableProviders.length === 0) { + t.skip("No real providers configured for auto-mode fallback coverage."); + return; + } + + preflightOrThrow(); + + const language = matrix.languages[0] ?? "java"; + const style = matrix.styles[0] ?? "stdout"; + const count = matrix.counts[0] ?? 1; + const summaryRows = []; + + try { + for (const expectedProvider of availableProviders) { + const maskedProviders = PROVIDER_ORDER.filter( + (provider) => provider !== expectedProvider && hasProviderKey(provider) && PROVIDER_ORDER.indexOf(provider) < PROVIDER_ORDER.indexOf(expectedProvider) + ); + const label = + maskedProviders.length > 0 + ? `auto falls back to ${expectedProvider} when ${maskedProviders.join(", ")} are unavailable` + : `auto selects ${expectedProvider} when it is the highest-priority configured provider`; + const row = { + provider: `auto->${expectedProvider}`, + language, + style, + count, + status: "RUNNING", + durationMs: 0, + activityId: undefined, + failureKind: undefined, + slotIndex: undefined, + error: undefined, + }; + summaryRows.push(row); + + const startedAt = Date.now(); + try { + await t.test(label, { timeout: 90 * 60 * 1000 }, async () => { + const result = await runGenerationCase({ + providerLabel: `auto:${expectedProvider}`, + envPatch: buildAutoProviderPatch(expectedProvider), + expectedProvider, + language, + style, + count, + }); + row.activityId = result.activityId; + }); + row.status = "PASS"; + } catch (err) { + row.status = "FAIL"; + row.failureKind = err?.kind; + row.slotIndex = err?.slotIndex; + row.error = truncateOneLine(err?.message ?? err, 160); + throw err; + } finally { + row.durationMs = Date.now() - startedAt; + } + } + } finally { + printMatrixSummary(summaryRows); + } + } + ); +} + +module.exports = { + registerRealActivityGenerationE2e, + registerRealActivityGenerationAllProvidersE2e, + registerRealActivityGenerationAutoFallbackE2e, +}; diff --git a/apps/backend/test/integration/llm/realActivityGenerationE2e.test.js b/apps/backend/test/integration/llm/realActivityGenerationE2e.test.js new file mode 100644 index 0000000..a172354 --- /dev/null +++ b/apps/backend/test/integration/llm/realActivityGenerationE2e.test.js @@ -0,0 +1,16 @@ +const { + registerRealActivityGenerationAllProvidersE2e, + registerRealActivityGenerationAutoFallbackE2e, +} = require("./realActivityGenerationE2e.shared"); + +registerRealActivityGenerationAllProvidersE2e({ + defaultLanguages: ["java"], + defaultCounts: ["1"], + defaultStyles: ["stdout"], +}); + +registerRealActivityGenerationAutoFallbackE2e({ + defaultLanguages: ["java"], + defaultCounts: ["1"], + defaultStyles: ["stdout"], +}); diff --git a/apps/backend/test/integration/llm/realProviderIntegrations.test.js b/apps/backend/test/integration/llm/realProviderIntegrations.test.js index 4b8dd7b..59f57ee 100644 --- a/apps/backend/test/integration/llm/realProviderIntegrations.test.js +++ b/apps/backend/test/integration/llm/realProviderIntegrations.test.js @@ -1,11 +1,8 @@ require("../../helpers/setupBase"); +require("../../helpers/loadRealProviderAuth").loadRealProviderAuth(); // This integration test intentionally hits real provider APIs (token-costing). // It skips automatically when a provider key is not present. -require("dotenv").config({ - path: require("node:path").resolve(__dirname, "../../../.env"), - quiet: true, -}); const test = require("node:test"); const assert = require("node:assert/strict"); @@ -38,6 +35,40 @@ function extractText(out) { return blocks.map((b) => (b && b.type === "text" ? String(b.text || "") : "")).join("\n"); } +test( + "llm (real smoke): OpenAI completion works (skips if CODEX_API_KEY/OPENAI_API_KEY missing)", + { timeout: 60_000 }, + async (t) => { + if (!RUN_SMOKE) { + t.skip("Set CODEMM_RUN_REAL_PROVIDER_SMOKE=1 to run real provider smoke tests."); + return; + } + + if (!process.env.CODEX_API_KEY && !process.env.OPENAI_API_KEY) { + t.skip("CODEX_API_KEY/OPENAI_API_KEY not set"); + return; + } + + withEnv(t, { + CODEX_PROVIDER: "openai", + ANTHROPIC_API_KEY: null, + GEMINI_API_KEY: null, + GOOGLE_API_KEY: null, + }); + + const out = await createCodemmCompletion({ + system: 'Reply with exactly "OK". No other text.', + user: "ping", + temperature: 0, + maxTokens: 20, + }); + + const text = extractText(out).trim(); + assert.ok(text.length > 0); + assert.ok(/^ok\b/i.test(text), `Unexpected response: ${JSON.stringify(text)}`); + } +); + test( "llm (real): Anthropic completion works (skips if ANTHROPIC_API_KEY missing)", { timeout: 60_000 }, diff --git a/apps/backend/test/unit/generation/constraintLockRetry.test.js b/apps/backend/test/unit/generation/constraintLockRetry.test.js deleted file mode 100644 index 86c98f8..0000000 --- a/apps/backend/test/unit/generation/constraintLockRetry.test.js +++ /dev/null @@ -1,77 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateProblemsFromPlan } = require("../../../src/generation"); - -function installPythonGeneratorStub(t, drafts) { - let n = 0; - - return { - generateSingleProblem: async () => { - const payload = drafts[Math.min(n, drafts.length - 1)]; - n++; - return { draft: payload, meta: { llmOutputHash: `stub-${n}` } }; - }, - getCalls: () => n, - }; -} - -test("generation: constraint mismatch triggers retry and can recover", async (t) => { - const slotConstraints = - "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced."; - - const bad = { - language: "python", - id: "py-bad-1", - title: "Echo", - description: "Return the input number.", - starter_code: "def solve(x):\n # TODO\n raise NotImplementedError\n", - reference_solution: "def solve(x):\n return x\n", - test_suite: `import pytest -from solution import solve - -def test_case_1(): assert solve(1) == 1 -def test_case_2(): assert solve(2) == 2 -def test_case_3(): assert solve(3) == 3 -def test_case_4(): assert solve(4) == 4 -def test_case_5(): assert solve(5) == 5 -def test_case_6(): assert solve(6) == 6 -def test_case_7(): assert solve(7) == 7 -def test_case_8(): assert solve(8) == 8 -`, - constraints: "WRONG", - sample_inputs: ["x=1"], - sample_outputs: ["1"], - difficulty: "hard", - topic_tag: "arrays", - }; - - const good = { ...bad, id: "py-good-1", constraints: slotConstraints }; - - const { generateSingleProblem, getCalls } = installPythonGeneratorStub(t, [bad, good]); - - const plan = [ - { - index: 0, - language: "python", - difficulty: "hard", - topics: ["arrays"], - problem_style: "return", - constraints: slotConstraints, - test_case_count: 8, - }, - ]; - - const result = await generateProblemsFromPlan(plan, { - deps: { - generateSingleProblem, - validateReferenceSolution: async () => {}, - runTestStrengthGate: async () => {}, - }, - }); - - assert.equal(result.problems.length, 1); - assert.equal(getCalls(), 2); -}); diff --git a/apps/backend/test/unit/generation/constraintLocking.test.js b/apps/backend/test/unit/generation/constraintLocking.test.js deleted file mode 100644 index 454a486..0000000 --- a/apps/backend/test/unit/generation/constraintLocking.test.js +++ /dev/null @@ -1,80 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateSingleProblem } = require("../../../src/generation/perSlotGenerator"); - -function installPythonGeneratorStub(t, drafts) { - const codex = require("../../../src/infra/llm/codemmProvider"); - const originalCreateCodemm = codex.createCodemmCompletion; - const originalCreateCodex = codex.createCodexCompletion; - let n = 0; - - const stub = async ({ system }) => { - if (String(system).includes("Python problem generator")) { - const payload = drafts[Math.min(n, drafts.length - 1)]; - n++; - return { content: [{ type: "text", text: JSON.stringify(payload) }] }; - } - throw new Error(`Unexpected LLM call in test (system=${String(system).slice(0, 80)})`); - }; - - codex.createCodemmCompletion = stub; - codex.createCodexCompletion = stub; - - t.after(() => { - codex.createCodemmCompletion = originalCreateCodemm; - codex.createCodexCompletion = originalCreateCodex; - }); - - return { getCalls: () => n }; -} - -test("constraints locking: mismatched constraints triggers contract failure", async (t) => { - const slot = { - index: 0, - language: "python", - difficulty: "hard", - topics: ["arrays"], - problem_style: "return", - constraints: "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced.", - test_case_count: 8, - }; - - const bad = { - id: "py-bad-1", - title: "Echo", - description: "Return the input number.", - starter_code: "def solve(x):\n # TODO\n raise NotImplementedError\n", - reference_solution: "def solve(x):\n return x\n", - test_suite: `import pytest -from solution import solve - -def test_case_1(): assert solve(1) == 1 -def test_case_2(): assert solve(2) == 2 -def test_case_3(): assert solve(3) == 3 -def test_case_4(): assert solve(4) == 4 -def test_case_5(): assert solve(5) == 5 -def test_case_6(): assert solve(6) == 6 -def test_case_7(): assert solve(7) == 7 -def test_case_8(): assert solve(8) == 8 -`, - constraints: "WRONG", - sample_inputs: ["x=1"], - sample_outputs: ["1"], - difficulty: "hard", - topic_tag: "arrays", - }; - - installPythonGeneratorStub(t, [bad]); - - await assert.rejects( - () => generateSingleProblem(slot), - (e) => { - assert.match(String(e && e.message), /Invalid constraints/i); - return true; - } - ); -}); - diff --git a/apps/backend/test/unit/generation/cppStarterSynthesis.test.js b/apps/backend/test/unit/generation/cppStarterSynthesis.test.js index df55362..2999fdb 100644 --- a/apps/backend/test/unit/generation/cppStarterSynthesis.test.js +++ b/apps/backend/test/unit/generation/cppStarterSynthesis.test.js @@ -3,7 +3,7 @@ require("../../helpers/setupBase"); const test = require("node:test"); const assert = require("node:assert/strict"); -const { __test__ } = require("../../../src/generation/perSlotGenerator"); +const { __test__ } = require("../../../src/pipeline/slotStages"); test("cpp starter synthesis: strips comments before checking for solve()", () => { const starter = `#include @@ -25,10 +25,7 @@ long long solve(int n, const std::vector>& edges) { const signature = __test__.extractCppSolveSignature(reference); assert.equal(signature, "long long solve(int n, const std::vector>& edges)"); - const starter = __test__.synthesizeCppStarterCodeFromReference({ - referenceSolution: reference, - fallbackTopic: "minimum spanning tree", - }); + const starter = __test__.deriveCppStarter(reference, "minimum spanning tree"); assert.ok(starter); assert.match(starter, /long long solve\s*\(/); assert.match(starter, /throw std::runtime_error\("TODO"\);/); diff --git a/apps/backend/test/unit/generation/executionBundle.test.js b/apps/backend/test/unit/generation/executionBundle.test.js new file mode 100644 index 0000000..a05c75c --- /dev/null +++ b/apps/backend/test/unit/generation/executionBundle.test.js @@ -0,0 +1,164 @@ +require("../../helpers/setupBase"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { + buildValidatedExecutionBundle, + ExecutionBundleValidationError, +} = require("../../../src/generation/services/executionBundle"); + +function makeSlot(overrides = {}) { + return { + index: 0, + language: "python", + difficulty: "easy", + topics: ["functions"], + problem_style: "return", + constraints: "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced.", + test_case_count: 8, + ...overrides, + }; +} + +test("execution bundle rejects python stdin-driven references before judge execution", () => { + const slot = makeSlot(); + const draft = { + language: "python", + id: "py-1", + title: "Add One", + description: "Return x + 1.", + starter_code: "def solve(x):\n raise NotImplementedError\n", + reference_solution: "def solve(x):\n value = input()\n return int(value) + 1\n", + test_suite: [ + "import pytest", + "from solution import solve", + "def test_case_1(): assert solve(1) == 2", + "def test_case_2(): assert solve(2) == 3", + "def test_case_3(): assert solve(3) == 4", + "def test_case_4(): assert solve(4) == 5", + "def test_case_5(): assert solve(5) == 6", + "def test_case_6(): assert solve(6) == 7", + "def test_case_7(): assert solve(7) == 8", + "def test_case_8(): assert solve(8) == 9", + ].join("\n"), + constraints: slot.constraints, + sample_inputs: ["1"], + sample_outputs: ["2"], + difficulty: "easy", + topic_tag: "functions", + }; + + assert.throws( + () => buildValidatedExecutionBundle({ slot, draft }), + (error) => { + assert.ok(error instanceof ExecutionBundleValidationError); + assert.equal(error.kind, "static_rule_violation"); + assert.match(error.message, /stdin/i); + return true; + } + ); +}); + +test("execution bundle rejects obvious non-terminating loops before judge execution", () => { + const slot = makeSlot({ + language: "cpp", + topics: ["loops"], + constraints: "C++20, g++ (GNU), standard library only, no filesystem access, no networking, deterministic behavior.", + }); + const draft = { + language: "cpp", + id: "cpp-1", + title: "Loop Forever", + description: "Count values.", + starter_code: "#include \nint solve() { return 0; }\n", + reference_solution: "#include \nint solve() { while (true) {} return 0; }\n", + test_suite: [ + '#include "solution.cpp"', + "#include ", + "#define RUN_TEST(name, ...) do { bool ok = (__VA_ARGS__); std::cout << (ok ? \"[PASS] \" : \"[FAIL] \") << name << \"\\n\"; } while (0)", + "bool test_case_1() { return solve() == 0; }", + "bool test_case_2() { return solve() == 0; }", + "bool test_case_3() { return solve() == 0; }", + "bool test_case_4() { return solve() == 0; }", + "bool test_case_5() { return solve() == 0; }", + "bool test_case_6() { return solve() == 0; }", + "bool test_case_7() { return solve() == 0; }", + "bool test_case_8() { return solve() == 0; }", + "int main() {", + ' RUN_TEST("test_case_1", test_case_1());', + ' RUN_TEST("test_case_2", test_case_2());', + ' RUN_TEST("test_case_3", test_case_3());', + ' RUN_TEST("test_case_4", test_case_4());', + ' RUN_TEST("test_case_5", test_case_5());', + ' RUN_TEST("test_case_6", test_case_6());', + ' RUN_TEST("test_case_7", test_case_7());', + ' RUN_TEST("test_case_8", test_case_8());', + "}", + ].join("\n"), + constraints: slot.constraints, + sample_inputs: ["0"], + sample_outputs: ["0"], + difficulty: "easy", + topic_tag: "loops", + }; + + assert.throws( + () => buildValidatedExecutionBundle({ slot, draft }), + (error) => { + assert.ok(error instanceof ExecutionBundleValidationError); + assert.equal(error.kind, "complexity_risk_exceeded"); + assert.match(error.message, /unbounded loop/i); + return true; + } + ); +}); + +test("execution bundle builds normalized hashes and budget profile for a valid java draft", () => { + const slot = makeSlot({ + language: "java", + topics: ["encapsulation"], + problem_style: "stdout", + constraints: "Java 21, JUnit 5, standard library only.", + }); + const draft = { + language: "java", + id: "java-1", + title: "Encapsulated Counter", + description: "Print the counter value.", + starter_code: "public class Counter { }\n", + reference_solution: [ + "public class Counter {", + " private int value;", + " public void increment() { value++; }", + " public int getValue() { return value; }", + " public void printValue() { System.out.println(value); }", + "}", + ].join("\n"), + test_suite: [ + "import org.junit.jupiter.api.Test;", + "import static org.junit.jupiter.api.Assertions.*;", + "import java.io.*;", + "public class CounterTest {", + " @Test void test_case_1() {", + " Counter counter = new Counter();", + " ByteArrayOutputStream out = new ByteArrayOutputStream();", + " System.setOut(new PrintStream(out));", + " counter.printValue();", + " assertEquals(\"0\", out.toString().trim());", + " }", + "}", + ].join("\n"), + constraints: slot.constraints, + sample_inputs: ["sample"], + sample_outputs: ["0"], + difficulty: "easy", + topic_tag: "encapsulation", + }; + + const bundle = buildValidatedExecutionBundle({ slot, draft }); + assert.equal(bundle.language, "java"); + assert.equal(typeof bundle.bundleHash, "string"); + assert.equal(typeof bundle.artifactHashes.reference, "string"); + assert.equal(typeof bundle.executionBudgetProfile.overallTimeoutMs, "number"); +}); diff --git a/apps/backend/test/unit/generation/generationMultilangPipeline.test.js b/apps/backend/test/unit/generation/generationMultilangPipeline.test.js index 41c12d9..a6fa383 100644 --- a/apps/backend/test/unit/generation/generationMultilangPipeline.test.js +++ b/apps/backend/test/unit/generation/generationMultilangPipeline.test.js @@ -41,9 +41,10 @@ test("generation: pipeline can finalize multiple languages (reference artifacts }, ]; - const generateSingleProblem = async (slot) => { + const runSlotPipeline = async ({ slot }) => { if (slot.language === "cpp") { return { + envelope: {}, draft: { language: "cpp", id: "cpp-smoke-1", @@ -79,12 +80,13 @@ int main() { difficulty: slot.difficulty, topic_tag: slot.topics[0], }, - meta: { llmOutputHash: "stub" }, + meta: { llmOutputHash: "stub", promptTemplateId: "test", routePlan: null }, }; } if (slot.language === "python") { return { + envelope: {}, draft: { language: "python", id: "py-smoke-1", @@ -110,7 +112,7 @@ def test_case_8(): assert solve("x" * 20) == 20 difficulty: slot.difficulty, topic_tag: slot.topics[0], }, - meta: { llmOutputHash: "stub" }, + meta: { llmOutputHash: "stub", promptTemplateId: "test", routePlan: null }, }; } @@ -125,6 +127,7 @@ def test_case_8(): assert solve("x" * 20) == 20 })), }; return { + envelope: {}, draft: { language: "sql", id: "sql-smoke-1", @@ -139,12 +142,13 @@ def test_case_8(): assert solve("x" * 20) == 20 difficulty: slot.difficulty, topic_tag: slot.topics[0], }, - meta: { llmOutputHash: "stub" }, + meta: { llmOutputHash: "stub", promptTemplateId: "test", routePlan: null }, }; } // java return { + envelope: {}, draft: { language: "java", id: "java-smoke-1", @@ -188,21 +192,15 @@ public class SumArrayTest { difficulty: slot.difficulty, topic_tag: slot.topics[0], }, - meta: { llmOutputHash: "stub" }, + meta: { llmOutputHash: "stub", promptTemplateId: "test", routePlan: null }, }; }; - let validateCalls = 0; - const validateReferenceSolution = async () => { - validateCalls++; - }; - const result = await generateProblemsFromPlan(plan, { - deps: { generateSingleProblem, validateReferenceSolution, runTestStrengthGate: async () => {} }, + deps: { runSlotPipeline }, }); assert.equal(result.problems.length, 4); - assert.equal(validateCalls, 4); for (const p of result.problems) { assert.equal("reference_solution" in p, false); assert.equal("reference_workspace" in p, false); diff --git a/apps/backend/test/unit/generation/generationProgressEvents.test.js b/apps/backend/test/unit/generation/generationProgressEvents.test.js deleted file mode 100644 index eed3c52..0000000 --- a/apps/backend/test/unit/generation/generationProgressEvents.test.js +++ /dev/null @@ -1,63 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateProblemsFromPlan } = require("../../../src/generation"); - -test("generation progress: emits per-slot event ordering", async () => { - const events = []; - - const plan = [ - { - index: 0, - difficulty: "easy", - topics: ["arrays"], - language: "java", - problem_style: "return", - constraints: "No extra constraints.", - test_case_count: 8, - }, - ]; - - const draft = { - language: "java", - id: "p1", - title: "Example", - description: "Example description.", - constraints: plan[0].constraints, - sample_inputs: [], - sample_outputs: [], - difficulty: "easy", - topic_tag: "arrays", - test_suite: "class Dummy {}", - starter_code: "class Solution {}", - reference_solution: "class Solution {}", - }; - - await generateProblemsFromPlan(plan, { - onProgress: (ev) => events.push(ev), - deps: { - generateSingleProblem: async () => ({ draft, meta: { llmOutputHash: "x" } }), - validateReferenceSolution: async () => {}, - runTestStrengthGate: async () => {}, - }, - }); - - assert.deepEqual( - events.map((e) => e.type), - [ - "slot_started", - "problem_started", - "slot_llm_attempt_started", - "attempt_started", - "slot_contract_validated", - "slot_evidence", - "slot_docker_validation_started", - "validation_started", - "slot_attempt_summary", - "slot_completed", - "problem_validated", - ] - ); -}); diff --git a/apps/backend/test/unit/generation/generationRetryLimits.test.js b/apps/backend/test/unit/generation/generationRetryLimits.test.js deleted file mode 100644 index 08c2253..0000000 --- a/apps/backend/test/unit/generation/generationRetryLimits.test.js +++ /dev/null @@ -1,140 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateProblemsFromPlan } = require("../../../src/generation"); - -function mkCppDraft(slot) { - return { - draft: { - language: "cpp", - id: `cpp-${slot.index}`, - title: "Retry Limit Stub", - description: "stub", - starter_code: "#include \n\nint solve() { return 0; }\n", - reference_solution: "#include \n\nint solve() { return 0; }\n", - test_suite: `#include -#include "solution.cpp" -static int __codem_failures = 0; -#define RUN_TEST(name, ...) do { \\ - try { __VA_ARGS__; std::cout << "[PASS] " << (name) << "\\n"; } \\ - catch (const std::exception&) { std::cout << "[FAIL] " << (name) << "\\n"; __codem_failures++; } \\ - catch (...) { std::cout << "[FAIL] " << (name) << "\\n"; __codem_failures++; } \\ -} while (0) -int main() { - RUN_TEST("test_case_1", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_2", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_3", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_4", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_5", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_6", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_7", { if (solve() != 0) throw std::runtime_error("fail"); }); - RUN_TEST("test_case_8", { if (solve() != 0) throw std::runtime_error("fail"); }); - return __codem_failures ? 1 : 0; -} -`, - constraints: slot.constraints, - sample_inputs: [], - sample_outputs: [], - difficulty: slot.difficulty, - topic_tag: slot.topics[0], - }, - meta: { llmOutputHash: "stub" }, - }; -} - -function mkPythonDraft(slot) { - return { - draft: { - language: "python", - id: `py-${slot.index}`, - title: "Retry Limit Stub", - description: "stub", - starter_code: "def solve():\n return 0\n", - reference_solution: "def solve():\n return 0\n", - test_suite: `import pytest -from solution import solve - -def test_case_1(): assert solve() == 0 -def test_case_2(): assert solve() == 0 -def test_case_3(): assert solve() == 0 -def test_case_4(): assert solve() == 0 -def test_case_5(): assert solve() == 0 -def test_case_6(): assert solve() == 0 -def test_case_7(): assert solve() == 0 -def test_case_8(): assert solve() == 0 -`, - constraints: slot.constraints, - sample_inputs: [], - sample_outputs: [], - difficulty: slot.difficulty, - topic_tag: slot.topics[0], - }, - meta: { llmOutputHash: "stub" }, - }; -} - -test("generation: retries C++ up to 3 attempts", async () => { - const plan = [ - { - index: 0, - language: "cpp", - difficulty: "easy", - topics: ["graphs"], - problem_style: "return", - constraints: "C++20, g++ (GNU), standard library only.", - }, - ]; - - let generateCalls = 0; - const generateSingleProblem = async (slot) => { - generateCalls++; - return mkCppDraft(slot); - }; - - let validateCalls = 0; - const validateReferenceSolution = async () => { - validateCalls++; - throw new Error("fail"); - }; - - await assert.rejects(() => - generateProblemsFromPlan(plan, { deps: { generateSingleProblem, validateReferenceSolution } }) - ); - - assert.equal(generateCalls, 3); - assert.equal(validateCalls, 3); -}); - -test("generation: retries non-C++ up to 3 attempts", async () => { - const plan = [ - { - index: 0, - language: "python", - difficulty: "easy", - topics: ["strings"], - problem_style: "return", - constraints: "Python 3.11, deterministic.", - }, - ]; - - let generateCalls = 0; - const generateSingleProblem = async (slot) => { - generateCalls++; - return mkPythonDraft(slot); - }; - - let validateCalls = 0; - const validateReferenceSolution = async () => { - validateCalls++; - throw new Error("fail"); - }; - - await assert.rejects(() => - generateProblemsFromPlan(plan, { deps: { generateSingleProblem, validateReferenceSolution } }) - ); - - assert.equal(generateCalls, 3); - assert.equal(validateCalls, 3); -}); diff --git a/apps/backend/test/unit/generation/guidedScaffoldingDecay.test.js b/apps/backend/test/unit/generation/guidedScaffoldingDecay.test.js index 9ab4f84..ee19eaf 100644 --- a/apps/backend/test/unit/generation/guidedScaffoldingDecay.test.js +++ b/apps/backend/test/unit/generation/guidedScaffoldingDecay.test.js @@ -77,9 +77,11 @@ public class GraphAlgo { let n = 0; const { problems } = await generateProblemsFromPlan(plan, { deps: { - generateSingleProblem: async () => ({ draft: { ...baseDraft, id: `p${n++}` }, meta: { llmOutputHash: "x" } }), - validateReferenceSolution: async () => {}, - runTestStrengthGate: async () => {}, + runSlotPipeline: async () => ({ + envelope: {}, + draft: { ...baseDraft, id: `p${n++}` }, + meta: { llmOutputHash: "x", promptTemplateId: "test", routePlan: null }, + }), }, }); diff --git a/apps/backend/test/unit/generation/javaStructuralRetry.test.js b/apps/backend/test/unit/generation/javaStructuralRetry.test.js deleted file mode 100644 index 183d53c..0000000 --- a/apps/backend/test/unit/generation/javaStructuralRetry.test.js +++ /dev/null @@ -1,174 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateProblemsFromPlan } = require("../../../src/generation"); - -function installJavaGeneratorStub(t) { - let n = 0; - - const invalidDraft = { - language: "java", - id: "java-bad-1", - title: "Billing", - description: "Compute billing cost.", - starter_code: ` -public class Billing { - public void solve(String plan, int minutes) { - // TODO - System.out.println(0); - } -} -`.trim(), - reference_solution: ` -public class Billing { - public void solve(String plan, int minutes) { - System.out.println(minutes); - } -} -`.trim(), - test_suite: ` -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import java.io.*; - -public class BillingTest { - private String run(String plan, int minutes) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - PrintStream prev = System.out; - System.setOut(new PrintStream(out)); - try { new Billing().solve(plan, minutes); } - finally { System.setOut(prev); } - return out.toString().trim(); - } - - @Test void test_case_1(){ assertEquals("3", run("basic", 3)); } - @Test void test_case_2(){ assertEquals("3", run("premium", 3)); } - @Test void test_case_3(){ assertEquals("0", run("basic", 0)); } - @Test void test_case_4(){ assertEquals("0", run("premium", 0)); } - @Test void test_case_5(){ assertEquals("1", run("basic", 1)); } - @Test void test_case_6(){ assertEquals("1", run("premium", 1)); } - @Test void test_case_7(){ assertEquals("2", run("basic", 2)); } - @Test void test_case_8(){ assertEquals("2", run("premium", 2)); } -} -`.trim(), - constraints: "Java 17, JUnit 5, no package declarations.", - sample_inputs: ["plan=basic, minutes=3"], - sample_outputs: ["3"], - difficulty: "hard", - topic_tag: "polymorphism", - }; - - const validDraft = { - language: "java", - id: "java-good-1", - title: "Billing", - description: "Compute billing cost.", - starter_code: ` -public class Billing { - public void solve(String plan, int minutes) { - // TODO - System.out.println(0); - } -} - -interface PricingPlan { - int cost(int minutes); -} - -class BasicPlan implements PricingPlan { - public int cost(int minutes) { return 0; } -} - -class PremiumPlan implements PricingPlan { - public int cost(int minutes) { return 0; } -} -`.trim(), - reference_solution: ` -public class Billing { - public void solve(String plan, int minutes) { - PricingPlan p = plan.equals("premium") ? new PremiumPlan() : new BasicPlan(); - System.out.println(p.cost(minutes)); - } -} - -interface PricingPlan { - int cost(int minutes); -} - -class BasicPlan implements PricingPlan { - public int cost(int minutes) { return minutes; } -} - -class PremiumPlan implements PricingPlan { - public int cost(int minutes) { return minutes * 2; } -} -`.trim(), - test_suite: ` -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import java.io.*; - -public class BillingTest { - @Test void test_case_1(){ PricingPlan p = new BasicPlan(); assertEquals(3, p.cost(3)); } - @Test void test_case_2(){ PricingPlan p = new PremiumPlan(); assertEquals(6, p.cost(3)); } - private String run(String plan, int minutes) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - PrintStream prev = System.out; - System.setOut(new PrintStream(out)); - try { new Billing().solve(plan, minutes); } - finally { System.setOut(prev); } - return out.toString().trim(); - } - - @Test void test_case_3(){ assertEquals("3", run("basic", 3)); } - @Test void test_case_4(){ assertEquals("6", run("premium", 3)); } - @Test void test_case_5(){ assertEquals("0", run("basic", 0)); } - @Test void test_case_6(){ assertEquals("0", run("premium", 0)); } - @Test void test_case_7(){ assertEquals("1", run("basic", 1)); } - @Test void test_case_8(){ assertEquals("2", run("premium", 1)); } -} -`.trim(), - constraints: "Java 17, JUnit 5, no package declarations.", - sample_inputs: ["plan=basic, minutes=3"], - sample_outputs: ["3"], - difficulty: "hard", - topic_tag: "polymorphism", - }; - - return { - generateSingleProblem: async () => { - const payload = n++ === 0 ? invalidDraft : validDraft; - return { draft: payload, meta: { llmOutputHash: `stub-${n}` } }; - }, - getCalls: () => n, - }; -} - -test("generation: java structural topic violation triggers retry and can recover", async (t) => { - const { generateSingleProblem, getCalls } = installJavaGeneratorStub(t); - - const plan = [ - { - index: 0, - language: "java", - difficulty: "hard", - topics: ["polymorphism"], - problem_style: "stdout", - constraints: "Java 17, JUnit 5, no package declarations.", - test_case_count: 8, - }, - ]; - - const result = await generateProblemsFromPlan(plan, { - deps: { - generateSingleProblem, - validateReferenceSolution: async () => {}, - runTestStrengthGate: async () => {}, - }, - }); - - assert.equal(result.problems.length, 1); - assert.equal(getCalls(), 2); -}); diff --git a/apps/backend/test/unit/generation/orchestratorResume.test.js b/apps/backend/test/unit/generation/orchestratorResume.test.js new file mode 100644 index 0000000..611c319 --- /dev/null +++ b/apps/backend/test/unit/generation/orchestratorResume.test.js @@ -0,0 +1,119 @@ +require("../../helpers/setupBase"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { generateProblemsFromPlan } = require("../../../src/generation"); + +function makePlan(count = 3) { + return Array.from({ length: count }, (_, index) => ({ + index, + language: "python", + difficulty: "easy", + topics: [`topic-${index}`], + problem_style: "return", + constraints: "Python 3.11, deterministic.", + })); +} + +function makeDraft(slotIndex) { + return { + language: "python", + id: `problem-${slotIndex}`, + title: `Problem ${slotIndex}`, + description: `Solve slot ${slotIndex}.`, + starter_code: "def solve(x):\n return x\n", + reference_solution: `def solve(x):\n return ${slotIndex}\n`, + test_suite: "from solution import solve\n\ndef test_case_1():\n assert solve(1) == 1\n", + constraints: "Python 3.11, deterministic.", + sample_inputs: [], + sample_outputs: [], + difficulty: "easy", + topic_tag: `topic-${slotIndex}`, + }; +} + +test("generation: targeted rerun preserves successful slots and repairs only failed slot indexes", async () => { + const plan = makePlan(3); + const resumeProblems = [ + { ...makeDraft(0), reference_solution: undefined }, + { ...makeDraft(2), reference_solution: undefined }, + ]; + const resumeOutcomes = [ + { slotIndex: 0, success: true, status: "SUCCEEDED", retries: 0 }, + { + slotIndex: 1, + success: false, + status: "RETRYABLE_FAILURE", + retries: 0, + failureKind: "time_budget_exceeded", + failureCode: "TIME_BUDGET_EXCEEDED", + message: "timed out", + }, + { slotIndex: 2, success: true, status: "SUCCEEDED", retries: 0 }, + ]; + + const visited = []; + const result = await generateProblemsFromPlan(plan, { + resume: { problems: resumeProblems, outcomes: resumeOutcomes }, + targetSlotIndexes: [1], + deps: { + runSlotPipeline: async ({ slot }) => { + visited.push(slot.index); + return { + envelope: {}, + draft: makeDraft(slot.index), + meta: { llmOutputHash: `hash-${slot.index}`, promptTemplateId: "test", routePlan: null }, + }; + }, + }, + }); + + assert.deepEqual(visited, [1]); + assert.deepEqual( + result.slotResults.map((slot) => [slot.slotIndex, slot.terminalStatus]), + [ + [0, "SUCCEEDED"], + [1, "SUCCEEDED"], + [2, "SUCCEEDED"], + ], + ); + assert.deepEqual( + result.problems.map((problem) => problem.title), + ["Problem 0", "Problem 1", "Problem 2"], + ); + assert.equal(result.outcomes.length, 3); +}); + +test("generation: bounded concurrency runs multiple slots in parallel but preserves ordered output", async () => { + const plan = makePlan(4); + let inFlight = 0; + let maxInFlight = 0; + + const result = await generateProblemsFromPlan(plan, { + concurrency: 2, + deps: { + runSlotPipeline: async ({ slot }) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 25)); + inFlight -= 1; + return { + envelope: {}, + draft: makeDraft(slot.index), + meta: { llmOutputHash: `hash-${slot.index}`, promptTemplateId: "test", routePlan: null }, + }; + }, + }, + }); + + assert.equal(maxInFlight, 2); + assert.deepEqual( + result.slotResults.map((slot) => slot.slotIndex), + [0, 1, 2, 3], + ); + assert.deepEqual( + result.problems.map((problem) => problem.title), + ["Problem 0", "Problem 1", "Problem 2", "Problem 3"], + ); +}); diff --git a/apps/backend/test/unit/generation/testStrengthGateIntegration.test.js b/apps/backend/test/unit/generation/testStrengthGateIntegration.test.js deleted file mode 100644 index 385fb54..0000000 --- a/apps/backend/test/unit/generation/testStrengthGateIntegration.test.js +++ /dev/null @@ -1,87 +0,0 @@ -require("../../helpers/setupBase"); - -const test = require("node:test"); -const assert = require("node:assert/strict"); - -const { generateProblemsFromPlan } = require("../../../src/generation"); -const { GenerationSlotFailureError } = require("../../../src/generation/errors"); -const { runTestStrengthGate } = require("../../../src/generation/testStrengthGate"); - -function mkJudgeResult(success) { - return { - success, - passedTests: success ? ["t"] : [], - failedTests: success ? [] : ["t"], - stdout: "", - stderr: "", - executionTimeMs: 1, - exitCode: success ? 0 : 1, - timedOut: false, - }; -} - -test("generation: test strength gate failure is contract-equivalent and deterministic (kind=quality)", async () => { - const plan = [ - { - index: 0, - language: "python", - difficulty: "hard", - topics: ["arrays"], - problem_style: "return", - constraints: "Python 3.11, pytest, standard library only, no filesystem access, no networking, time limit enforced.", - test_case_count: 8, - }, - ]; - - const starter = "def solve(*args, **kwargs):\n return 0\n"; - const draft = { - language: "python", - id: "p1", - title: "Gate", - description: "desc", - starter_code: starter, - reference_solution: "def solve(*args, **kwargs):\n return 1\n", - test_suite: "import pytest\nfrom solution import solve\n\ndef test_case_1(): assert solve(1) == 1\n", - constraints: plan[0].constraints, - sample_inputs: ["x=1"], - sample_outputs: ["1"], - difficulty: "hard", - topic_tag: "arrays", - }; - - let generateCalls = 0; - const generateSingleProblem = async () => { - generateCalls++; - return { draft, meta: { llmOutputHash: "x" } }; - }; - - let validateCalls = 0; - const validateReferenceSolution = async () => { - validateCalls++; - }; - - const judgeAdapter = { - judge: async (req) => { - if (req.kind === "code" && req.code === starter) return mkJudgeResult(true); - return mkJudgeResult(false); - }, - }; - - const strengthGate = async (d, s) => runTestStrengthGate(d, s, { judgeAdapter }); - - await assert.rejects( - () => - generateProblemsFromPlan(plan, { - deps: { generateSingleProblem, validateReferenceSolution, runTestStrengthGate: strengthGate }, - }), - (e) => { - assert.ok(e instanceof GenerationSlotFailureError); - assert.equal(e.kind, "quality"); - return true; - } - ); - - assert.equal(generateCalls, 3); - // Dedup guard avoids re-running Docker validation when artifacts are identical across attempts. - assert.equal(validateCalls, 1); -}); diff --git a/apps/backend/test/unit/generation/validationService.test.js b/apps/backend/test/unit/generation/validationService.test.js new file mode 100644 index 0000000..ad8a824 --- /dev/null +++ b/apps/backend/test/unit/generation/validationService.test.js @@ -0,0 +1,34 @@ +require("../../helpers/setupBase"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { inferFailureKind, progressSummaryForFailure } = require("../../../src/generation/services/validationService"); + +test("validation service: preserves explicit terminal error kinds", () => { + assert.equal(inferFailureKind({ kind: "timeout", message: "Reference solution timed out" }), "timeout"); + assert.equal(inferFailureKind({ kind: "contract", message: "Repair regenerated the same artifact" }), "contract"); +}); + +test("validation service: slot failure summary keeps timeout terminal errors classified", () => { + const out = progressSummaryForFailure({ + slotIndex: 0, + attempt: 1, + maxAttempts: 1, + err: { kind: "timeout", stage: "repair", message: "Reference solution timed out" }, + slotIntent: { + slotIndex: 0, + language: "java", + difficulty: "easy", + topics: ["encapsulation"], + constraints: "Java 21", + problemStyle: "stdout", + testCaseCount: 8, + }, + final: true, + }); + + assert.equal(out.summary.kind, "timeout"); + assert.equal(out.failure.kind, "timeout"); + assert.equal(out.summary.phase, "generate"); +}); diff --git a/apps/backend/test/unit/ipc/threadGenerationSubscriptionSecurity.test.js b/apps/backend/test/unit/ipc/threadGenerationSubscriptionSecurity.test.js new file mode 100644 index 0000000..4fcedf9 --- /dev/null +++ b/apps/backend/test/unit/ipc/threadGenerationSubscriptionSecurity.test.js @@ -0,0 +1,56 @@ +require("../../helpers/setupDb"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); + +const { createThreadHandlers } = require("../../../src/ipc/threads"); +const { createThread } = require("../../../src/services/sessionService"); +const { runRepository } = require("../../../src/database/repositories/runRepository"); + +test("threads.subscribeGeneration rejects runIds that belong to a different thread", async () => { + const handlers = createThreadHandlers({ sendEvent: () => {} }); + const subscribe = handlers["threads.subscribeGeneration"].handler; + + const threadA = createThread("practice"); + const threadB = createThread("practice"); + const otherRunId = crypto.randomUUID(); + runRepository.create(otherRunId, "generation", { threadId: threadB.sessionId, metaJson: "{}" }); + + await assert.rejects( + () => subscribe({ threadId: threadA.sessionId, runId: otherRunId }), + /runId does not belong to the provided threadId/i + ); +}); + +test("threads.subscribeGeneration accepts runIds owned by the requested thread", async () => { + const handlers = createThreadHandlers({ sendEvent: () => {} }); + const subscribe = handlers["threads.subscribeGeneration"].handler; + const unsubscribe = handlers["threads.unsubscribeGeneration"].handler; + + const thread = createThread("practice"); + const runId = crypto.randomUUID(); + runRepository.create(runId, "generation", { threadId: thread.sessionId, metaJson: "{}" }); + + const result = await subscribe({ threadId: thread.sessionId, runId }); + assert.equal(result.runId, runId); + assert.ok(typeof result.subId === "string" && result.subId.length > 0); + + await unsubscribe({ subId: result.subId }); +}); + +test("threads.subscribeGeneration accepts caller-generated future runIds before persistence", async () => { + const handlers = createThreadHandlers({ sendEvent: () => {} }); + const subscribe = handlers["threads.subscribeGeneration"].handler; + const unsubscribe = handlers["threads.unsubscribeGeneration"].handler; + + const thread = createThread("practice"); + const futureRunId = crypto.randomUUID(); + + const result = await subscribe({ threadId: thread.sessionId, runId: futureRunId }); + assert.equal(result.runId, futureRunId); + assert.deepEqual(result.buffered, []); + assert.ok(typeof result.subId === "string" && result.subId.length > 0); + + await unsubscribe({ subId: result.subId }); +}); diff --git a/apps/backend/test/unit/pipeline/slotStages.test.js b/apps/backend/test/unit/pipeline/slotStages.test.js new file mode 100644 index 0000000..6af4444 --- /dev/null +++ b/apps/backend/test/unit/pipeline/slotStages.test.js @@ -0,0 +1,142 @@ +require("../../helpers/setupBase"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { SlotPipelineTerminalError, __test__ } = require("../../../src/pipeline/slotStages"); + +function makeJavaSlot(topics = ["encapsulation"]) { + return { + index: 0, + language: "java", + difficulty: "easy", + topics, + problem_style: "stdout", + constraints: "Java 21, JUnit 5, standard library only.", + test_case_count: 8, + }; +} + +function makeJavaDraft(referenceSolution, testSuite) { + return { + language: "java", + id: "java-encapsulation-1", + title: "Encapsulated Music Profile", + description: "Build a small encapsulation exercise.", + starter_code: "public class UserProfile {\n // TODO\n}\n", + reference_solution: referenceSolution, + test_suite: testSuite, + constraints: "Java 21, JUnit 5, standard library only.", + sample_inputs: ["sample"], + sample_outputs: ["sample"], + difficulty: "easy", + topic_tag: "encapsulation", + }; +} + +test("slot stages: preflight rejects stdin-driven Java structural-topic references before Docker validation", () => { + const slot = makeJavaSlot(["encapsulation"]); + const draft = makeJavaDraft( + [ + "import java.util.Scanner;", + "public class UserProfile {", + " private String name = \"guest\";", + " public static void main(String[] args) {", + " Scanner scanner = new Scanner(System.in);", + " System.out.println(scanner.nextLine());", + " }", + " public String getName() { return name; }", + " public void setName(String value) { this.name = value; }", + "}", + ].join("\n"), + [ + "import org.junit.jupiter.api.Test;", + "import static org.junit.jupiter.api.Assertions.*;", + "public class UserProfileTest {", + " @Test void testStatefulEncapsulation() {", + " UserProfile profile = new UserProfile();", + " profile.setName(\"A\");", + " assertEquals(\"A\", profile.getName());", + " }", + "}", + ].join("\n") + ); + + assert.throws( + () => __test__.preflightValidateDraft({ slot, draft, stage: "reference" }), + (error) => { + assert.ok(error instanceof SlotPipelineTerminalError); + assert.equal(error.stage, "reference"); + assert.equal(error.kind, "contract"); + assert.match(error.message, /stdin reads/i); + return true; + } + ); +}); + +test("slot stages: preflight rejects Java structural-topic drafts that violate structural requirements", () => { + const slot = makeJavaSlot(["encapsulation"]); + const draft = makeJavaDraft( + [ + "public class UserProfile {", + " public String name = \"guest\";", + " public String getName() { return name; }", + "}", + ].join("\n"), + [ + "import org.junit.jupiter.api.Test;", + "import static org.junit.jupiter.api.Assertions.*;", + "public class UserProfileTest {", + " @Test void testGetterOnly() {", + " UserProfile profile = new UserProfile();", + " assertEquals(\"guest\", profile.getName());", + " }", + "}", + ].join("\n") + ); + + assert.throws( + () => __test__.preflightValidateDraft({ slot, draft, stage: "reference" }), + (error) => { + assert.ok(error instanceof SlotPipelineTerminalError); + assert.equal(error.kind, "contract"); + assert.match(error.message, /private field|public fields/i); + return true; + } + ); +}); + +test("slot stages: detects no-op repair when hash or source is unchanged", () => { + assert.equal( + __test__.isNoOpReferenceRepair({ + previousReferenceSource: "class A {}", + previousReferenceHash: "abc", + nextReferenceSource: "class B {}", + nextReferenceHash: "abc", + }), + true + ); + + assert.equal( + __test__.isNoOpReferenceRepair({ + previousReferenceSource: "class A {}", + nextReferenceSource: " class A {} \n", + }), + true + ); + + assert.equal( + __test__.isNoOpReferenceRepair({ + previousReferenceSource: "class A {}", + previousReferenceHash: "abc", + nextReferenceSource: "class B {}", + nextReferenceHash: "def", + }), + false + ); +}); + +test("slot stages: preserves explicit terminal failure kinds for downstream summaries", () => { + assert.equal(__test__.inferFailureKind({ kind: "timeout", message: "timed out" }), "timeout"); + assert.equal(__test__.inferFailureKind({ kind: "contract", message: "bad draft" }), "contract"); +}); diff --git a/apps/backend/test/unit/services/generationStateMachine.test.js b/apps/backend/test/unit/services/generationStateMachine.test.js new file mode 100644 index 0000000..b48d68d --- /dev/null +++ b/apps/backend/test/unit/services/generationStateMachine.test.js @@ -0,0 +1,110 @@ +require("../../helpers/setupBase"); + +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { deriveRunStatus, mapRunStatusToThreadState } = require("../../../src/services/threads/generationState"); +const { + subscribeGenerationProgress, + publishGenerationProgress, + getGenerationProgressBuffer, +} = require("../../../src/generation/progressBus"); +const { + buildJudgeResult, + COMPILE_TIMEOUT_MARKER, + EXEC_TIMEOUT_MARKER, +} = require("../../../src/judge/outcome"); + +test("deriveRunStatus maps mixed slot outcomes to incomplete", () => { + const status = deriveRunStatus([ + { + slotIndex: 0, + terminalStatus: "SUCCEEDED", + retries: 0, + problem: { id: "p1" }, + outcome: { slotIndex: 0, success: true, status: "SUCCEEDED", retries: 0 }, + }, + { + slotIndex: 1, + terminalStatus: "RETRYABLE_FAILURE", + retries: 1, + outcome: { + slotIndex: 1, + success: false, + status: "RETRYABLE_FAILURE", + retries: 1, + failureKind: "timeout", + failureCode: "TIME_BUDGET_EXCEEDED", + message: "Reference execution timed out.", + }, + failure: { + kind: "timeout", + code: "TIME_BUDGET_EXCEEDED", + message: "Reference execution timed out.", + stage: "VALIDATING_REFERENCE", + }, + }, + ]); + + assert.equal(status, "INCOMPLETE"); + assert.equal(mapRunStatusToThreadState(status), "INCOMPLETE"); +}); + +test("progress bus isolates buffered and live events by runId", () => { + const seenRun1 = []; + const seenRun2 = []; + const off1 = subscribeGenerationProgress("run-1", (event) => seenRun1.push(event)); + const off2 = subscribeGenerationProgress("run-2", (event) => seenRun2.push(event)); + + publishGenerationProgress("run-1", { type: "generation_started", runId: "run-1", totalSlots: 2, run: 1 }); + publishGenerationProgress("run-2", { type: "generation_started", runId: "run-2", totalSlots: 1, run: 1 }); + publishGenerationProgress("run-1", { type: "generation_failed", runId: "run-1", error: "boom" }); + + assert.equal(seenRun1.length, 2); + assert.equal(seenRun2.length, 1); + assert.equal(getGenerationProgressBuffer("run-1").length, 2); + assert.equal(getGenerationProgressBuffer("run-2").length, 1); + + off1(); + off2(); +}); + +test("judge outcome preserves timeout stage and strips internal timeout markers", () => { + const compileTimeout = buildJudgeResult({ + success: false, + passedTests: [], + failedTests: [], + executionTimeMs: 1234, + capture: { + stdout: "", + stderr: `${COMPILE_TIMEOUT_MARKER}\ncompiler stalled`, + exitCode: 124, + timedOut: false, + outputLimitExceeded: false, + }, + }); + + assert.equal(compileTimeout.failureCategory, "TIME_BUDGET_EXCEEDED"); + assert.equal(compileTimeout.timeoutStage, "compile"); + assert.equal(compileTimeout.watchdogSource, "inner"); + assert.ok(!compileTimeout.stderr.includes(COMPILE_TIMEOUT_MARKER)); + + const execTimeout = buildJudgeResult({ + success: false, + passedTests: [], + failedTests: [], + executionTimeMs: 2500, + capture: { + stdout: "", + stderr: `${EXEC_TIMEOUT_MARKER}\nprogram stalled`, + exitCode: 124, + timedOut: false, + outputLimitExceeded: false, + }, + }); + + assert.equal(execTimeout.failureCategory, "TIME_BUDGET_EXCEEDED"); + assert.equal(execTimeout.timeoutStage, "execute"); + assert.equal(execTimeout.watchdogSource, "inner"); + assert.ok(!execTimeout.stderr.includes(EXEC_TIMEOUT_MARKER)); +}); diff --git a/apps/frontend/src/app/activity/[id]/page.tsx b/apps/frontend/src/app/activity/[id]/page.tsx index 79c3aef..ccad23e 100644 --- a/apps/frontend/src/app/activity/[id]/page.tsx +++ b/apps/frontend/src/app/activity/[id]/page.tsx @@ -86,6 +86,34 @@ export default function ActivityPage() { ); } + if (activity.status === "INCOMPLETE") { + return ( +
+
+
This activity is incomplete
+
+ Generation only partially succeeded. Open review mode to inspect the surviving problems and repair or + discard the incomplete result before using it as a learner activity. +
+
+ + +
+
+
+ ); + } + return (