Generation State Machine Revamp

CONTRIBUTING.md

@@ -45,6 +45,8 @@ Keep in mind:
 - Codemm’s “agent logic” is backend-owned; the IDE should remain a thin shell over backend contracts.
 - Judging relies on Docker; don’t add a path that executes untrusted code outside Docker.
 - For generation APIs, preserve IPC naming stability (`threads.generate`, `threads.generateV2`, `threads.regenerateSlot`, `threads.getGenerationDiagnostics`).
+- Generation progress is `runId`-scoped. Do not add thread-wide progress reducers or subscriptions that can mix overlapping runs.
+- Persistent generation state lives in `generation_runs`, `generation_slot_runs`, and `generation_slot_transitions`. Avoid rebuilding run state from transient UI reducers or `problems_json` length.
 
 ## Style / Guardrails
 

README.md

@@ -35,7 +35,17 @@ There is no internal HTTP API for engine calls. UI → engine is IPC only.
 ## Local State & Persistence
 
 - Per-workspace DB: `<workspaceDataDir>/codemm.db` (preferred: `<workspace>/.codemm/codemm.db`)
-- Key tables (IDE-first): `threads`, `thread_messages`, `activities`, `runs`, `run_events`
+- Key tables (IDE-first): `threads`, `thread_messages`, `activities`, `runs`, `run_events`, `generation_runs`, `generation_slot_runs`, `generation_slot_transitions`
+
+## Generation State Machine
+
+Generation is now persisted as three explicit aggregates:
+
+- `threads`: user-facing lifecycle (`DRAFT` -> `CLARIFYING` -> `READY` -> `GENERATE_PENDING` -> `GENERATING` -> terminal outcome)
+- `generation_runs`: one durable activity-generation run per request, correlated by `runId`
+- `generation_slot_runs`: one durable slot record per run, with stage transitions and terminal status per slot
+
+Thread state is derived from persisted run outcomes. Slot failures no longer abort the full thread on first exception, and stale `RUNNING` generation state is reconciled on engine startup.
 
 ## Security Model (Practical)
 
@@ -99,6 +109,7 @@ Builds are typically produced on the target OS (mac builds on macOS, win builds
 
 - IDE-first mental model + topology: `docs/architecture/IDE_FIRST.md`
 - Local runtime orchestration: `docs/architecture/LOCAL_LLM_ORCHESTRATION.md`
+- Generation state machine: `docs/architecture/GENERATION_STATE_MACHINE.md`
 - Migration phases: `docs/architecture/MIGRATION.md`
 - Wrapper behavior: `docs/FUNCTIONS.md`
 - Security notes: `docs/SECURITY.md`

apps/backend/scripts/runTests.js

@@ -54,6 +54,13 @@ function main(argv) {
     return 1;
   }
 
+  const contractsBuild = spawnSync("npm", ["--workspace", "@codemm/shared-contracts", "run", "build"], {
+    stdio: "inherit",
+  });
+  if ((contractsBuild.status ?? 1) !== 0) {
+    return contractsBuild.status ?? 1;
+  }
+
   // ts-node compilation dominates cost when Node runs each file in an isolated worker.
   // Running with no isolation keeps a single module cache and is significantly faster.
   // Note: Node's test runner flags vary across Node versions. Prefer broad compatibility.

apps/backend/src/contracts/generationOutcome.ts

@@ -1,7 +1,12 @@
+import type { GenerationFailureKind, GenerationSlotTerminalStatus } from "@codemm/shared-contracts";
+
 export type GenerationOutcome = {
   slotIndex: number;
   success: boolean;
+  status: GenerationSlotTerminalStatus;
   retries: number;
+  failureKind?: GenerationFailureKind;
+  failureCode?: string;
+  message?: string;
   appliedFallback?: string;
 };
-

apps/backend/src/contracts/session.ts

@@ -10,9 +10,13 @@ export const SessionStateSchema = z.enum([
   "DRAFT",
   "CLARIFYING",
   "READY",
+  "GENERATE_PENDING",
   "GENERATING",
-  "SAVED",
-  "FAILED",
+  "COMPLETED",
+  "INCOMPLETE",
+  "PARTIAL_SUCCESS",
+  "RETRYABLE_FAILURE",
+  "HARD_FAILURE",
 ]);
 export type SessionState = z.infer<typeof SessionStateSchema>;
 
@@ -61,10 +65,14 @@ export type Session = z.infer<typeof SessionSchema>;
 const ALLOWED_TRANSITIONS: Record<SessionState, SessionState[]> = {
   DRAFT: ["CLARIFYING"],
   CLARIFYING: ["CLARIFYING", "READY"],
-  READY: ["GENERATING"],
-  GENERATING: ["SAVED", "FAILED", "READY"],
-  SAVED: [],
-  FAILED: ["READY"],
+  READY: ["GENERATE_PENDING"],
+  GENERATE_PENDING: ["GENERATING", "RETRYABLE_FAILURE", "HARD_FAILURE"],
+  GENERATING: ["COMPLETED", "INCOMPLETE", "PARTIAL_SUCCESS", "RETRYABLE_FAILURE", "HARD_FAILURE"],
+  COMPLETED: ["GENERATE_PENDING"],
+  INCOMPLETE: ["GENERATE_PENDING"],
+  PARTIAL_SUCCESS: ["GENERATE_PENDING"],
+  RETRYABLE_FAILURE: ["GENERATE_PENDING", "READY"],
+  HARD_FAILURE: ["GENERATE_PENDING", "READY"],
 };
 
 export function canTransition(from: SessionState, to: SessionState): boolean {

apps/backend/src/database.ts

@@ -10,6 +10,22 @@ export {
   activityRepository as activityDb,
   submissionRepository as submissionDb,
 } from "./database/repositories/activityRepository";
+export type {
+  DBGenerationExecutionAttempt,
+  DBGenerationRunFailureCacheEntry,
+  DBGenerationRun,
+  DBGenerationSlotRun,
+  DBGenerationSlotDiagnosis,
+  DBGenerationSlotTransition,
+} from "./database/repositories/generationRunRepository";
+export {
+  generationExecutionAttemptRepository as generationExecutionAttemptDb,
+  generationRunFailureCacheRepository as generationRunFailureCacheDb,
+  generationRunRepository as generationRunDb,
+  generationSlotRunRepository as generationSlotRunDb,
+  generationSlotDiagnosisRepository as generationSlotDiagnosisDb,
+  generationSlotTransitionRepository as generationSlotTransitionDb,
+} from "./database/repositories/generationRunRepository";
 export type {
   DBSession,
   DBSessionCollector,

apps/backend/src/database/migrations.ts

@@ -158,13 +158,140 @@ export function initializeDatabase() {
     )
   `);
 
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_runs (
+      id TEXT PRIMARY KEY,
+      thread_id TEXT NOT NULL,
+      status TEXT NOT NULL,
+      activity_id TEXT,
+      total_slots INTEGER NOT NULL DEFAULT 0,
+      completed_slots INTEGER NOT NULL DEFAULT 0,
+      successful_slots INTEGER NOT NULL DEFAULT 0,
+      failed_slots INTEGER NOT NULL DEFAULT 0,
+      last_failure_kind TEXT,
+      last_failure_code TEXT,
+      last_failure_message TEXT,
+      meta_json TEXT,
+      started_at TEXT,
+      finished_at TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY (thread_id) REFERENCES threads(id) ON DELETE CASCADE,
+      FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE SET NULL
+    )
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_slot_runs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id TEXT NOT NULL,
+      slot_index INTEGER NOT NULL,
+      status TEXT NOT NULL,
+      current_stage TEXT,
+      attempt_count INTEGER NOT NULL DEFAULT 0,
+      title TEXT,
+      topic TEXT,
+      language TEXT,
+      started_at TEXT,
+      ended_at TEXT,
+      last_failure_kind TEXT,
+      last_failure_code TEXT,
+      last_failure_message TEXT,
+      last_artifact_hash TEXT,
+      result_json TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+      UNIQUE(run_id, slot_index),
+      FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE
+    )
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_slot_transitions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id TEXT NOT NULL,
+      slot_index INTEGER NOT NULL,
+      attempt INTEGER,
+      stage TEXT,
+      status TEXT NOT NULL,
+      payload_json TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE
+    )
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_execution_attempts (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id TEXT NOT NULL,
+      slot_index INTEGER NOT NULL,
+      attempt INTEGER NOT NULL,
+      execution_phase TEXT NOT NULL,
+      bundle_hash TEXT NOT NULL,
+      strategy TEXT,
+      budget_profile_json TEXT,
+      started_at TEXT NOT NULL,
+      finished_at TEXT,
+      exit_code INTEGER,
+      timeout_stage TEXT,
+      watchdog_source TEXT,
+      failure_category TEXT,
+      stdout_hash TEXT,
+      stderr_hash TEXT,
+      stdout_snippet TEXT,
+      stderr_snippet TEXT,
+      parsed_failures_json TEXT,
+      trace_json TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE
+    )
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_slot_diagnoses (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id TEXT NOT NULL,
+      slot_index INTEGER NOT NULL,
+      attempt INTEGER NOT NULL,
+      diagnosis_class TEXT NOT NULL,
+      recoverability TEXT NOT NULL,
+      normalized_symptom TEXT NOT NULL,
+      recommended_repair_strategy TEXT,
+      source_execution_attempt_id INTEGER,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE,
+      FOREIGN KEY (source_execution_attempt_id) REFERENCES generation_execution_attempts(id) ON DELETE SET NULL
+    )
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS generation_run_failure_cache (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      run_id TEXT NOT NULL,
+      language TEXT NOT NULL,
+      topic_signature TEXT NOT NULL,
+      failure_class TEXT NOT NULL,
+      normalized_symptom TEXT NOT NULL,
+      guardrail_patch_json TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY (run_id) REFERENCES generation_runs(id) ON DELETE CASCADE
+    )
+  `);
+
   db.exec(`
     CREATE INDEX IF NOT EXISTS idx_threads_state ON threads(state);
     CREATE INDEX IF NOT EXISTS idx_thread_messages_thread_id ON thread_messages(thread_id);
     CREATE INDEX IF NOT EXISTS idx_thread_collectors_thread_id ON thread_collectors(thread_id);
     CREATE INDEX IF NOT EXISTS idx_submissions_activity_id ON submissions(activity_id);
     CREATE INDEX IF NOT EXISTS idx_runs_thread_id ON runs(thread_id);
     CREATE INDEX IF NOT EXISTS idx_run_events_run_id ON run_events(run_id);
+    CREATE INDEX IF NOT EXISTS idx_generation_runs_thread_id ON generation_runs(thread_id);
+    CREATE INDEX IF NOT EXISTS idx_generation_runs_status ON generation_runs(status);
+    CREATE INDEX IF NOT EXISTS idx_generation_slot_runs_run_id ON generation_slot_runs(run_id);
+    CREATE INDEX IF NOT EXISTS idx_generation_slot_transitions_run_id ON generation_slot_transitions(run_id);
+    CREATE INDEX IF NOT EXISTS idx_generation_execution_attempts_run_slot ON generation_execution_attempts(run_id, slot_index, attempt);
+    CREATE INDEX IF NOT EXISTS idx_generation_slot_diagnoses_run_slot ON generation_slot_diagnoses(run_id, slot_index, attempt);
+    CREATE INDEX IF NOT EXISTS idx_generation_run_failure_cache_run_id ON generation_run_failure_cache(run_id);
   `);
 
   console.log("Database initialized successfully");

apps/backend/src/database/repositories/activityRepository.ts

@@ -36,7 +36,7 @@ export const activityRepository = {
     title: string,
     problems: string,
     prompt?: string,
-    opts?: { status?: "DRAFT" | "PUBLISHED"; timeLimitSeconds?: number | null }
+    opts?: { status?: "DRAFT" | "INCOMPLETE" | "PUBLISHED"; timeLimitSeconds?: number | null }
   ) => {
     const stmt = db.prepare(
       `INSERT INTO activities (id, title, prompt, problems, status, time_limit_seconds, created_at)
@@ -78,7 +78,7 @@ export const activityRepository = {
       prompt?: string;
       problems?: string;
       time_limit_seconds?: number | null;
-      status?: "DRAFT" | "PUBLISHED";
+      status?: "DRAFT" | "INCOMPLETE" | "PUBLISHED";
     }
   ): DBActivity | undefined => {
     const sets: string[] = [];