From d57b30b839665550c71ff5742c3688d00db4d5c6 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 5 Jun 2026 15:40:04 +0800 Subject: [PATCH 1/3] {"schema":"decodex/commit/1","summary":"Retain dirty runtime failures as partial progress","authority":"manual"} --- apps/decodex/src/orchestrator/daemon.rs | 1 + apps/decodex/src/orchestrator/execution.rs | 65 +++++- .../src/orchestrator/reconciliation.rs | 1 + .../tests/recovery/reconciliation.rs | 15 +- .../tests/recovery/terminal_failures.rs | 219 ++++++++++++++++++ .../src/orchestrator/tests/runtime/failure.rs | 1 + apps/decodex/src/orchestrator/types.rs | 1 + docs/spec/linear-execution-ledger.md | 4 +- docs/spec/runtime.md | 2 +- 9 files changed, 294 insertions(+), 15 deletions(-) diff --git a/apps/decodex/src/orchestrator/daemon.rs b/apps/decodex/src/orchestrator/daemon.rs index dd2733e..a2737dc 100644 --- a/apps/decodex/src/orchestrator/daemon.rs +++ b/apps/decodex/src/orchestrator/daemon.rs @@ -1290,6 +1290,7 @@ where issue_identifier: issue.identifier.clone(), run_id: child.run_id.to_owned(), worktree_path: worktree_path.clone(), + source_error_class: None, }) } else { Report::msg(format!( diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index cadae12..a2556d1 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -71,6 +71,7 @@ struct TerminalFailureLifecycle<'a> { target_state: &'a str, worktree_path: &'a str, manual_attention_requested: bool, + retained_source_error_class: Option<&'a str>, } struct RunStartedLifecycleFields<'a> { @@ -450,13 +451,21 @@ fn terminal_failure_lifecycle_event( record.next_action = Some(failure.next_action.to_owned()); if retained_partial_progress { - record.blockers = Some(vec![String::from( - "Retained tracked worktree changes require operator recovery.", - )]); - record.evidence = Some(vec![format!( + let mut evidence = vec![format!( "Attempt {} stopped with tracked worktree changes retained.", issue_run.attempt_number + )]; + + if let Some(source_error_class) = failure.retained_source_error_class { + evidence.push(format!( + "Source failure class `{source_error_class}` was preserved for recovery context." + )); + } + + record.blockers = Some(vec![String::from( + "Retained tracked worktree changes require operator recovery.", )]); + record.evidence = Some(evidence); record.summary = Some(String::from("Decodex retained partial progress and needs attention.")); record.terminal_path = Some(String::from("retained_partial_progress")); } else { @@ -1626,7 +1635,7 @@ fn retained_partial_progress_error( issue_run: &IssueRunPlan, worktree_path: &str, ) -> Option { - if terminal_failure_has_specific_error_class(error) + if retained_progress_should_defer_to_terminal_intent(error) || !worktree_has_tracked_changes(&issue_run.worktree.path) { return None; @@ -1636,20 +1645,48 @@ fn retained_partial_progress_error( issue_identifier: issue_run.issue.identifier.clone(), run_id: issue_run.run_id.clone(), worktree_path: worktree_path.to_owned(), + source_error_class: retained_progress_source_error_class(error).map(ToOwned::to_owned), })) } -fn terminal_failure_has_specific_error_class(error: &Report) -> bool { +fn retained_progress_should_defer_to_terminal_intent(error: &Report) -> bool { error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() +} + +fn retained_progress_source_error_class(error: &Report) -> Option<&'static str> { + if let Some(app_server_failure) = error.downcast_ref::() { + Some(app_server_failure.error_class()) + } else if error.downcast_ref::().is_some() { + Some("stalled_run_detected") + } else if error.downcast_ref::().is_some() { + Some("github_credentials_unavailable") + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + Some(app_server_failure.error_class()) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + Some(app_server_failure.error_class()) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + Some(app_server_failure.error_class()) + } else if let Some(app_server_failure) = + error.downcast_ref::() + { + Some(app_server_failure.error_class()) + } else if let Some(app_server_failure) = error.downcast_ref::() { + Some(app_server_failure.error_class()) + } else if let Some(repo_gate_failure) = error.downcast_ref::() { + Some(repo_gate_failure.error_class()) + } else { + None + } } fn write_retry_schedule_marker_for_runtime_retry( @@ -1764,6 +1801,9 @@ where let (error_class, next_action) = terminal_failure_comment_details(manual_attention_requested, error, &recovery_gate); let pr_url = terminal_failure_pr_url(error); + let retained_source_error_class = error + .downcast_ref::() + .and_then(|partial_progress| partial_progress.source_error_class.as_deref()); let comment = format_terminal_failure_comment( &issue_run.run_id, issue_run.attempt_number, @@ -1783,6 +1823,7 @@ where target_state: terminal_failure_state_name, worktree_path, manual_attention_requested, + retained_source_error_class, }, ); let projection = tracker::prepare_linear_execution_event_comment( diff --git a/apps/decodex/src/orchestrator/reconciliation.rs b/apps/decodex/src/orchestrator/reconciliation.rs index f4a99ce..a5f44f5 100644 --- a/apps/decodex/src/orchestrator/reconciliation.rs +++ b/apps/decodex/src/orchestrator/reconciliation.rs @@ -445,6 +445,7 @@ where issue_identifier: action.issue.identifier.clone(), run_id: action.run_attempt.run_id().to_owned(), worktree_path, + source_error_class: Some(String::from("stalled_run_detected")), }), )?; diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs index 5b5682a..b8923d2 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -1181,7 +1181,11 @@ fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktr && comment.contains("finish validation and PR handoff or reset the patch manually") && comment.contains(".worktrees/PUB-102") })); - assert!(comments.iter().all(|comment| !comment.contains("stalled_run_detected"))); + assert!( + comments + .iter() + .all(|comment| !comment.contains("- error_class: `stalled_run_detected`")) + ); assert!(comments.iter().all(|comment| !comment.contains("decodex run failed and needs attention"))); let ledger_event = comments @@ -1212,6 +1216,15 @@ fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktr .any(|item| item.contains("tracked worktree changes retained"))), "retained partial progress evidence should mention retained tracked changes" ); + assert!( + ledger_event + .evidence + .as_deref() + .is_some_and(|evidence| evidence + .iter() + .any(|item| item.contains("Source failure class `stalled_run_detected`"))), + "retained partial progress evidence should preserve the stalled source class" + ); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index ea63684..003a30e 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -1,5 +1,6 @@ use orchestrator::ReviewHandoffNeedsAttention; use orchestrator::PassiveRetainedAttentionRuntime; +use orchestrator::RepoGateFailure; #[test] fn terminal_failures_without_needs_attention_label_use_nonstartable_guard_state() { @@ -161,6 +162,79 @@ fn terminal_failures_apply_incremental_label_mutations_when_issue_labels_paginat assert_eq!(ledger_event.terminal_path.as_deref(), Some("manual_attention")); } +#[test] +fn terminal_failure_with_retained_tracked_changes_records_retained_partial_progress() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-101"); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-101", ".worktrees/PUB-101", "main"], + ); + + fs::write(worktree_path.join("README.md"), "retained terminal patch\n") + .expect("tracked worktree file should change"); + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("In Progress"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: issue.project_slug.clone().expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(RepoGateFailure::new( + RepoGateFailureKind::CommandSpawnFailed, + String::from("Failed to spawn repo gate command `cargo make test`."), + )); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("terminal failure handling should succeed"); + + let comments = tracker.comments.borrow(); + + assert!(comments.iter().any(|comment| { + comment.contains("decodex retained partial progress and needs attention") + && comment.contains("partial_progress_retained") + && comment.contains("finish validation and PR handoff or reset the patch manually") + })); + assert!(comments.iter().all(|comment| !comment.contains("decodex run failed and needs attention"))); + + let ledger_event = comments + .iter() + .find_map(|comment| records::parse_linear_execution_event_record(comment)) + .expect("retained partial progress should write a Linear execution event"); + + assert_eq!(ledger_event.event_type, "needs_attention"); + assert_eq!(ledger_event.error_class.as_deref(), Some("partial_progress_retained")); + assert_eq!(ledger_event.terminal_path.as_deref(), Some("retained_partial_progress")); + assert!( + ledger_event + .evidence + .as_deref() + .is_some_and(|evidence| evidence + .iter() + .any(|item| item.contains("Source failure class `repo_gate_command_spawn_failed`"))), + "retained partial progress evidence should preserve the source failure class" + ); +} + #[test] fn duplicate_terminal_failure_event_does_not_reapply_tracker_writeback() { let (_temp_dir, config, workflow) = temp_project_layout(); @@ -536,6 +610,151 @@ fn app_server_failures_skip_retry_and_require_attention() { ); } +#[test] +fn dirty_runtime_failures_record_retained_progress_instead_of_terminal_failure() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-101"); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-101", ".worktrees/PUB-101", "main"], + ); + + fs::write(worktree_path.join("README.md"), "retained runtime recovery work\n") + .expect("tracked worktree file should change"); + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Retry, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2-123"), + retry_budget_base: 1, + }; + let error = Report::new(AppServerCapabilityPreflightFailure::blocked_for_test( + "model", + "configured model was not present in model/list.", + )); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("dirty runtime failure should retain partial progress"); + + let comments = tracker.comments.borrow(); + + assert!(comments.iter().any(|comment| { + comment.contains("decodex retained partial progress and needs attention") + && comment.contains("partial_progress_retained") + && comment.contains("app_server_runtime_preflight_failed") + })); + assert!( + comments + .iter() + .all(|comment| !comment.contains("decodex run failed and needs attention")) + ); + + let ledger_event = comments + .iter() + .find_map(|comment| records::parse_linear_execution_event_record(comment)) + .expect("retained runtime failure should write a Linear execution event"); + + assert_eq!(ledger_event.event_type, "needs_attention"); + assert_eq!(ledger_event.error_class.as_deref(), Some("partial_progress_retained")); + assert_eq!(ledger_event.terminal_path.as_deref(), Some("retained_partial_progress")); + assert!( + ledger_event + .evidence + .as_deref() + .is_some_and(|evidence| evidence + .iter() + .any(|item| item.contains("app_server_runtime_preflight_failed"))), + "retained progress evidence should preserve the source runtime error class" + ); +} + +#[test] +fn explicit_manual_attention_keeps_manual_terminal_path_with_dirty_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-101"); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-101", ".worktrees/PUB-101", "main"], + ); + + fs::write(worktree_path.join("README.md"), "manual attention work\n") + .expect("tracked worktree file should change"); + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::new(ManualAttentionRequested { + issue_identifier: issue.identifier.clone(), + label: String::from("decodex:needs-attention"), + run_id: issue_run.run_id.clone(), + }); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("manual attention should keep its terminal path"); + + let ledger_event = tracker + .comments + .borrow() + .iter() + .find_map(|comment| records::parse_linear_execution_event_record(comment)) + .expect("manual attention should write a Linear execution event"); + + assert_eq!(ledger_event.event_type, "needs_attention"); + assert_eq!(ledger_event.error_class.as_deref(), Some("human_attention_required")); + assert_eq!(ledger_event.terminal_path.as_deref(), Some("manual_attention")); + assert_eq!( + ledger_event.summary.as_deref(), + Some("Decodex run failed and needs attention.") + ); +} + #[test] fn prepare_issue_run_clears_terminal_guard_marker_when_new_attempt_starts() { let (_temp_dir, base_config, workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs index 74ac9cb..4887772 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/failure.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -171,6 +171,7 @@ fn retained_partial_progress_uses_actionable_terminal_failure_comment() { issue_identifier: String::from("PUB-101"), run_id: String::from("pub-101-attempt-3-123"), worktree_path: String::from(".worktrees/PUB-101"), + source_error_class: None, }); let (error_class, next_action) = orchestrator::terminal_failure_comment_details( false, diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index 727b925..a612b75 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -635,6 +635,7 @@ struct RetainedPartialProgress { issue_identifier: String, run_id: String, worktree_path: String, + source_error_class: Option, } impl Display for RetainedPartialProgress { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { diff --git a/docs/spec/linear-execution-ledger.md b/docs/spec/linear-execution-ledger.md index 1ea3e49..6da2659 100644 --- a/docs/spec/linear-execution-ledger.md +++ b/docs/spec/linear-execution-ledger.md @@ -224,7 +224,9 @@ repair completion this is `review_repair`; for explicit human-required exits thi Retained partial progress is a `needs_attention` event with `error_class = "partial_progress_retained"` and `terminal_path = "retained_partial_progress"`. It must describe retained tracked -worktree changes and must not be emitted as `terminal_failure`. +worktree changes and must not be emitted as `terminal_failure`. If the retained +disposition absorbs a later runtime failure, the producer should preserve the source +failure class in `evidence` instead of changing the event type or terminal path. `failed_command` and `raw_error` are public-summary fields, not private evidence escape hatches. Producers must validate those values before writing a Linear comment. diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 0c21e31..c3be69d 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -490,7 +490,7 @@ After a process restart, recent-run history, active lease ownership, retained po - Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. - Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it. - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. -- If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. +- If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. - If Linear still has `decodex:active:` on an issue that also remains queued, but the local runtime cannot prove a matching active lease, status must classify the queued row as blocked with reason `linear_active_label_present`; it must not treat the issue as ready intake. If the retained marker or private execution event rows for that run are missing, status must surface `evidence_missing` in the recovery details. If the retained worktree has tracked changes, that dirty worktree remains owned by queued recovery/attention instead of being hidden as cleanup-only state. - During an active run, operator snapshots must expose `thread_id` as soon as the Codex thread exists, plus monotonically advancing `event_count`, `last_event_type`, and `last_event_at` once protocol events are recorded. These fields may be hydrated either from the current process journal or from the active lane's `.decodex-run-activity` marker when `status` is running in a separate process. - `thread_id = null` is expected only before the worker creates the Codex thread for the current run. `event_count = 0`, `last_event_type = null`, and `last_event_at = null` are expected only before the first protocol event for that same run. After the thread exists and protocol activity has started, those empty values indicate missing hydration rather than normal progress. From 734131e6ada9af53deea5ae3affad2990296d674 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 5 Jun 2026 15:41:09 +0800 Subject: [PATCH 2/3] {"schema":"decodex/commit/1","summary":"Fix retained progress test import","authority":"manual"} --- .../decodex/src/orchestrator/tests/recovery/terminal_failures.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index 003a30e..da81675 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -1,6 +1,5 @@ use orchestrator::ReviewHandoffNeedsAttention; use orchestrator::PassiveRetainedAttentionRuntime; -use orchestrator::RepoGateFailure; #[test] fn terminal_failures_without_needs_attention_label_use_nonstartable_guard_state() { From 9b59e99c8a6491e31b133e846a195295fb0bc08e Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 5 Jun 2026 15:45:12 +0800 Subject: [PATCH 3/3] {"schema":"decodex/commit/1","summary":"Shorten retained progress reconciliation test","authority":"manual"} --- .../decodex/src/orchestrator/tests/recovery/reconciliation.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs index b8923d2..90717ad 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -1173,8 +1173,10 @@ fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktr "stalled" ); - let comments = tracker.comments.borrow(); + assert_dirty_stalled_retained_progress_comments(&tracker.comments.borrow()); +} +fn assert_dirty_stalled_retained_progress_comments(comments: &[String]) { assert!(comments.iter().any(|comment| { comment.contains("decodex retained partial progress and needs attention") && comment.contains("partial_progress_retained")