diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index c43fa4a0..f66e76b4 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -1605,7 +1605,6 @@ fn retained_partial_progress_error( fn terminal_failure_has_specific_error_class(error: &Report) -> bool { error.downcast_ref::().is_some() || error.downcast_ref::().is_some() - || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index 5f7bf9f9..5661cfa7 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -1720,10 +1720,17 @@ fn operator_queued_issue_attention_summary( worktree_has_tracked_changes: bool, attention_error_class: Option<&str>, ) -> String { - if retry_budget_attempts > 0 && worktree_has_tracked_changes { - return format!( - "Partial worktree changes are retained after {retry_budget_attempts} failed attempts; inspect the patch, finish validation, then land or reset manually." - ); + if worktree_has_tracked_changes { + if retry_budget_attempts > 0 { + return format!( + "Partial worktree changes are retained after {retry_budget_attempts} failed attempts; inspect the patch, finish validation, then land or reset manually." + ); + } + if attention_error_class == Some("partial_progress_retained") { + return String::from( + "Partial worktree changes are retained after a stalled or failed attempt; inspect the patch, finish validation, then land or reset manually.", + ); + } } if attention_error_class == Some("app_server_plugin_list_timeout") { return String::from( diff --git a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs index b3846020..bb563c30 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs @@ -509,6 +509,85 @@ fn live_operator_status_snapshot_surfaces_retained_partial_progress() { ); } +#[test] +fn live_operator_status_snapshot_surfaces_stalled_retained_partial_progress() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-needs-attention", + "PUB-110", + "Todo", + &["decodex:needs-attention"], + Some(2), + "2026-03-13T09:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + let tracker = FakeTracker::new(vec![issue.clone()]); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-110", ".worktrees/PUB-110", "main"], + ); + + fs::write(worktree_path.join("README.md"), "retained stalled patch\n") + .expect("tracked worktree file should change"); + + tracker.issue_comments.borrow_mut().insert( + issue.id.clone(), + vec![linear_execution_history_comment( + &issue, + "terminal_failure", + "2026-03-13T09:20:00Z", + "stalled-retained-partial-progress", + |record| { + record.error_class = Some(String::from("partial_progress_retained")); + record.next_action = Some(String::from( + "inspect retained worktree `.worktrees/PUB-110`, finish validation and PR handoff or reset the patch manually", + )); + record.summary = Some(String::from("Decodex run retained partial progress.")); + record.blockers = Some(vec![String::from( + "tracked worktree changes were retained after stalled reconciliation", + )]); + record.evidence = Some(vec![String::from( + "worktree `.worktrees/PUB-110` has tracked changes", + )]); + }, + )], + ); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = snapshot + .queued_candidates + .iter() + .find(|candidate| candidate.issue_identifier == "PUB-110") + .expect("needs-attention queued issue should exist"); + let attention = candidate.attention.as_ref().expect("attention details should render"); + + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.reason, "issue_needs_attention"); + assert_eq!(attention.attention_error_class.as_deref(), Some("partial_progress_retained")); + assert!(attention.worktree_has_tracked_changes); + assert_eq!(attention.retry_budget_attempt_count, None); + assert!( + attention.summary.contains("Partial worktree changes are retained"), + "summary should explain retained stalled patch recovery, got {:?}", + attention.summary + ); + assert!( + attention + .attention_next_action + .as_deref() + .is_some_and(|action| action.contains("finish validation and PR handoff")) + ); +} + #[test] fn live_operator_status_snapshot_surfaces_git_credential_failures() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs index 1cdf4087..ff1a1560 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -999,6 +999,90 @@ fn stalled_run_reconciliation_routes_to_needs_attention_without_cleanup() { })); } +#[test] +fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let tracker = FakeTracker::new(vec![]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let issue = sample_issue("In Progress", &[]); + let run_id = "run-stalled-dirty"; + let worktree_path = config.worktree_root().join("PUB-102"); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-102", ".worktrees/PUB-102", "main"], + ); + + fs::write(worktree_path.join("README.md"), "retained partial work\n") + .expect("tracked worktree file should change"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-102", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + let action = ActiveRunReconciliation { + issue: issue.clone(), + run_attempt: state_store + .run_attempt(run_id) + .expect("run attempt query should succeed") + .expect("run attempt should exist"), + worktree_mapping: state_store + .worktree_for_issue(&issue.id) + .expect("worktree query should succeed"), + disposition: ActiveRunDisposition::Stalled { + idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + }, + workflow: workflow.clone(), + }; + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + vec![action], + ) + .expect("reconciliation should succeed"); + + assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none()); + assert!( + state_store + .worktree_for_issue(&issue.id) + .expect("worktree lookup should succeed") + .is_some() + ); + assert_eq!( + state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status(), + "stalled" + ); + + let comments = tracker.comments.borrow(); + + assert!(comments.iter().any(|comment| { + comment.contains("partial_progress_retained") + && comment.contains("finish validation and PR handoff or reset the patch manually") + && comment.contains(".worktrees/PUB-102") + })); + assert!(comments.iter().all(|comment| !comment.contains("stalled_run_detected"))); +} + #[test] fn project_reconciliation_routes_orphaned_active_worktree_run_to_needs_attention() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 4e2a7129..b42bc3e6 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -145,7 +145,7 @@ outside the local operator surface. | `Accounts` | Shared Codex account pool and usage table from `~/.codex/decodex/accounts.jsonl` when `[codex.accounts]` is enabled for a project. Account identity can be obscured from the `Account` column header eye without changing the underlying snapshot. Selecting an account writes the global `[codex.accounts].fixed_account` selector in `~/.codex/decodex/config.toml`; clearing it returns all new account-pool runs to balanced account selection. Account display-name rerolls write `[codex.account_names.offsets]` in the same global config so Decodex App and the dashboard share the privacy-preserving names. Theme, sort, and identity-visibility preferences are client-local presentation state. The selector is global and does not pin a project to an account. | | `Projects` | Fleet-level project table. The section-level filter toggles between active project work and the full registry. Location is its own compact path column and can be obscured from the location header eye. `Activity` shows a relative timestamp or `-`; `Work` is `running/waiting/attention`. It should not duplicate per-lane details already shown below. | | `Running Lanes` | Active leased or live-executing issue lanes. A lane here is currently owned by this local control plane, or a live process/thread/protocol marker still explains active execution even when the queue lease is not held. It shows issue identity, phase, operation, attempt, queue lease state, execution liveness, thread/protocol status, child-agent activity when captured, timing, branch, and worktree. | -| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after retries, the candidate is partial retained progress and not just a generic retry-budget hold. Running lanes are not repeated as normal intake work. | +| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after stalled reconciliation, failure writeback, or retries, the candidate is partial retained progress and not just a generic stalled or retry-budget hold. Running lanes are not repeated as normal intake work. | | `Review & Landing` | Retained PR lanes after review handoff. This section owns post-review repair, wait-for-review, ready-to-land, closeout, cleanup, and blocked retained-lane visibility. | | `Recovery Worktrees` | Retained local worktrees that are not currently owned by `Running Lanes`, `Review & Landing`, or queued attention in `Intake Queue`. This is the cleanup or recovery inbox for recovered paths, retained PR leftovers, and cleanup-only local worktrees. Empty is the normal healthy state. | | `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Raw local attempts and heartbeat details stay in debug expansion. |