Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion apps/decodex/src/orchestrator/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1605,7 +1605,6 @@ fn retained_partial_progress_error(
fn terminal_failure_has_specific_error_class(error: &Report) -> bool {
error.downcast_ref::<ManualAttentionRequested>().is_some()
|| error.downcast_ref::<ReviewHandoffNeedsAttention>().is_some()
|| error.downcast_ref::<StalledRunNeedsAttention>().is_some()
|| error.downcast_ref::<AgentGitCredentialsUnavailable>().is_some()
|| error.downcast_ref::<AppServerCapabilityPreflightFailure>().is_some()
|| error.downcast_ref::<AppServerHomePreflightFailure>().is_some()
Expand Down
15 changes: 11 additions & 4 deletions apps/decodex/src/orchestrator/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1720,10 +1720,17 @@ fn operator_queued_issue_attention_summary(
worktree_has_tracked_changes: bool,
attention_error_class: Option<&str>,
) -> String {
if retry_budget_attempts > 0 && worktree_has_tracked_changes {
return format!(
"Partial worktree changes are retained after {retry_budget_attempts} failed attempts; inspect the patch, finish validation, then land or reset manually."
);
if worktree_has_tracked_changes {
if retry_budget_attempts > 0 {
return format!(
"Partial worktree changes are retained after {retry_budget_attempts} failed attempts; inspect the patch, finish validation, then land or reset manually."
);
}
if attention_error_class == Some("partial_progress_retained") {
return String::from(
"Partial worktree changes are retained after a stalled or failed attempt; inspect the patch, finish validation, then land or reset manually.",
);
}
}
if attention_error_class == Some("app_server_plugin_list_timeout") {
return String::from(
Expand Down
79 changes: 79 additions & 0 deletions apps/decodex/src/orchestrator/tests/operator/status/queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,85 @@ fn live_operator_status_snapshot_surfaces_retained_partial_progress() {
);
}

#[test]
fn live_operator_status_snapshot_surfaces_stalled_retained_partial_progress() {
let (_temp_dir, config, workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");
let issue = sample_issue_with_sort_fields(
"issue-needs-attention",
"PUB-110",
"Todo",
&["decodex:needs-attention"],
Some(2),
"2026-03-13T09:16:17.133Z",
);
let worktree_path = config.worktree_root().join(&issue.identifier);
let tracker = FakeTracker::new(vec![issue.clone()]);

git_status_success(
config.repo_root(),
&["worktree", "add", "-b", "x/pubfi-pub-110", ".worktrees/PUB-110", "main"],
);

fs::write(worktree_path.join("README.md"), "retained stalled patch\n")
.expect("tracked worktree file should change");

tracker.issue_comments.borrow_mut().insert(
issue.id.clone(),
vec![linear_execution_history_comment(
&issue,
"terminal_failure",
"2026-03-13T09:20:00Z",
"stalled-retained-partial-progress",
|record| {
record.error_class = Some(String::from("partial_progress_retained"));
record.next_action = Some(String::from(
"inspect retained worktree `.worktrees/PUB-110`, finish validation and PR handoff or reset the patch manually",
));
record.summary = Some(String::from("Decodex run retained partial progress."));
record.blockers = Some(vec![String::from(
"tracked worktree changes were retained after stalled reconciliation",
)]);
record.evidence = Some(vec![String::from(
"worktree `.worktrees/PUB-110` has tracked changes",
)]);
},
)],
);

let snapshot = orchestrator::build_live_operator_status_snapshot(
&tracker,
&config,
&workflow,
&state_store,
10,
)
.expect("snapshot should build");
let candidate = snapshot
.queued_candidates
.iter()
.find(|candidate| candidate.issue_identifier == "PUB-110")
.expect("needs-attention queued issue should exist");
let attention = candidate.attention.as_ref().expect("attention details should render");

assert_eq!(candidate.classification, "blocked");
assert_eq!(candidate.reason, "issue_needs_attention");
assert_eq!(attention.attention_error_class.as_deref(), Some("partial_progress_retained"));
assert!(attention.worktree_has_tracked_changes);
assert_eq!(attention.retry_budget_attempt_count, None);
assert!(
attention.summary.contains("Partial worktree changes are retained"),
"summary should explain retained stalled patch recovery, got {:?}",
attention.summary
);
assert!(
attention
.attention_next_action
.as_deref()
.is_some_and(|action| action.contains("finish validation and PR handoff"))
);
}

#[test]
fn live_operator_status_snapshot_surfaces_git_credential_failures() {
let (_temp_dir, config, workflow) = temp_project_layout();
Expand Down
84 changes: 84 additions & 0 deletions apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,90 @@ fn stalled_run_reconciliation_routes_to_needs_attention_without_cleanup() {
}));
}

#[test]
fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktree() {
let (_temp_dir, config, workflow) = temp_project_layout();
let tracker = FakeTracker::new(vec![]);
let state_store = StateStore::open_in_memory().expect("state store should open");
let worktree_manager =
WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root());
let issue = sample_issue("In Progress", &[]);
let run_id = "run-stalled-dirty";
let worktree_path = config.worktree_root().join("PUB-102");

git_status_success(
config.repo_root(),
&["worktree", "add", "-b", "x/pubfi-pub-102", ".worktrees/PUB-102", "main"],
);

fs::write(worktree_path.join("README.md"), "retained partial work\n")
.expect("tracked worktree file should change");

state_store
.record_run_attempt(run_id, &issue.id, 1, "running")
.expect("run attempt should record");
state_store
.upsert_lease("pubfi", &issue.id, run_id, "In Progress")
.expect("lease should record");
state_store
.upsert_worktree(
"pubfi",
&issue.id,
"x/pubfi-pub-102",
&worktree_path.display().to_string(),
)
.expect("worktree mapping should record");

let action = ActiveRunReconciliation {
issue: issue.clone(),
run_attempt: state_store
.run_attempt(run_id)
.expect("run attempt query should succeed")
.expect("run attempt should exist"),
worktree_mapping: state_store
.worktree_for_issue(&issue.id)
.expect("worktree query should succeed"),
disposition: ActiveRunDisposition::Stalled {
idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1),
},
workflow: workflow.clone(),
};

orchestrator::apply_active_run_reconciliation(
&tracker,
&config,
&state_store,
&worktree_manager,
vec![action],
)
.expect("reconciliation should succeed");

assert!(state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none());
assert!(
state_store
.worktree_for_issue(&issue.id)
.expect("worktree lookup should succeed")
.is_some()
);
assert_eq!(
state_store
.run_attempt(run_id)
.expect("run attempt lookup should succeed")
.expect("run attempt should exist")
.status(),
"stalled"
);

let comments = tracker.comments.borrow();

assert!(comments.iter().any(|comment| {
comment.contains("partial_progress_retained")
&& comment.contains("finish validation and PR handoff or reset the patch manually")
&& comment.contains(".worktrees/PUB-102")
}));
assert!(comments.iter().all(|comment| !comment.contains("stalled_run_detected")));
}

#[test]
fn project_reconciliation_routes_orphaned_active_worktree_run_to_needs_attention() {
let (_temp_dir, config, workflow) = temp_project_layout();
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/operator-control-plane.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ outside the local operator surface.
| `Accounts` | Shared Codex account pool and usage table from `~/.codex/decodex/accounts.jsonl` when `[codex.accounts]` is enabled for a project. Account identity can be obscured from the `Account` column header eye without changing the underlying snapshot. Selecting an account writes the global `[codex.accounts].fixed_account` selector in `~/.codex/decodex/config.toml`; clearing it returns all new account-pool runs to balanced account selection. Account display-name rerolls write `[codex.account_names.offsets]` in the same global config so Decodex App and the dashboard share the privacy-preserving names. Theme, sort, and identity-visibility preferences are client-local presentation state. The selector is global and does not pin a project to an account. |
| `Projects` | Fleet-level project table. The section-level filter toggles between active project work and the full registry. Location is its own compact path column and can be obscured from the location header eye. `Activity` shows a relative timestamp or `-`; `Work` is `running/waiting/attention`. It should not duplicate per-lane details already shown below. |
| `Running Lanes` | Active leased or live-executing issue lanes. A lane here is currently owned by this local control plane, or a live process/thread/protocol marker still explains active execution even when the queue lease is not held. It shows issue identity, phase, operation, attempt, queue lease state, execution liveness, thread/protocol status, child-agent activity when captured, timing, branch, and worktree. |
| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after retries, the candidate is partial retained progress and not just a generic retry-budget hold. Running lanes are not repeated as normal intake work. |
| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after stalled reconciliation, failure writeback, or retries, the candidate is partial retained progress and not just a generic stalled or retry-budget hold. Running lanes are not repeated as normal intake work. |
| `Review & Landing` | Retained PR lanes after review handoff. This section owns post-review repair, wait-for-review, ready-to-land, closeout, cleanup, and blocked retained-lane visibility. |
| `Recovery Worktrees` | Retained local worktrees that are not currently owned by `Running Lanes`, `Review & Landing`, or queued attention in `Intake Queue`. This is the cleanup or recovery inbox for recovered paths, retained PR leftovers, and cleanup-only local worktrees. Empty is the normal healthy state. |
| `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Raw local attempts and heartbeat details stay in debug expansion. |
Expand Down