Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/decodex/src/orchestrator/entrypoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,7 @@ fn append_control_plane_project_snapshot(
add_operator_snapshot_warning(snapshot, &warning);
}

snapshot.warning_details.extend(project_snapshot.warning_details);
snapshot.connector_backoffs.extend(project_snapshot.connector_backoffs);
snapshot.accounts.extend(project_snapshot.accounts);
snapshot.active_runs.extend(project_snapshot.active_runs);
Expand Down Expand Up @@ -1375,6 +1376,7 @@ fn empty_control_plane_snapshot(limit: usize) -> OperatorStatusSnapshot {
project_id: String::from("all"),
run_limit: limit,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down
24 changes: 23 additions & 1 deletion apps/decodex/src/orchestrator/operator_dashboard.html
Original file line number Diff line number Diff line change
Expand Up @@ -8562,14 +8562,36 @@ <h2 id="recent-title">Run History</h2>
return notices;
}

function warningNotice(warning) {
function warningDetailsFor(warning, snapshot) {
return (snapshot?.warning_details ?? []).filter((detail) => detail?.warning === warning);
}

function warningNotice(warning, snapshot) {
const details = warningDetailsFor(warning, snapshot);
if (warning === "worktree_hygiene_unavailable" && details.length) {
return {
tone: "warning",
title: "Worktree hygiene unavailable",
copy: details.map(worktreeHygieneWarningCopy).join(" "),
};
}

return {
tone: "warning",
title: "Snapshot warning",
copy: displayToken(warning),
};
}

function worktreeHygieneWarningCopy(detail) {
const project = detail.project_id || "project";
const repo = detail.repo_root ? ` Repo: ${detail.repo_root}.` : "";
const reason = detail.reason || "Worktree hygiene scan failed.";
const nextAction = detail.next_action ? ` ${detail.next_action}` : "";

return `${project}: ${reason}.${repo}${nextAction}`;
}

function renderNoticeDock(notices) {
const hasNotices = notices.length > 0;
nodes.noticeDock.classList.toggle("visible", hasNotices);
Expand Down
80 changes: 75 additions & 5 deletions apps/decodex/src/orchestrator/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -235,12 +235,14 @@ fn build_operator_status_snapshot_with_account_mode(
}

let history_lanes = operator_history_lanes(&active_runs, &recent_runs);
let (worktrees, mut warnings) = operator_status_worktrees(project, state_store)?;
let (worktrees, mut warnings, warning_details) =
operator_status_worktrees(project, state_store)?;
let accounts = codex_account_activity_summaries(project, &mut warnings, account_activity_mode);
let mut snapshot = OperatorStatusSnapshot {
project_id: project.service_id().to_owned(),
run_limit: limit,
warnings,
warning_details,
connector_backoffs: Vec::new(),
projects: vec![OperatorProjectStatus {
project_id: project.service_id().to_owned(),
Expand Down Expand Up @@ -970,7 +972,11 @@ fn project_last_activity_at(snapshot: &OperatorStatusSnapshot) -> Option<String>
fn operator_status_worktrees(
project: &ServiceConfig,
state_store: &StateStore,
) -> crate::prelude::Result<(Vec<OperatorWorktreeStatus>, Vec<String>)> {
) -> crate::prelude::Result<(
Vec<OperatorWorktreeStatus>,
Vec<String>,
Vec<OperatorSnapshotWarningDetail>,
)> {
let mut worktrees = state_store
.list_worktrees(project.service_id())?
.into_iter()
Expand All @@ -991,6 +997,7 @@ fn operator_status_worktrees(
let mut seen_paths =
worktrees.iter().map(|worktree| worktree.worktree_path.clone()).collect::<HashSet<_>>();
let mut warnings = Vec::new();
let mut warning_details = Vec::new();

for issue_identifier in recoverable_worktree_identifiers(project.worktree_root())? {
let worktree_path = project.worktree_root().join(&issue_identifier);
Expand Down Expand Up @@ -1019,7 +1026,13 @@ fn operator_status_worktrees(
});
}

append_merged_worktree_cleanup_debts(project, &mut worktrees, &mut seen_paths, &mut warnings);
append_merged_worktree_cleanup_debts(
project,
&mut worktrees,
&mut seen_paths,
&mut warnings,
&mut warning_details,
);

worktrees.sort_by(|left, right| {
left.issue_id
Expand All @@ -1028,14 +1041,15 @@ fn operator_status_worktrees(
.then_with(|| left.worktree_path.cmp(&right.worktree_path))
});

Ok((worktrees, warnings))
Ok((worktrees, warnings, warning_details))
}

fn append_merged_worktree_cleanup_debts(
project: &ServiceConfig,
worktrees: &mut Vec<OperatorWorktreeStatus>,
seen_paths: &mut HashSet<String>,
warnings: &mut Vec<String>,
warning_details: &mut Vec<OperatorSnapshotWarningDetail>,
) {
let debts = match project_merged_worktree_cleanup_debts(project) {
Ok(debts) => debts,
Expand All @@ -1047,6 +1061,7 @@ fn append_merged_worktree_cleanup_debts(
);

warnings.push(String::from("worktree_hygiene_unavailable"));
warning_details.push(worktree_hygiene_unavailable_warning_detail(project, &error));

return;
},
Expand Down Expand Up @@ -1088,6 +1103,21 @@ fn append_merged_worktree_cleanup_debts(
}
}

fn worktree_hygiene_unavailable_warning_detail(
project: &ServiceConfig,
error: &Report,
) -> OperatorSnapshotWarningDetail {
OperatorSnapshotWarningDetail {
warning: String::from("worktree_hygiene_unavailable"),
project_id: Some(project.service_id().to_owned()),
repo_root: Some(project.repo_root().display().to_string()),
reason: format!("Worktree hygiene scan failed: {error}"),
next_action: Some(String::from(
"Remove the stale project registration or restore the Git checkout before running automation.",
)),
}
}

fn operator_worktree_status_from_cleanup_debt(
debt: MergedWorktreeCleanupDebt,
relative_path: String,
Expand Down Expand Up @@ -4762,7 +4792,7 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String {
output.push_str(&format!("Warnings: {}\n", snapshot.warnings.len()));

if !snapshot.warnings.is_empty() {
output.push_str(&format!("Warning details: {}\n", snapshot.warnings.join(", ")));
output.push_str(&format!("Warning details: {}\n", render_warning_details(snapshot)));
}

output.push_str(&format!("Running lanes: {}\n", snapshot.active_runs.len()));
Expand Down Expand Up @@ -4859,6 +4889,46 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String {
output
}

fn render_warning_details(snapshot: &OperatorStatusSnapshot) -> String {
snapshot
.warnings
.iter()
.flat_map(|warning| {
let details = snapshot
.warning_details
.iter()
.filter(|detail| &detail.warning == warning)
.collect::<Vec<_>>();

if details.is_empty() {
return vec![warning.clone()];
}

details.into_iter().map(format_warning_detail).collect()
})
.collect::<Vec<_>>()
.join("; ")
}

fn format_warning_detail(detail: &OperatorSnapshotWarningDetail) -> String {
let mut parts = vec![detail.warning.clone()];

if let Some(project_id) = detail.project_id.as_deref() {
parts.push(format!("project={project_id}"));
}
if let Some(repo_root) = detail.repo_root.as_deref() {
parts.push(format!("repo_root={repo_root}"));
}

parts.push(format!("reason={}", detail.reason));

if let Some(next_action) = detail.next_action.as_deref() {
parts.push(format!("next_action={next_action}"));
}

parts.join(" ")
}

fn render_queue_explain(
config: &ServiceConfig,
queued_candidates: &[OperatorQueuedIssueStatus],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ fn agent_evidence_snapshot_writes_index_blockers_capsules_and_event_stream() {
project_id: String::from(TEST_SERVICE_ID),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1257,10 +1257,13 @@ fn operator_dashboard_projects_show_compact_activity_work_and_location() {
assert!(response.contains("label: \"sync degraded\""));
assert!(response.contains("label: \"sync degraded\", tone: \"tone-muted\""));
assert!(response.contains("project.connector_state === \"config_error\""));
assert!(response.contains("function warningNotice(warning)"));
assert!(response.contains("function warningDetailsFor(warning, snapshot)"));
assert!(response.contains("function warningNotice(warning, snapshot)"));
assert!(response.contains("title: \"Worktree hygiene unavailable\""));
assert!(response.contains("worktree_hygiene_unavailable"));
assert!(response.contains("copy: displayToken(warning)"));
assert!(!response.contains("title: projectSummary"));
assert!(!response.contains("remove it or re-register the project"));
assert!(response.contains("const nextAction = detail.next_action ?"));
assert!(response.contains("return { label: \"ok\", tone: \"tone-ready\""));
assert!(!response.contains("function projectSyncMeta(project, health)"));
assert!(!response.contains("const connectorCopy = projectSyncMeta(project, health);"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,44 @@ fn operator_status_snapshot_surfaces_merged_dirty_ad_hoc_worktree() {
assert!(error.to_string().contains("Post-land worktree cleanup is pending"));
}

#[test]
fn operator_status_snapshot_explains_unavailable_worktree_hygiene() {
let (_temp_dir, config, _workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");

fs::remove_dir_all(config.repo_root().join(".git"))
.expect("repo metadata should be removable for the fixture");

let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10)
.expect("snapshot should degrade instead of failing");
let detail = snapshot
.warning_details
.iter()
.find(|detail| detail.warning == "worktree_hygiene_unavailable")
.expect("hygiene warning should include operator-facing detail");

assert!(snapshot.warnings.contains(&String::from("worktree_hygiene_unavailable")));
assert_eq!(detail.project_id.as_deref(), Some("pubfi"));

let repo_root = config.repo_root().display().to_string();

assert_eq!(detail.repo_root.as_deref(), Some(repo_root.as_str()));
assert!(detail.reason.contains("not a git repository"));
assert!(
detail
.next_action
.as_deref()
.is_some_and(|action| action.contains("Remove the stale project registration")),
"detail should tell the operator how to clear a stale project registration"
);

let rendered = orchestrator::render_operator_status(&snapshot);

assert!(rendered.contains("project=pubfi"));
assert!(rendered.contains("repo_root="));
assert!(rendered.contains("Remove the stale project registration"));
}

#[test]
fn operator_status_snapshot_updates_owned_merged_worktree_hygiene_without_global_warning() {
let (_temp_dir, config, _workflow) = temp_project_layout();
Expand Down Expand Up @@ -309,6 +347,7 @@ fn idle_operator_status_snapshot_has_no_runtime_or_recovery_noise() {

for field in [
"warnings",
"warning_details",
"active_runs",
"recent_runs",
"history_lanes",
Expand Down
6 changes: 6 additions & 0 deletions apps/decodex/src/orchestrator/tests/operator/status/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ fn operator_status_text_renders_human_readable_sections() {
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down Expand Up @@ -139,6 +140,7 @@ fn operator_status_text_explains_empty_backlog_checks() {
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down Expand Up @@ -173,6 +175,7 @@ fn operator_status_text_surfaces_cleanup_blocker_pr_url() {
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down Expand Up @@ -237,6 +240,7 @@ fn operator_status_text_terminal_run_freshness_uses_terminal_update() {
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down Expand Up @@ -274,6 +278,7 @@ fn operator_status_text_active_run_without_live_activity_does_not_promote_update
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down Expand Up @@ -306,6 +311,7 @@ fn operator_status_text_explains_unleased_live_running_lane() {
project_id: String::from("pubfi"),
run_limit: 10,
warnings: Vec::new(),
warning_details: Vec::new(),
connector_backoffs: Vec::new(),
projects: Vec::new(),
account_control: OperatorCodexAccountControlStatus {
Expand Down
10 changes: 10 additions & 0 deletions apps/decodex/src/orchestrator/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,7 @@ struct OperatorStatusSnapshot {
project_id: String,
run_limit: usize,
warnings: Vec<String>,
warning_details: Vec<OperatorSnapshotWarningDetail>,
connector_backoffs: Vec<OperatorConnectorBackoffStatus>,
projects: Vec<OperatorProjectStatus>,
account_control: OperatorCodexAccountControlStatus,
Expand All @@ -755,6 +756,15 @@ struct OperatorStatusSnapshot {
post_review_lanes: Vec<OperatorPostReviewLaneStatus>,
}

#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
struct OperatorSnapshotWarningDetail {
warning: String,
project_id: Option<String>,
repo_root: Option<String>,
reason: String,
next_action: Option<String>,
}

#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
struct OperatorConnectorBackoffStatus {
project_id: String,
Expand Down
5 changes: 5 additions & 0 deletions docs/reference/operator-control-plane.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,11 @@ and manual retry controls are intentionally not shown. `runActivity.activeRunsCo
marks whether a payload is the complete active-run list; subscription-filtered
payloads set it to `false`, so consumers must not treat a missing run in that payload
as ended.
Snapshot `warnings` remain stable machine-readable tokens. When a warning needs
operator action, snapshots may also include `warning_details` entries with the
affected `project_id`, `repo_root`, reason, and next action; for example, a stale
registered project whose repo path is no longer a Git checkout can explain the bad
project instead of only surfacing `worktree_hygiene_unavailable`.
The stop control signals the recorded child process for that run, marks the local
attempt interrupted, and releases the local queue lease. `ack` is dashboard-local
acknowledgement only. The socket is not a browser connection to Codex app-server,
Expand Down