diff --git a/apps/decodex/src/agent/app_server.rs b/apps/decodex/src/agent/app_server.rs index 571fec6..0acd7a2 100644 --- a/apps/decodex/src/agent/app_server.rs +++ b/apps/decodex/src/agent/app_server.rs @@ -10,6 +10,7 @@ use std::{ fmt::{self, Display, Formatter}, fs, mem, path::{Path, PathBuf}, + process::Command, time::{Duration, Instant}, }; @@ -38,6 +39,7 @@ use crate::{ agent::{ app_server::protocol::LoginAccountResponse, codex_accounts::{CodexAccountLogin, CodexAccountProvider}, + json_rpc, json_rpc::{ AppServerHomePreflightFailure, AppServerOutputTimeout, AppServerProcessEnv, JsonRpcConnection, JsonRpcError, JsonRpcMessage, JsonRpcNotification, JsonRpcRequest, @@ -92,9 +94,39 @@ const PREFLIGHT_CHECK_PLUGINS: &str = "plugins"; const PREFLIGHT_CHECK_MCP: &str = "mcp"; const PREFLIGHT_CHECK_COMPATIBILITY: &str = "compatibility"; const PREFLIGHT_PLUGIN_MARKETPLACE_KIND: &str = "local"; -const APP_SERVER_COMPATIBILITY_SUPPORT_CLAIM: &str = "local_verified_codex_cli_0.136"; +const APP_SERVER_COMPATIBILITY_SUPPORT_CLAIM: &str = "local_verified_codex_cli_exact_versions"; const APP_SERVER_COMPATIBILITY_EVIDENCE: &str = "initialize.userAgent plus successful app-server capability preflight"; +const APP_SERVER_COMPATIBILITY_CAPABILITY_SOURCE: &str = "bounded_app_server_preflight"; +const APP_SERVER_SCHEMA_GENERATE_COMMAND: &str = + "codex app-server generate-json-schema --experimental"; +const APP_SERVER_SCHEMA_PROBE_OUT_DIR: &str = "target/decodex-app-server-schema-check"; +const APP_SERVER_SCHEMA_NOT_CHECKED_REASON: &str = + "normal dispatch uses exact-version allowlist; run decodex probe before expanding support"; +const APP_SERVER_SCHEMA_REQUIRED_MARKERS: &[&str] = &[ + "initialize", + "config/read", + "model/list", + "modelProvider/capabilities/read", + "skills/list", + "plugin/list", + "mcpServerStatus/list", + "thread/start", + "thread/resume", + "turn/start", + "thread/archive", + "command/exec", + "item/tool/call", + "thread/status/changed", + "turn/completed", + "dynamicTools", + "namespace", + "deferLoading", + "inputText", + "marketplaceKinds", +]; +const APP_SERVER_SCHEMA_PROSE_KEYS: &[&str] = + &["$comment", "comment", "description", "examples", "markdownDescription", "title"]; const CODEX_CLI_VERSION_STABLE_0_136_0: &str = "0.136.0"; const CODEX_CLI_VERSION_BETA_0_136_0_ALPHA_2: &str = "0.136.0-alpha.2"; const CODEX_CLI_VERSION_DESKTOP_0_137_0_ALPHA_4: &str = "0.137.0-alpha.4"; @@ -231,6 +263,26 @@ impl AppServerCapabilityPreflightReport { self.compatibility_detail("supported_versions") } + pub(crate) fn compatibility_support_decision(&self) -> Option<&str> { + self.compatibility_detail("support_decision") + } + + pub(crate) fn compatibility_capability_evidence(&self) -> Option<&str> { + self.compatibility_detail("capability_evidence") + } + + pub(crate) fn compatibility_schema_evidence(&self) -> Option<&str> { + self.compatibility_detail("schema_evidence") + } + + pub(crate) fn compatibility_schema_cache(&self) -> Option<&str> { + self.compatibility_detail("schema_cache") + } + + pub(crate) fn compatibility_schema_marker_count(&self) -> Option<&str> { + self.compatibility_detail("schema_marker_count") + } + fn compatibility_check(&self) -> Option<&AppServerCapabilityPreflightCheck> { self.checks.iter().find(|check| check.name == PREFLIGHT_CHECK_COMPATIBILITY) } @@ -470,6 +522,7 @@ pub(crate) struct AppServerRunRequest<'a> { pub(crate) dynamic_tool_handler: Option<&'a dyn DynamicToolHandler>, pub(crate) continuation_guard: Option<&'a dyn TurnContinuationGuard>, pub(crate) codex_account_provider: Option<&'a dyn CodexAccountProvider>, + pub(crate) compatibility_schema_evidence: Option, } pub(crate) struct AppServerThreadArchiveRequest<'a> { @@ -504,6 +557,22 @@ impl CommandExecHealthCheck { } } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AppServerSchemaProbeEvidence { + cache_path: String, + marker_count: usize, + required_markers: Vec<&'static str>, +} +impl AppServerSchemaProbeEvidence { + fn checked(cache_path: String, required_markers: &[&'static str]) -> Self { + Self { + cache_path, + marker_count: required_markers.len(), + required_markers: required_markers.to_vec(), + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct AppServerRunResult { pub(crate) user_agent: String, @@ -1014,6 +1083,15 @@ enum AppServerCapabilityPreflightStatus { Warning, Blocked, } +impl AppServerCapabilityPreflightStatus { + fn as_str(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Warning => "warning", + Self::Blocked => "blocked", + } + } +} #[derive(Clone, Debug, Eq, PartialEq)] enum AppServerCapabilityPreflightFailureKind { @@ -1127,6 +1205,7 @@ pub(crate) fn probe_app_server( ) -> crate::prelude::Result { let state_store = StateStore::open_in_memory()?; let probe_tool_handler = ProbeDynamicToolHandler; + let schema_evidence = probe_app_server_schema(&AppServerProcessEnv::default())?; let result = execute_app_server_run( &AppServerRunRequest { project_id: String::from("probe"), @@ -1149,6 +1228,7 @@ pub(crate) fn probe_app_server( dynamic_tool_handler: Some(&probe_tool_handler), continuation_guard: None, codex_account_provider: None, + compatibility_schema_evidence: Some(schema_evidence), }, &state_store, )?; @@ -1185,7 +1265,16 @@ fn preflight_check_blocker_summary(check: &AppServerCapabilityPreflightCheck) -> summary.push_str(error); } if check.name == PREFLIGHT_CHECK_COMPATIBILITY { - for detail_name in ["codex_cli_version", "user_agent", "supported_versions"] { + for detail_name in [ + "support_decision", + "parsed_version", + "codex_cli_version", + "user_agent", + "supported_versions", + "capability_evidence", + "schema_evidence", + "schema_cache", + ] { if let Some(detail) = check.details.get(detail_name) { summary.push(' '); summary.push_str(detail_name); @@ -2297,6 +2386,7 @@ fn execute_app_server_run_inner( &request.cwd, &initialize_response.user_agent, request.allow_unverified_codex, + request.compatibility_schema_evidence.as_ref(), )?; write_activity_marker_best_effort_for_request(request); @@ -2389,6 +2479,7 @@ fn run_app_server_capability_preflight( cwd: &str, user_agent: &str, allow_unverified_codex: bool, + compatibility_schema_evidence: Option<&AppServerSchemaProbeEvidence>, ) -> crate::prelude::Result { let mut report = AppServerCapabilityPreflightReport::new(); let config = preflight_request(recorder, &report, "config/read", || { @@ -2447,7 +2538,12 @@ fn run_app_server_capability_preflight( } if !report.has_blockers() { - record_app_server_compatibility_guard(&mut report, user_agent, allow_unverified_codex); + record_app_server_compatibility_guard( + &mut report, + user_agent, + allow_unverified_codex, + compatibility_schema_evidence, + ); } record_app_server_preflight_report(recorder, &report)?; @@ -2867,25 +2963,185 @@ fn record_mcp_preflight_degraded(report: &mut AppServerCapabilityPreflightReport ); } +fn probe_app_server_schema( + process_env: &AppServerProcessEnv, +) -> crate::prelude::Result { + let out_dir = PathBuf::from(APP_SERVER_SCHEMA_PROBE_OUT_DIR); + + if out_dir.exists() { + fs::remove_dir_all(&out_dir)?; + } + + if let Some(parent) = out_dir.parent() { + fs::create_dir_all(parent)?; + } + + let mut command = Command::new(json_rpc::app_server_command_program()); + + command.args(["app-server", "generate-json-schema", "--experimental", "--out"]); + command.arg(&out_dir); + process_env.apply_to(&mut command)?; + + let output = command.output()?; + + if !output.status.success() { + eyre::bail!( + "`{APP_SERVER_SCHEMA_GENERATE_COMMAND}` failed with status {}: stdout={} stderr={}", + output.status, + command_output_excerpt(&output.stdout), + command_output_excerpt(&output.stderr) + ); + } + + validate_generated_app_server_schema(&out_dir)?; + + Ok(AppServerSchemaProbeEvidence::checked( + APP_SERVER_SCHEMA_PROBE_OUT_DIR.to_owned(), + APP_SERVER_SCHEMA_REQUIRED_MARKERS, + )) +} + +fn validate_generated_app_server_schema(out_dir: &Path) -> crate::prelude::Result<()> { + let mut marker_presence = APP_SERVER_SCHEMA_REQUIRED_MARKERS + .iter() + .map(|marker| (*marker, false)) + .collect::>(); + let schema_file_count = collect_schema_markers(out_dir, &mut marker_presence)?; + + if schema_file_count == 0 { + eyre::bail!( + "Generated app-server schema directory `{}` contained no JSON files.", + out_dir.display() + ); + } + + let missing_markers = marker_presence + .iter() + .filter_map(|(marker, present)| (!*present).then_some(*marker)) + .collect::>(); + + if !missing_markers.is_empty() { + eyre::bail!( + "Generated app-server schema was missing required Decodex markers: {}", + missing_markers.join(", ") + ); + } + + Ok(()) +} + +fn collect_schema_markers( + path: &Path, + marker_presence: &mut BTreeMap<&'static str, bool>, +) -> crate::prelude::Result { + let mut json_file_count = 0; + + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + json_file_count += collect_schema_markers(&path, marker_presence)?; + } else if path.extension().and_then(|extension| extension.to_str()) == Some("json") { + let schema = fs::read_to_string(&path)?; + let value: Value = serde_json::from_str(&schema)?; + + json_file_count += 1; + + record_schema_markers_from_value(&value, marker_presence); + } + } + + Ok(json_file_count) +} + +fn record_schema_markers_from_value( + value: &Value, + marker_presence: &mut BTreeMap<&'static str, bool>, +) { + match value { + Value::Object(object) => + for (key, value) in object { + if schema_prose_key(key) { + continue; + } + + record_schema_marker_from_text(key, marker_presence); + record_schema_markers_from_value(value, marker_presence); + }, + Value::Array(values) => + for value in values { + record_schema_markers_from_value(value, marker_presence); + }, + Value::String(value) => record_schema_marker_from_text(value, marker_presence), + Value::Null | Value::Bool(_) | Value::Number(_) => {}, + } +} + +fn schema_prose_key(key: &str) -> bool { + APP_SERVER_SCHEMA_PROSE_KEYS.contains(&key) +} + +fn record_schema_marker_from_text(value: &str, marker_presence: &mut BTreeMap<&'static str, bool>) { + for (marker, present) in marker_presence { + if value.contains(*marker) { + *present = true; + } + } +} + +fn command_output_excerpt(output: &[u8]) -> String { + let text = String::from_utf8_lossy(output); + let trimmed = text.trim(); + let excerpt = trimmed.chars().take(1_000).collect::(); + + if excerpt.is_empty() { String::from("") } else { excerpt } +} + fn record_app_server_compatibility_guard( report: &mut AppServerCapabilityPreflightReport, user_agent: &str, allow_unverified_codex: bool, + schema_evidence: Option<&AppServerSchemaProbeEvidence>, ) { let codex_cli_version = codex_cli_version_from_user_agent(user_agent); + let matched_supported_version = supported_codex_cli_version_from_user_agent(user_agent); let mut details = BTreeMap::new(); details.insert(String::from("user_agent"), user_agent.to_owned()); + details.insert( + String::from("parsed_version"), + codex_cli_version + .as_deref() + .map(|version| format!("codex-cli {version}")) + .unwrap_or_else(|| String::from("unparsed")), + ); details.insert(String::from("supported_versions"), supported_codex_cli_versions_display()); details.insert(String::from("allow_unverified_codex"), allow_unverified_codex.to_string()); details .insert(String::from("support_claim"), APP_SERVER_COMPATIBILITY_SUPPORT_CLAIM.to_owned()); + details.insert( + String::from("support_decision"), + compatibility_support_decision( + codex_cli_version.as_deref(), + matched_supported_version, + allow_unverified_codex, + ) + .to_owned(), + ); details.insert(String::from("evidence"), APP_SERVER_COMPATIBILITY_EVIDENCE.to_owned()); + details.insert(String::from("capability_evidence"), compatibility_capability_evidence(report)); + details.insert( + String::from("capability_evidence_source"), + APP_SERVER_COMPATIBILITY_CAPABILITY_SOURCE.to_owned(), + ); + + record_schema_evidence_details(&mut details, schema_evidence); if let Some(version) = codex_cli_version.as_deref() { details.insert(String::from("codex_cli_version"), format!("codex-cli {version}")); } - if let Some(version) = supported_codex_cli_version_from_user_agent(user_agent) { + if let Some(version) = matched_supported_version { details.insert(String::from("matched_supported_version"), format!("codex-cli {version}")); report.push_ok( PREFLIGHT_CHECK_COMPATIBILITY, @@ -2895,10 +3151,10 @@ fn record_app_server_compatibility_guard( } else if allow_unverified_codex { details.insert(String::from("override"), String::from("allow_unverified_codex")); report.push_warning( - PREFLIGHT_CHECK_COMPATIBILITY, - "app-server userAgent is outside the locally verified Codex CLI capability range; continuing because unverified Codex versions are explicitly allowed.", - details, - ); + PREFLIGHT_CHECK_COMPATIBILITY, + "app-server userAgent is outside the locally verified Codex CLI capability range; continuing because unverified Codex versions are explicitly allowed.", + details, + ); } else { report.push_blocked( PREFLIGHT_CHECK_COMPATIBILITY, @@ -2908,6 +3164,56 @@ fn record_app_server_compatibility_guard( } } +fn compatibility_support_decision( + parsed_version: Option<&str>, + matched_supported_version: Option<&str>, + allow_unverified_codex: bool, +) -> &'static str { + match (parsed_version, matched_supported_version, allow_unverified_codex) { + (_, Some(_), _) => "supported_exact_version", + (Some(_), None, true) => "unverified_allowed_by_override", + (None, None, true) => "unparsed_user_agent_allowed_by_override", + (Some(_), None, false) => "unsupported_unverified_version", + (None, None, false) => "unsupported_unparsed_user_agent", + } +} + +fn compatibility_capability_evidence(report: &AppServerCapabilityPreflightReport) -> String { + let checks = report + .checks + .iter() + .filter(|check| check.name != PREFLIGHT_CHECK_COMPATIBILITY) + .map(|check| format!("{}={}", check.name, check.status.as_str())) + .collect::>(); + + if checks.is_empty() { String::from("none") } else { checks.join(", ") } +} + +fn record_schema_evidence_details( + details: &mut BTreeMap, + schema_evidence: Option<&AppServerSchemaProbeEvidence>, +) { + if let Some(schema_evidence) = schema_evidence { + details.insert(String::from("schema_evidence"), String::from("checked")); + details + .insert(String::from("schema_command"), APP_SERVER_SCHEMA_GENERATE_COMMAND.to_owned()); + details.insert(String::from("schema_cache"), schema_evidence.cache_path.clone()); + details + .insert(String::from("schema_marker_count"), schema_evidence.marker_count.to_string()); + details.insert(String::from("schema_markers"), schema_evidence.required_markers.join(", ")); + } else { + details.insert(String::from("schema_evidence"), String::from("not_checked")); + details.insert( + String::from("schema_evidence_reason"), + APP_SERVER_SCHEMA_NOT_CHECKED_REASON.to_owned(), + ); + details.insert( + String::from("schema_required_markers"), + APP_SERVER_SCHEMA_REQUIRED_MARKERS.join(", "), + ); + } +} + fn supported_codex_cli_version_from_user_agent(user_agent: &str) -> Option<&'static str> { let codex_cli_version = codex_cli_version_from_user_agent(user_agent)?; diff --git a/apps/decodex/src/agent/app_server/tests.rs b/apps/decodex/src/agent/app_server/tests.rs index 380f72d..978aabb 100644 --- a/apps/decodex/src/agent/app_server/tests.rs +++ b/apps/decodex/src/agent/app_server/tests.rs @@ -14,10 +14,11 @@ use tempfile::TempDir; use crate::{ agent::{ app_server::{ - AppServerCapabilityPreflightFailure, AppServerCapabilityPreflightReport, - AppServerDynamicToolFailure, AppServerRunResult, AppServerThreadArchiveRequest, - AppServerTurnFailure, CommandExecHealthCheck, CommandExecResponse, - EffectiveThreadConfig, InitializeResponse, ModelProviderCapabilitiesReadResponse, + APP_SERVER_SCHEMA_REQUIRED_MARKERS, AppServerCapabilityPreflightFailure, + AppServerCapabilityPreflightReport, AppServerDynamicToolFailure, AppServerRunResult, + AppServerSchemaProbeEvidence, AppServerThreadArchiveRequest, AppServerTurnFailure, + CommandExecHealthCheck, CommandExecResponse, EffectiveThreadConfig, InitializeResponse, + ModelProviderCapabilitiesReadResponse, PREFLIGHT_CHECK_CONFIG, PREFLIGHT_CHECK_MODEL, PluginListResponse, ProbeDynamicToolHandler, REQUEST_TIMEOUT, RequestWaitPhase, RunRecorder, RuntimeConfigSummary, SUPPORTED_CODEX_CLI_VERSION_DISPLAY_ORDER, SkillsListResponse, TurnContinuationGuard, UserInput, @@ -286,6 +287,11 @@ fn probe_result_shape_is_stable() { #[test] fn app_server_compatibility_guard_accepts_current_verified_codex_surfaces() { + let schema_evidence = AppServerSchemaProbeEvidence::checked( + String::from("target/decodex-app-server-schema-check"), + APP_SERVER_SCHEMA_REQUIRED_MARKERS, + ); + for (user_agent, expected_codex_cli_version) in [ ("codex-cli 0.136.0", "codex-cli 0.136.0"), ("codex-cli 0.136.0-alpha.2", "codex-cli 0.136.0-alpha.2"), @@ -309,10 +315,22 @@ fn app_server_compatibility_guard_accepts_current_verified_codex_surfaces() { ] { let mut report = AppServerCapabilityPreflightReport::new(); - super::record_app_server_compatibility_guard(&mut report, user_agent, false); + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + + super::record_app_server_compatibility_guard( + &mut report, + user_agent, + false, + Some(&schema_evidence), + ); assert!(!report.has_blockers(), "{user_agent} should be supported"); assert_eq!(report.compatibility_status(), "supported"); + assert_eq!(report.compatibility_support_decision(), Some("supported_exact_version")); assert_eq!( report.compatibility_codex_cli_version(), Some(expected_codex_cli_version), @@ -322,6 +340,12 @@ fn app_server_compatibility_guard_accepts_current_verified_codex_surfaces() { report.compatibility_supported_versions(), Some("codex-cli 0.136.0, codex-cli 0.136.0-alpha.2, codex-cli 0.137.0-alpha.4") ); + assert_eq!(report.compatibility_capability_evidence(), Some("config=ok")); + assert_eq!(report.compatibility_schema_evidence(), Some("checked")); + assert_eq!( + report.compatibility_schema_cache(), + Some("target/decodex-app-server-schema-check") + ); } } @@ -336,32 +360,224 @@ fn app_server_compatibility_guard_rejects_unverified_codex_surfaces() { ] { let mut report = AppServerCapabilityPreflightReport::new(); - super::record_app_server_compatibility_guard(&mut report, user_agent, false); + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + + super::record_app_server_compatibility_guard(&mut report, user_agent, false, None); assert!(report.has_blockers(), "{user_agent} should be outside support"); assert_eq!(report.compatibility_status(), "unsupported"); - assert_eq!(report.checks()[0].name, "compatibility"); - assert_eq!(report.checks()[0].status, super::AppServerCapabilityPreflightStatus::Blocked); - assert!(report.checks()[0].summary.contains("outside")); + assert!( + report + .compatibility_support_decision() + .is_some_and(|decision| decision.starts_with("unsupported_")), + "{user_agent} should record an unsupported decision" + ); + assert_eq!(report.compatibility_schema_evidence(), Some("not_checked")); + assert_eq!(report.checks()[1].name, "compatibility"); + assert_eq!(report.checks()[1].status, super::AppServerCapabilityPreflightStatus::Blocked); + assert!(report.checks()[1].summary.contains("outside")); + assert!(report.blocker_summary().contains("support_decision=unsupported_")); } } #[test] fn app_server_compatibility_guard_allows_unverified_codex_when_requested() { + let schema_evidence = AppServerSchemaProbeEvidence::checked( + String::from("target/decodex-app-server-schema-check"), + APP_SERVER_SCHEMA_REQUIRED_MARKERS, + ); let mut report = AppServerCapabilityPreflightReport::new(); - super::record_app_server_compatibility_guard(&mut report, "codex-cli 0.138.0-alpha.1", true); + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + + super::record_app_server_compatibility_guard( + &mut report, + "codex-cli 0.138.0-alpha.1", + true, + Some(&schema_evidence), + ); assert!(!report.has_blockers()); assert_eq!(report.compatibility_status(), "unverified_allowed"); - assert_eq!(report.checks()[0].name, "compatibility"); - assert_eq!(report.checks()[0].status, super::AppServerCapabilityPreflightStatus::Warning); + assert_eq!(report.compatibility_support_decision(), Some("unverified_allowed_by_override")); + assert_eq!(report.compatibility_schema_evidence(), Some("checked")); + assert_eq!(report.compatibility_capability_evidence(), Some("config=ok")); + assert_eq!(report.checks()[1].name, "compatibility"); + assert_eq!(report.checks()[1].status, super::AppServerCapabilityPreflightStatus::Warning); assert_eq!( - report.checks()[0].details.get("override").map(String::as_str), + report.checks()[1].details.get("override").map(String::as_str), Some("allow_unverified_codex") ); } +#[test] +fn app_server_compatibility_guard_keeps_unverified_version_blocked_with_schema_evidence() { + let schema_evidence = AppServerSchemaProbeEvidence::checked( + String::from("target/decodex-app-server-schema-check"), + APP_SERVER_SCHEMA_REQUIRED_MARKERS, + ); + let mut report = AppServerCapabilityPreflightReport::new(); + + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + + super::record_app_server_compatibility_guard( + &mut report, + "codex-cli 0.137.0", + false, + Some(&schema_evidence), + ); + + assert!(report.has_blockers()); + assert_eq!(report.compatibility_status(), "unsupported"); + assert_eq!(report.compatibility_support_decision(), Some("unsupported_unverified_version")); + assert_eq!(report.compatibility_schema_evidence(), Some("checked")); + assert_eq!(report.compatibility_capability_evidence(), Some("config=ok")); +} + +#[test] +fn app_server_compatibility_guard_records_capability_and_schema_evidence_diagnostics() { + let schema_evidence = AppServerSchemaProbeEvidence::checked( + String::from("target/decodex-app-server-schema-check"), + APP_SERVER_SCHEMA_REQUIRED_MARKERS, + ); + let mut report = AppServerCapabilityPreflightReport::new(); + + report.push_ok( + PREFLIGHT_CHECK_CONFIG, + "config/read returned effective runtime configuration.", + BTreeMap::new(), + ); + report.push_ok( + PREFLIGHT_CHECK_MODEL, + "model/list returned an executable model selection.", + BTreeMap::new(), + ); + + super::record_app_server_compatibility_guard( + &mut report, + "codex-cli 0.136.0", + false, + Some(&schema_evidence), + ); + + let serialized = serde_json::to_value(&report).expect("report should serialize"); + let compatibility = &serialized["checks"][2]; + + assert_eq!(compatibility["name"], "compatibility"); + assert_eq!(compatibility["status"], "ok"); + assert_eq!(compatibility["details"]["user_agent"], "codex-cli 0.136.0"); + assert_eq!(compatibility["details"]["parsed_version"], "codex-cli 0.136.0"); + assert_eq!(compatibility["details"]["support_decision"], "supported_exact_version"); + assert_eq!(compatibility["details"]["capability_evidence"], "config=ok, model=ok"); + assert_eq!( + compatibility["details"]["capability_evidence_source"], + "bounded_app_server_preflight" + ); + assert_eq!(compatibility["details"]["schema_evidence"], "checked"); + assert_eq!(compatibility["details"]["schema_cache"], "target/decodex-app-server-schema-check"); + assert_eq!( + compatibility["details"]["schema_marker_count"], + super::APP_SERVER_SCHEMA_REQUIRED_MARKERS.len().to_string() + ); + assert!( + compatibility["details"]["schema_markers"] + .as_str() + .expect("schema markers should serialize") + .contains("dynamicTools") + ); +} + +#[test] +fn generated_schema_marker_validation_accepts_required_markers() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let schema_path = temp_dir.path().join("app-server.schema.json"); + + fs::write( + &schema_path, + serde_json::json!({ + "description": "Decodex app-server compatibility fixture.", + "requiredMarkers": super::APP_SERVER_SCHEMA_REQUIRED_MARKERS, + "properties": { + "dynamicTools": { + "properties": { + "namespace": { "type": "string" }, + "deferLoading": { "type": "boolean" } + } + }, + "marketplaceKinds": { "type": "array" }, + "type": { "const": "inputText" } + } + }) + .to_string(), + ) + .expect("schema fixture should write"); + super::validate_generated_app_server_schema(temp_dir.path()) + .expect("required markers should pass schema validation"); +} + +#[test] +fn generated_schema_marker_validation_rejects_missing_markers() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let schema_path = temp_dir.path().join("app-server.schema.json"); + + fs::write( + &schema_path, + serde_json::json!({ + "methods": ["initialize"] + }) + .to_string(), + ) + .expect("schema fixture should write"); + + let error = super::validate_generated_app_server_schema(temp_dir.path()) + .expect_err("missing markers should fail schema validation"); + + assert!(error.to_string().contains("missing required Decodex markers")); + assert!(error.to_string().contains("turn/start")); + assert!(error.to_string().contains("marketplaceKinds")); +} + +#[test] +fn generated_schema_marker_validation_rejects_prose_only_markers() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let schema_path = temp_dir.path().join("app-server.schema.json"); + let prose_markers = APP_SERVER_SCHEMA_REQUIRED_MARKERS.join(", "); + + fs::write( + &schema_path, + serde_json::json!({ + "description": prose_markers.clone(), + "$comment": "Compatibility prose, not protocol structure.", + "properties": { + "documentationOnly": { + "description": prose_markers + } + } + }) + .to_string(), + ) + .expect("schema fixture should write"); + + let error = super::validate_generated_app_server_schema(temp_dir.path()) + .expect_err("prose-only markers should fail schema validation"); + + assert!(error.to_string().contains("missing required Decodex markers")); + assert!(error.to_string().contains("initialize")); + assert!(error.to_string().contains("marketplaceKinds")); +} + #[test] fn app_server_compatibility_versions_match_spec_table() { let spec_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -491,6 +707,7 @@ fn minimal_run_request<'a>() -> super::AppServerRunRequest<'a> { dynamic_tool_handler: None, continuation_guard: None, codex_account_provider: None, + compatibility_schema_evidence: None, } } @@ -2098,6 +2315,7 @@ fn live_app_server_resume_round_trip_updates_marker_and_state() { dynamic_tool_handler: Some(&handler), continuation_guard: Some(&guard), codex_account_provider: None, + compatibility_schema_evidence: None, }, &first_state_store, ) @@ -2144,6 +2362,7 @@ fn live_app_server_resume_round_trip_updates_marker_and_state() { dynamic_tool_handler: Some(&handler), continuation_guard: None, codex_account_provider: None, + compatibility_schema_evidence: None, }, &resumed_state_store, ) diff --git a/apps/decodex/src/agent/json_rpc.rs b/apps/decodex/src/agent/json_rpc.rs index a654d64..e942ba0 100644 --- a/apps/decodex/src/agent/json_rpc.rs +++ b/apps/decodex/src/agent/json_rpc.rs @@ -631,6 +631,10 @@ enum AppServerCodexHomePolicy { Explicit(ResolvedAppServerCodexHomeEnv), } +pub(crate) fn app_server_command_program() -> PathBuf { + app_server_command_program_from_env(env::var_os("PATH"), env::var_os("HOME")) +} + fn resolve_shared_codex_home_env() -> crate::prelude::Result { resolve_shared_codex_home_env_from_home(env::var_os("HOME")) } @@ -701,10 +705,6 @@ fn configure_app_server_command( process_env.apply_to(command) } -fn app_server_command_program() -> PathBuf { - app_server_command_program_from_env(env::var_os("PATH"), env::var_os("HOME")) -} - fn app_server_command_program_from_env( path_env: Option, home: Option, diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs index 3e875fe..d50ea63 100644 --- a/apps/decodex/src/cli.rs +++ b/apps/decodex/src/cli.rs @@ -1226,10 +1226,15 @@ impl ProbeCommand { let report = agent::probe_app_server(&self.transport, self.allow_unverified_codex)?; println!( - "probe ok: compatibility={} codex_version={} supported_versions=\"{}\" thread={} turn={} events={} output={}", + "probe ok: compatibility={} support_decision={} codex_version={} supported_versions=\"{}\" capability_evidence=\"{}\" schema_evidence={} schema_cache={} schema_marker_count={} thread={} turn={} events={} output={}", report.capability_preflight.compatibility_status(), + report.capability_preflight.compatibility_support_decision().unwrap_or("unknown"), report.capability_preflight.compatibility_codex_cli_version().unwrap_or("unknown"), report.capability_preflight.compatibility_supported_versions().unwrap_or("unknown"), + report.capability_preflight.compatibility_capability_evidence().unwrap_or("unknown"), + report.capability_preflight.compatibility_schema_evidence().unwrap_or("unknown"), + report.capability_preflight.compatibility_schema_cache().unwrap_or("none"), + report.capability_preflight.compatibility_schema_marker_count().unwrap_or("0"), report.thread_id, report.turn_id, report.event_count, diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 74a5b23..ce599b4 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -582,11 +582,11 @@ where issue_run, &review_context, ), - max_turns: workflow.frontmatter().execution().max_turns(), - timeout: ACTIVE_RUN_IDLE_TIMEOUT, - process_env: agent_git_credentials.process_env().clone(), - allow_unverified_codex, - continuation_user_input: Some(build_continuation_user_input( + max_turns: workflow.frontmatter().execution().max_turns(), + timeout: ACTIVE_RUN_IDLE_TIMEOUT, + process_env: agent_git_credentials.process_env().clone(), + allow_unverified_codex, + continuation_user_input: Some(build_continuation_user_input( &issue_run.issue, workflow, issue_run.dispatch_mode, @@ -603,6 +603,7 @@ where codex_account_provider: codex_account_pool .as_ref() .map(|pool| pool as &dyn CodexAccountProvider), + compatibility_schema_evidence: None, }, state_store, ) diff --git a/docs/spec/app-server.md b/docs/spec/app-server.md index 203c6dc..634392c 100644 --- a/docs/spec/app-server.md +++ b/docs/spec/app-server.md @@ -53,6 +53,22 @@ range only when all of these are true: - The executable Decodex compatibility guard reports `compatibility=supported` for the initialized app-server `userAgent` after the capability preflight succeeds. +Support has three distinct evidence layers: + +- Exact-version support: the initialized app-server `userAgent` must parse to one of + the locally verified Codex CLI versions in the executable allowlist. This is the + only layer that can make the dispatch guard pass without the explicit dogfood + override described below. +- Capability evidence: the bounded runtime preflight records the app-server methods + and inventories Decodex actually checked before the guard decision. This evidence + explains why the local runtime looked usable, but it does not authorize an + unlisted version. +- Schema evidence: `decodex probe stdio://` regenerates the local schema cache and + records which required schema markers were checked. Retained dispatch records when + schema evidence was not checked in that dispatch path. Schema evidence is required + before expanding the allowlist, but it does not make an unsupported version pass by + itself. + As of the 2026-06-03 self-compatibility pass, the verified local range is: | Codex surface | Version | Evidence | @@ -78,15 +94,20 @@ and the Codex Beta app bundle's `codex-cli 0.131.0-alpha.9`. Treat those as hist compatibility evidence, not the current upgrade target. `decodex probe stdio://` exposes the executable guard in its success line, including -`compatibility=supported`, the observed `codex_version`, and the executable -`supported_versions` list. During retained-lane dispatch, the same compatibility check -runs after the bounded capability preflight and before `thread/start` or -`thread/resume`; an app-server identity outside the locally verified list is a -pre-dispatch app-server preflight blocker rather than a promptable agent turn. +`compatibility=supported`, `support_decision=supported_exact_version`, the observed +`codex_version`, the executable `supported_versions` list, `capability_evidence`, +`schema_evidence`, `schema_cache`, and `schema_marker_count`; the private preflight +report also records the full `schema_markers` list. During retained-lane dispatch, +the same compatibility check runs after the bounded capability preflight and before +`thread/start` or `thread/resume`; an app-server identity outside the locally +verified list is a pre-dispatch app-server preflight blocker rather than a promptable +agent turn. Unsupported newer versions must report a structured unsupported decision +such as `unsupported_unverified_version` or `unsupported_unparsed_user_agent`. Operators may pass `--allow-unverified-codex` to `decodex run`, `decodex serve`, or `decodex probe` when deliberately dogfooding a development Codex build. This changes only the unsupported compatibility identity from a blocker to a warning with -`compatibility=unverified_allowed`; all other capability preflight blockers remain +`compatibility=unverified_allowed` and a support decision such as +`unverified_allowed_by_override`; all other capability preflight blockers remain fail-closed. Current upstream Codex signals are beyond the local support claim whenever they are @@ -97,6 +118,30 @@ probed locally. In that case Decodex must not force an upgrade. It should keep r the latest locally verified Codex surface, route the upstream change through Radar review, regenerate the app-server schema, run `decodex probe stdio://`, and only then promote the new Codex version or protocol shape into this compatibility range. +Latest upstream Codex remains unsupported until that promotion happens, even when a +local capability or schema check looks promising. + +To expand support for a new upstream app-server version: + +1. Install or select the target Codex binary locally without replacing the last known + verified runtime used by active lanes. +2. Run `codex app-server generate-json-schema --experimental --out + target/decodex-app-server-schema-check`. `decodex probe stdio://` uses the same + schema cache path, but an unlisted version must still fail the compatibility guard + until the allowlist is deliberately updated. +3. Confirm the generated schema contains every required marker in this spec: + `initialize`, `thread/start`, `thread/resume`, `turn/start`, `thread/archive`, + `command/exec`, bounded preflight methods, `item/tool/call`, dynamic tool + `namespace`, dynamic tool `deferLoading`, `inputText`, and + `PluginListParams.marketplaceKinds`. +4. Update the executable allowlist locally and add or update compatibility tests for + the target exact version and nearby unsupported versions. +5. Run `decodex probe stdio://` and require `PROBE_OK`, `compatibility=supported`, + `support_decision=supported_exact_version`, `capability_evidence`, and + `schema_evidence=checked` for the target version. +6. Update this table in the same change as the executable allowlist and compatibility + tests. Do not document a new version as supported before the local guard and probe + output agree. ## Implementation guidance