From 672a4796a1f8be8c6fe17722e1178201c0821354 Mon Sep 17 00:00:00 2001 From: yh928 Date: Sun, 2 Aug 2026 21:33:07 +0900 Subject: [PATCH 1/2] feat(composio): tell the agent what a toolkit's actions hand back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the agent reads before calling is input-side: the tool catalogue, the parameter schema, the toolkit description, the contract the gate delivers. Nothing states what comes back. So when a list action returns records keyed by id, the model has no statement that the id is the handle for the detail it actually wanted — and it re-issues the same list call. Observed live against Gmail. `toolkit_result_notes(slug)` adds that one missing sentence, carried on `ConnectedIntegration::result_notes` and rendered per connected toolkit in the integrations-agent prompt. Kept separate from `description` because only the agent that calls the actions needs it. Two entries, gmail and slack, and the rule for adding a third is in the function doc: state only what this repository establishes — which curated action carries which identifier, and which action that identifier is the argument for. A toolkit we pass through unreshaped has no such ground truth and gets no entry, because a guess about a response is worse here than silence. The doc also forbids reciting field-by-field record shapes, with the reason: both Composio dispatch routes prefer the backend's rendered `markdownFormatted` body and fall back to the JSON envelope only when it is absent, so the reshapes in `providers/*/post_process.rs` describe just one of two possible renderings. An earlier draft of this function recited their keys and thereby told the model that every Gmail read action answers with a markdown body, when only `GMAIL_FETCH_EMAILS` carries one. The gmail entry leads with the distinction that failure turned on: `GMAIL_LIST_THREADS` answers with ids and a snippet, never a body, so a thread whose snippet looks right still has to be read. A test pins that every action slug these notes name is one the toolkit actually exposes — a note pointing at a renamed or dropped action is worse than no note. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SRSNnqQsokuGmkbpLoLCGy --- src/openhuman/agent/harness/session/tests.rs | 5 + .../harness/subagent_runner/ops/runner.rs | 1 + .../orchestration/tools/tools_e2e_tests.rs | 1 + src/openhuman/agent/profiles/paths.rs | 1 + src/openhuman/agent/prompts/types.rs | 11 ++ .../registry/agents/context_scout/prompt.rs | 1 + .../agents/integrations_agent/prompt.rs | 63 ++++++++++ .../registry/agents/orchestrator/prompt.rs | 7 ++ .../channels/runtime/dispatch/routing.rs | 1 + src/openhuman/flows/tinyflows/caps/ops.rs | 1 + .../composio/connected_integrations.rs | 4 + .../integrations/composio/ops_tests.rs | 1 + .../sync/composio/providers/descriptions.rs | 117 +++++++++++++++++- .../memory/sync/composio/providers/mod.rs | 2 +- src/openhuman/tools/orchestrator_tools.rs | 3 + ...gent_harness_leftovers_raw_coverage_e2e.rs | 1 + .../agent_large_round25_raw_coverage_e2e.rs | 1 + .../raw_coverage/composio_raw_coverage_e2e.rs | 4 + .../inference_agent_raw_coverage_e2e.rs | 1 + ...ools_approval_channels_raw_coverage_e2e.rs | 1 + 20 files changed, 225 insertions(+), 2 deletions(-) diff --git a/src/openhuman/agent/harness/session/tests.rs b/src/openhuman/agent/harness/session/tests.rs index 5464898713..3dae5395f0 100644 --- a/src/openhuman/agent/harness/session/tests.rs +++ b/src/openhuman/agent/harness/session/tests.rs @@ -277,6 +277,7 @@ fn set_connected_integrations_marks_session_initialized_and_updates_hash() { agent.set_connected_integrations(vec![ crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email".into(), tools: vec![], @@ -306,6 +307,7 @@ fn refresh_delegation_tools_updates_schema_even_when_tool_arc_is_shared() { let mut agent = build_minimal_agent_with_definition_name(Some("orchestrator")); agent.set_connected_integrations(vec![ crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email".into(), tools: vec![], @@ -326,6 +328,7 @@ fn refresh_delegation_tools_updates_schema_even_when_tool_arc_is_shared() { let _shared_tools = agent.tools_arc(); agent.set_connected_integrations(vec![ crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email".into(), tools: vec![], @@ -335,6 +338,7 @@ fn refresh_delegation_tools_updates_schema_even_when_tool_arc_is_shared() { non_active_status: None, }, crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: "notion".into(), description: "Docs".into(), tools: vec![], @@ -370,6 +374,7 @@ fn refresh_delegation_tools_no_duplicate_specs_across_shared_arc_connects() { let conn = |slug: &str, desc: &str| crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: slug.into(), description: desc.into(), tools: vec![], diff --git a/src/openhuman/agent/harness/subagent_runner/ops/runner.rs b/src/openhuman/agent/harness/subagent_runner/ops/runner.rs index 439cde2ca2..30c22ad081 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops/runner.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops/runner.rs @@ -866,6 +866,7 @@ async fn run_typed_mode( } }; let integration = crate::openhuman::agent::context::prompt::ConnectedIntegration { + result_notes: None, toolkit: cached_integration.toolkit.clone(), description: cached_integration.description.clone(), tools: fresh_actions, diff --git a/src/openhuman/agent/orchestration/tools/tools_e2e_tests.rs b/src/openhuman/agent/orchestration/tools/tools_e2e_tests.rs index 9bf552e940..316871c889 100644 --- a/src/openhuman/agent/orchestration/tools/tools_e2e_tests.rs +++ b/src/openhuman/agent/orchestration/tools/tools_e2e_tests.rs @@ -346,6 +346,7 @@ async fn skill_delegation_tool_runs_integrations_agent_e2e() { workspace.path(), provider.clone(), vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".to_string(), description: "Email access.".to_string(), tools: Vec::new(), diff --git a/src/openhuman/agent/profiles/paths.rs b/src/openhuman/agent/profiles/paths.rs index a6c0ac674f..ac655251a7 100644 --- a/src/openhuman/agent/profiles/paths.rs +++ b/src/openhuman/agent/profiles/paths.rs @@ -662,6 +662,7 @@ mod tests { toolkit: &str, ) -> crate::openhuman::agent::prompts::ConnectedIntegration { crate::openhuman::agent::prompts::ConnectedIntegration { + result_notes: None, toolkit: toolkit.to_string(), description: String::new(), tools: Vec::new(), diff --git a/src/openhuman/agent/prompts/types.rs b/src/openhuman/agent/prompts/types.rs index d0a4237da4..7c5e3e1ee0 100644 --- a/src/openhuman/agent/prompts/types.rs +++ b/src/openhuman/agent/prompts/types.rs @@ -133,6 +133,17 @@ pub struct ConnectedIntegration { pub toolkit: String, /// Human-readable one-line description of what this integration can do. pub description: String, + /// What this toolkit's actions hand back, and which returned field feeds + /// which follow-up action. `None` for a toolkit we have not established + /// this for — see + /// [`toolkit_result_notes`](crate::openhuman::integrations::composio::providers::toolkit_result_notes). + /// + /// Separate from [`Self::description`] because the two answer different + /// questions for different readers. The description is a routing signal and + /// belongs anywhere a service is named, including the delegator's guide; + /// this is only useful to whoever actually calls the actions and reads what + /// comes back, so only that agent renders it. + pub result_notes: Option, /// Per-action catalogue (only populated when `connected == true`). pub tools: Vec, /// Per-action catalogue for actions that the toolkit **does** support but diff --git a/src/openhuman/agent/registry/agents/context_scout/prompt.rs b/src/openhuman/agent/registry/agents/context_scout/prompt.rs index f6b37c0ebd..fe1e5a7e2b 100644 --- a/src/openhuman/agent/registry/agents/context_scout/prompt.rs +++ b/src/openhuman/agent/registry/agents/context_scout/prompt.rs @@ -176,6 +176,7 @@ mod tests { fn integration(toolkit: &str, connected: bool) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.to_string(), description: String::new(), tools: vec![], diff --git a/src/openhuman/agent/registry/agents/integrations_agent/prompt.rs b/src/openhuman/agent/registry/agents/integrations_agent/prompt.rs index 778e4f14cb..ed6f9886c1 100644 --- a/src/openhuman/agent/registry/agents/integrations_agent/prompt.rs +++ b/src/openhuman/agent/registry/agents/integrations_agent/prompt.rs @@ -108,6 +108,20 @@ fn render_connected_integrations(integrations: &[ConnectedIntegration]) -> Strin } else { let _ = writeln!(out, "- **{}** — {}", ci.toolkit, ci.description); } + // What the actions hand back. This agent is the one that calls them and + // reads the results, so it is the one that needs to know a returned id + // is the handle for the detail it wanted. Nothing else it reads says so: + // the catalogue, the parameter schemas, and the contract the gate + // delivers all describe arguments only. Omitted for a toolkit we have + // not established a result shape for. + if let Some(notes) = ci + .result_notes + .as_deref() + .map(str::trim) + .filter(|n| !n.is_empty()) + { + let _ = writeln!(out, " Results: {notes}"); + } } // Surface pref-gated tools so the agent can honestly say "I have this @@ -215,6 +229,7 @@ mod tests { #[test] fn build_includes_connected_integrations_in_executor_voice() { let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email access.".into(), tools: Vec::new(), @@ -247,6 +262,7 @@ mod tests { #[test] fn build_skips_unconnected_integrations() { let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "notion".into(), description: "Pages.".into(), tools: Vec::new(), @@ -258,4 +274,51 @@ mod tests { let body = build(&ctx_with(&integrations)).unwrap(); assert!(!body.contains("## Connected Integrations")); } + + /// This agent is the one that calls the actions and reads what comes back, + /// so it is the one that has to know a returned id is the handle for the + /// detail it wanted. Everything else it reads describes arguments. + #[test] + fn build_states_what_the_actions_hand_back() { + let integrations = vec![ConnectedIntegration { + toolkit: "gmail".into(), + description: "Email access.".into(), + result_notes: Some( + "Read actions answer with one record per message carrying id and threadId. \ + To read one message in full, pass its id to \ + GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID." + .into(), + ), + tools: Vec::new(), + gated_tools: Vec::new(), + connected: true, + connections: Vec::new(), + non_active_status: None, + }]; + let body = build(&ctx_with(&integrations)).unwrap(); + assert!( + body.contains("Results: Read actions answer with one record per message"), + "result notes must reach the executor prompt: {body}" + ); + assert!(body.contains("GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID")); + } + + /// A toolkit whose result shape this repository has not established renders + /// nothing rather than a guess. + #[test] + fn build_omits_result_notes_when_there_are_none() { + let integrations = vec![ConnectedIntegration { + toolkit: "notion".into(), + description: "Pages.".into(), + result_notes: None, + tools: Vec::new(), + gated_tools: Vec::new(), + connected: true, + connections: Vec::new(), + non_active_status: None, + }]; + let body = build(&ctx_with(&integrations)).unwrap(); + assert!(body.contains("- **notion** — Pages.")); + assert!(!body.contains("Results:"), "no notes, no line: {body}"); + } } diff --git a/src/openhuman/agent/registry/agents/orchestrator/prompt.rs b/src/openhuman/agent/registry/agents/orchestrator/prompt.rs index b411abd982..c7f39bef66 100644 --- a/src/openhuman/agent/registry/agents/orchestrator/prompt.rs +++ b/src/openhuman/agent/registry/agents/orchestrator/prompt.rs @@ -636,6 +636,7 @@ mod tests { #[test] fn build_emits_delegation_guide_with_collapsed_tool() { let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email access.".into(), tools: Vec::new(), @@ -685,6 +686,7 @@ mod tests { ); let gmail = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email access.".into(), tools: Vec::new(), @@ -718,6 +720,7 @@ mod tests { #[test] fn delegation_guide_uses_compact_collapsed_format() { let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email access.".into(), tools: Vec::new(), @@ -736,6 +739,7 @@ mod tests { fn gmail_only() -> Vec { vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email access.".into(), tools: Vec::new(), @@ -820,6 +824,7 @@ mod tests { // focused on what the orchestrator can actually delegate. let integrations = vec![ ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email.".into(), tools: Vec::new(), @@ -829,6 +834,7 @@ mod tests { non_active_status: None, }, ConnectedIntegration { + result_notes: None, toolkit: "linear".into(), description: "Tracker.".into(), tools: Vec::new(), @@ -876,6 +882,7 @@ mod tests { #[test] fn build_omits_guide_when_no_integrations_connected() { let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "linear".into(), description: "Tracker.".into(), tools: Vec::new(), diff --git a/src/openhuman/channels/runtime/dispatch/routing.rs b/src/openhuman/channels/runtime/dispatch/routing.rs index d5532d39e8..037b03271d 100644 --- a/src/openhuman/channels/runtime/dispatch/routing.rs +++ b/src/openhuman/channels/runtime/dispatch/routing.rs @@ -244,6 +244,7 @@ mod connected_fallback_tests { fn integration(toolkit: &str) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.into(), description: String::new(), tools: vec![], diff --git a/src/openhuman/flows/tinyflows/caps/ops.rs b/src/openhuman/flows/tinyflows/caps/ops.rs index 3df609aec0..1813ec1358 100644 --- a/src/openhuman/flows/tinyflows/caps/ops.rs +++ b/src/openhuman/flows/tinyflows/caps/ops.rs @@ -1181,6 +1181,7 @@ mod tests { connections: Vec, ) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.to_string(), description: String::new(), tools: Vec::new(), diff --git a/src/openhuman/integrations/composio/connected_integrations.rs b/src/openhuman/integrations/composio/connected_integrations.rs index 7118db469a..cb6b499709 100644 --- a/src/openhuman/integrations/composio/connected_integrations.rs +++ b/src/openhuman/integrations/composio/connected_integrations.rs @@ -959,6 +959,10 @@ async fn fetch_connected_integrations_uncached( integrations.push(ConnectedIntegration { toolkit: slug.clone(), description: resolve_toolkit_description(&catalog_descriptions, slug), + // Unlike the description, this has no catalog counterpart to prefer + // — the Composio catalog publishes what an action takes, never what + // it returns — so the local table is the only source. + result_notes: super::providers::toolkit_result_notes(slug).map(str::to_string), tools, gated_tools, connected, diff --git a/src/openhuman/integrations/composio/ops_tests.rs b/src/openhuman/integrations/composio/ops_tests.rs index 478bc655f4..141c0aa6e5 100644 --- a/src/openhuman/integrations/composio/ops_tests.rs +++ b/src/openhuman/integrations/composio/ops_tests.rs @@ -1448,6 +1448,7 @@ fn seed_cache(key: &str, integrations: Vec) { /// Only `toolkit` + `connected` matter for diff-based invalidation. fn integration(toolkit: &str, connected: bool) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.to_string(), description: String::new(), tools: Vec::new(), diff --git a/src/openhuman/memory/sync/composio/providers/descriptions.rs b/src/openhuman/memory/sync/composio/providers/descriptions.rs index 09468447e2..8a2a00a887 100644 --- a/src/openhuman/memory/sync/composio/providers/descriptions.rs +++ b/src/openhuman/memory/sync/composio/providers/descriptions.rs @@ -1,4 +1,5 @@ -//! Human-readable capability summaries for Composio toolkit slugs. +//! Human-readable capability summaries for Composio toolkit slugs, plus what +//! the toolkit's actions hand back. /// Human-readable capability summary for a Composio toolkit slug. /// @@ -59,3 +60,117 @@ pub fn toolkit_description(slug: &str) -> &'static str { _ => "Interact with this connected service via its available actions", } } + +/// What a toolkit's actions hand back, and which field feeds which follow-up +/// action. `None` for a toolkit we have not established this for. +/// +/// [`toolkit_description`] answers "what can this service do", which is an +/// **input**-side question, and so is everything else the model reads before +/// calling: the tool catalogue, the parameter schema, the contract the gate +/// delivers. Nothing tells it what comes back. So a list action returns records +/// keyed by id, the model has no statement that the id is the handle for the +/// detail it actually wanted, and it re-issues the same list call — observed +/// live against Gmail. +/// +/// The rule for adding an entry: state only what this repository establishes — +/// which curated action carries which identifier, and which action that +/// identifier is the argument for. A toolkit we pass through unreshaped has no +/// such ground truth and gets no entry, because a guess about a response is +/// worse here than silence. +/// +/// **Do not describe field-by-field record shapes here.** A note may say what a +/// result *contains* and what to do with it, not how it is serialized. Both +/// Composio dispatch routes prefer the backend's rendered `markdownFormatted` +/// body and fall back to the JSON envelope only when it is absent or the call +/// failed (`composio/action_tool.rs`, `composio/tools.rs`), so the reshapes in +/// `providers/*/post_process.rs` describe just one of two possible renderings. +/// A note that recites their keys is true only on the fallback path — which is +/// how an earlier revision of this function came to tell the model that every +/// Gmail read action answers with a markdown body, when only +/// `GMAIL_FETCH_EMAILS` carries one. +pub fn toolkit_result_notes(slug: &str) -> Option<&'static str> { + match slug { + // Reshapes: `gmail/post_process.rs::{reshape_fetch_emails, reshape_list_threads}`. + // Slugs: `gmail/tools.rs::GMAIL_CURATED`. + // + // The thread/message distinction is the whole point of this entry. Live, + // a sub-agent searched with GMAIL_LIST_THREADS, got no message body back, + // and reported that mail which does exist could not be found. + "gmail" => Some( + "GMAIL_LIST_THREADS answers with thread ids, a one-line snippet, and a message \ + count — never a message body, so a thread whose snippet looks right still has \ + to be read. Pass a thread id to GMAIL_FETCH_MESSAGE_BY_THREAD_ID, or a message \ + id to GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID, to get the body; GMAIL_FETCH_EMAILS \ + carries one already. Bodies are the backend's rendered text, not the raw \ + message, and attachments arrive as a filename and type that GMAIL_GET_ATTACHMENT \ + fetches. Repeating a search returns the same snippets, so read the thread \ + instead of searching again.", + ), + // Reshapes: `slack/post_process.rs`. + // Slugs: `catalogs.rs::SLACK_CURATED`. + "slack" => Some( + "SLACK_LIST_CONVERSATIONS answers with a channel id per channel, and that id is \ + the channel argument SLACK_FETCH_CONVERSATION_HISTORY and the post actions take. \ + History entries identify their author by Slack user id, not display name, so \ + resolve it with SLACK_FIND_USERS before quoting a name, and identify themselves \ + by a ts timestamp, which is what threads and reactions key on.", + ), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every action slug these notes tell the model to call must be one the + /// toolkit actually exposes. A note naming a slug that was renamed or + /// dropped from the curated list is worse than no note: it sends the model + /// after a tool that is not in its list. + #[test] + fn result_notes_only_name_curated_action_slugs() { + let curated: Vec<&str> = super::super::gmail::GMAIL_CURATED + .iter() + .map(|t| t.slug) + .chain(super::super::catalogs::SLACK_CURATED.iter().map(|t| t.slug)) + .collect(); + + for slug in ["gmail", "slack"] { + let notes = toolkit_result_notes(slug).expect("both toolkits have notes"); + for word in notes.split(|c: char| !(c.is_ascii_uppercase() || c == '_')) { + // An all-caps underscored token in this prose is an action slug. + if word.len() > 6 && word.contains('_') { + assert!( + curated.contains(&word), + "{slug} notes name `{word}`, which is not a curated action" + ); + } + } + } + } + + /// A toolkit we have not established a result shape for gets no entry — a + /// guess about a response shape is worse here than silence. + #[test] + fn result_notes_absent_for_unestablished_toolkits() { + assert!(toolkit_result_notes("notion").is_none()); + assert!(toolkit_result_notes("definitely_not_a_toolkit").is_none()); + } + + /// The failure this entry exists for: a sub-agent searched threads, got + /// snippets rather than bodies, and reported that mail which does exist + /// could not be found. The note has to name both halves — that a thread + /// listing has no body, and which action produces one. + #[test] + fn gmail_notes_separate_finding_a_thread_from_reading_it() { + let notes = toolkit_result_notes("gmail").expect("gmail has notes"); + assert!( + notes.contains("GMAIL_LIST_THREADS") && notes.contains("never a message body"), + "must say a thread listing carries no body: {notes}" + ); + assert!( + notes.contains("GMAIL_FETCH_MESSAGE_BY_THREAD_ID"), + "must name the action that reads the thread: {notes}" + ); + } +} diff --git a/src/openhuman/memory/sync/composio/providers/mod.rs b/src/openhuman/memory/sync/composio/providers/mod.rs index cefe9ae178..9514090136 100644 --- a/src/openhuman/memory/sync/composio/providers/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/mod.rs @@ -275,7 +275,7 @@ pub fn agent_ready_toolkits() -> Vec<&'static str> { slugs } -pub use descriptions::toolkit_description; +pub use descriptions::{toolkit_description, toolkit_result_notes}; pub(crate) use helpers::{first_array_str, merge_extra, pick_str}; pub use registry::{ all_providers, get_provider, init_default_providers, register_provider, ProviderArc, diff --git a/src/openhuman/tools/orchestrator_tools.rs b/src/openhuman/tools/orchestrator_tools.rs index 5d5211d86c..38f7a847de 100644 --- a/src/openhuman/tools/orchestrator_tools.rs +++ b/src/openhuman/tools/orchestrator_tools.rs @@ -323,6 +323,7 @@ mod tests { fn integration(toolkit: &str, description: &str) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.into(), description: description.into(), tools: vec![], @@ -533,6 +534,7 @@ mod tests { let integrations = vec![ integration("gmail", "Send and read email."), ConnectedIntegration { + result_notes: None, toolkit: "github".into(), description: "GitHub access.".into(), tools: vec![], @@ -602,6 +604,7 @@ mod tests { let reg = registry_with_targets(); let integrations = vec![ ConnectedIntegration { + result_notes: None, toolkit: "Brand.New".into(), description: " ".into(), tools: vec![], diff --git a/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs b/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs index f4736fe2cb..756f10f406 100644 --- a/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs @@ -665,6 +665,7 @@ fn subagent_prompt_renderer_handles_formats_caps_and_stale_tool_indices() -> Res include_memory_md: true, }; let connected = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".to_string(), description: "Mail".to_string(), tools: Vec::new(), diff --git a/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs b/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs index a64767e922..46a0b76d3e 100644 --- a/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs @@ -339,6 +339,7 @@ fn parent(workspace_dir: PathBuf, model: Arc) -> ParentExecutionC session_id: "round25-session".to_string(), channel: "round25".to_string(), connected_integrations: vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".to_string(), description: "Round25 Gmail".to_string(), tools: vec![ConnectedIntegrationTool { diff --git a/tests/raw_coverage/composio_raw_coverage_e2e.rs b/tests/raw_coverage/composio_raw_coverage_e2e.rs index 5cd288f38d..13a9dc1014 100644 --- a/tests/raw_coverage/composio_raw_coverage_e2e.rs +++ b/tests/raw_coverage/composio_raw_coverage_e2e.rs @@ -307,6 +307,7 @@ async fn composio_connected_integrations_public_helpers_handle_empty_auth_and_id assert!(cached_active_integrations(&config).is_none()); let first = ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Gmail".into(), tools: Vec::new(), @@ -316,6 +317,7 @@ async fn composio_connected_integrations_public_helpers_handle_empty_auth_and_id non_active_status: None, }; let second = ConnectedIntegration { + result_notes: None, toolkit: "slack".into(), description: "Slack".into(), tools: Vec::new(), @@ -325,6 +327,7 @@ async fn composio_connected_integrations_public_helpers_handle_empty_auth_and_id non_active_status: None, }; let disconnected = ConnectedIntegration { + result_notes: None, toolkit: "notion".into(), description: "Notion".into(), tools: Vec::new(), @@ -340,6 +343,7 @@ async fn composio_connected_integrations_public_helpers_handle_empty_auth_and_id assert_ne!( connected_set_hash(&[]), connected_set_hash(&[ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: String::new(), tools: Vec::new(), diff --git a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs index 69ae7500e0..af64a5e48e 100644 --- a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs +++ b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs @@ -2927,6 +2927,7 @@ fn agent_pformat_and_prompt_renderers_cover_public_paths() { let prompt_tools = PromptTool::from_tools(&tools); let skills = Vec::new(); let integrations = vec![ConnectedIntegration { + result_notes: None, toolkit: "gmail".into(), description: "Email account".into(), tools: vec![], diff --git a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs index 09bee73f1a..53f7942f23 100644 --- a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs +++ b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs @@ -316,6 +316,7 @@ fn coverage_connected_integration( connected: bool, ) -> ConnectedIntegration { ConnectedIntegration { + result_notes: None, toolkit: toolkit.into(), description: description.into(), tools: vec![], From c6142a998a3492f6d40478f8370b8de352c85e9a Mon Sep 17 00:00:00 2001 From: yh928 Date: Wed, 5 Aug 2026 15:57:36 +0900 Subject: [PATCH 2/2] test(composio): check each toolkit's notes against its own catalogue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slug check pooled `GMAIL_CURATED` and `SLACK_CURATED`, so a Gmail note could name a Slack-only action and still pass — which is the mistake most likely to be made when editing prose that mentions both toolkits, and the one the test exists to catch. Each toolkit is now checked against its own list, and the failure message says whose list it missed. Adds the debug event for the lookup: toolkit and whether an entry exists. The notes themselves stay out of the log — they are prose bound for the prompt, and repeating a paragraph per lookup is noise. providers::descriptions 3 pass. Reported by CodeRabbit on #5322. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SRSNnqQsokuGmkbpLoLCGy --- .../sync/composio/providers/descriptions.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/openhuman/memory/sync/composio/providers/descriptions.rs b/src/openhuman/memory/sync/composio/providers/descriptions.rs index 8a2a00a887..157fe2fa87 100644 --- a/src/openhuman/memory/sync/composio/providers/descriptions.rs +++ b/src/openhuman/memory/sync/composio/providers/descriptions.rs @@ -89,6 +89,19 @@ pub fn toolkit_description(slug: &str) -> &'static str { /// Gmail read action answers with a markdown body, when only /// `GMAIL_FETCH_EMAILS` carries one. pub fn toolkit_result_notes(slug: &str) -> Option<&'static str> { + let notes = result_notes_for(slug); + // The result content is prose we authored, not user data, but log only + // whether an entry exists: the notes themselves belong in the prompt, and + // repeating a paragraph per lookup is noise at debug level. + tracing::debug!( + toolkit = %slug, + has_notes = notes.is_some(), + "[composio] toolkit result-notes lookup" + ); + notes +} + +fn result_notes_for(slug: &str) -> Option<&'static str> { match slug { // Reshapes: `gmail/post_process.rs::{reshape_fetch_emails, reshape_list_threads}`. // Slugs: `gmail/tools.rs::GMAIL_CURATED`. @@ -129,20 +142,26 @@ mod tests { /// after a tool that is not in its list. #[test] fn result_notes_only_name_curated_action_slugs() { - let curated: Vec<&str> = super::super::gmail::GMAIL_CURATED + // Each toolkit is checked against its OWN catalogue. Pooling them let a + // Gmail note name a Slack-only action and still pass, which is the + // mistake most likely to be made when editing prose that mentions both. + let gmail: Vec<&str> = super::super::gmail::GMAIL_CURATED + .iter() + .map(|t| t.slug) + .collect(); + let slack: Vec<&str> = super::super::catalogs::SLACK_CURATED .iter() .map(|t| t.slug) - .chain(super::super::catalogs::SLACK_CURATED.iter().map(|t| t.slug)) .collect(); - for slug in ["gmail", "slack"] { + for (slug, curated) in [("gmail", &gmail), ("slack", &slack)] { let notes = toolkit_result_notes(slug).expect("both toolkits have notes"); for word in notes.split(|c: char| !(c.is_ascii_uppercase() || c == '_')) { // An all-caps underscored token in this prose is an action slug. if word.len() > 6 && word.contains('_') { assert!( curated.contains(&word), - "{slug} notes name `{word}`, which is not a curated action" + "{slug} notes name `{word}`, which is not one of {slug}'s curated actions" ); } }