feat(agent-core): progressive tool disclosure via select_tools

.changeset/select-tools-progressive-disclosure.md

@@ -0,0 +1,6 @@
+---
+"@moonshot-ai/kimi-code": minor
+"@moonshot-ai/kimi-code-sdk": minor
+---
+
+Add experimental progressive tool disclosure (`select_tools`). When the `tool-select` experimental flag is on and the active model declares the `select_tools` capability, MCP tool schemas stay out of the request's top-level `tools[]` (preserving the provider prompt cache); the model loads tools on demand by exact name via the new built-in `select_tools` tool, guided by `<tools_added>/<tools_removed>` announcements. Off by default and inert on models without the capability — behavior is unchanged until a supporting model is catalogued. The SDK additionally maps the `select_tools` capability when building model aliases from a catalog and reports the new flag through `getExperimentalFeatures()`.

packages/agent-core/src/agent/compaction/full.ts

@@ -11,12 +11,19 @@ import {
   type GenerateResult,
   type Message,
   type TokenUsage,
+  type Tool,
   APIContextOverflowError,
   APIStatusError,
   createUserMessage,
 } from '@moonshot-ai/kosong';
 
 import type { Agent } from '..';
+import type { ContextMessage } from '../context/types';
+import {
+  collectLoadedDynamicToolNames,
+  DYNAMIC_TOOL_SCHEMA_VARIANT,
+  stripDynamicToolContext,
+} from '../context/dynamic-tools';
 import { isAbortError } from '../../loop/errors';
 import {
   retryBackoffDelays,
@@ -218,7 +225,9 @@ export class FullCompaction {
   private estimateRequestTokens(messages: readonly Message[]): number {
     return (
       estimateTokens(this.agent.config.systemPrompt) +
-      estimateTokensForTools(this.agent.tools.loopTools) +
+      // Deferred tools never reach the outbound top-level tools[] (kosong
+      // generate() strips them); keep the estimate aligned with the wire.
+      estimateTokensForTools(this.agent.tools.loopTools.filter((t) => t.deferred !== true)) +
       estimateTokensForMessages(messages)
     );
   }
@@ -357,6 +366,65 @@ export class FullCompaction {
     }).trimEnd();
   }
 
+  /**
+   * Keep-all rebuild (Phase 1): after compaction folded the history — and the
+   * dynamic tool schema messages with it — append ONE merged schema message so
+   * the model keeps calling its loaded tools without re-selecting. Schemas are
+   * read from the live registry, never copied from the old history, so a
+   * schema that changed since load self-heals here. Names whose server is
+   * currently disconnected have no registry schema and are not rebuilt (the
+   * model re-selects after reconnect); names that survived into the
+   * post-compaction history (none under today's users+summary rebuild, but
+   * guarded) are not duplicated. The message goes through the normal
+   * injection-origin append, so estimation and records pick it up as usual.
+   *
+   * Budget guard: the rebuilt floor (users + summary + schemas) is the one
+   * part of the post-compaction context that compaction itself can never
+   * shrink — if it lands inside the auto-compaction trigger band, every
+   * following step re-compacts and rebuilds in a loop. Admit schemas (in name
+   * order) only while the projected context stays within HALF the compaction
+   * trigger, so normal turn content still fits before the next compaction;
+   * anything dropped is simply re-selectable on demand, the same degradation
+   * as a disconnected server.
+   */
+  private rebuildDynamicToolSchemas(activeBefore: ReadonlySet<string>): void {
+    if (!this.agent.toolSelectEnabled) return;
+    if (activeBefore.size === 0) return;
+    const surviving = collectLoadedDynamicToolNames(this.agent.context.history);
+    const names = [...activeBefore]
+      .filter((name) => !surviving.has(name))
+      .toSorted((a, b) => a.localeCompare(b));
+    const candidates = names
+      .map((name) => this.agent.tools.getMcpToolSchema(name))
+      .filter((tool): tool is NonNullable<typeof tool> => tool !== undefined);
+    if (candidates.length === 0) return;
+    const tools: Tool[] = [];
+    let projected = this.tokenCountWithPending;
+    for (const tool of candidates) {
+      const toolTokens = estimateTokensForTools([tool]);
+      // shouldCompact is monotonic in usedSize, so doubling the projected
+      // size checks "within half the trigger" for both trigger branches
+      // (ratio and reserved-context).
+      if (this.strategy.shouldCompact((projected + toolTokens) * 2)) break;
+      tools.push(tool);
+      projected += toolTokens;
+    }
+    if (tools.length < candidates.length) {
+      this.agent.log.info('trimmed dynamic tool schema rebuild to stay clear of the compaction trigger', {
+        kept: tools.length,
+        dropped: candidates.slice(tools.length).map((tool) => tool.name),
+      });
+    }
+    if (tools.length === 0) return;
+    this.agent.context.appendMessage({
+      role: 'system',
+      content: [],
+      toolCalls: [],
+      tools,
+      origin: { kind: 'injection', variant: DYNAMIC_TOOL_SCHEMA_VARIANT },
+    });
+  }
+
   private postProcessSummary(summary: string): string {
     const storeData = this.agent.tools.storeData();
     const todos = (storeData[TODO_STORE_KEY] as readonly TodoItem[] | undefined) ?? [];
@@ -374,6 +442,11 @@ export class FullCompaction {
     const startedAt = Date.now();
     const originalHistory = [...this.agent.context.history];
     const tokensBefore = estimateTokensForMessages(originalHistory);
+    // Loaded-tools snapshot BEFORE the rebuild below folds the history away;
+    // read here so the keep-all schema rebuild after applyCompaction knows
+    // what was active. (The ledger scans history, which applyCompaction
+    // replaces.)
+    const activeDynamicToolsBefore = new Set(this.agent.tools.loadedDynamicToolNames());
     let retryCount = 0;
     try {
       await this.triggerPreCompactHook(data, tokensBefore, signal);
@@ -407,7 +480,15 @@ export class FullCompaction {
       // Compact the whole history, trimming old messages only when the
       // summarizer request itself cannot fit. Any trimmed messages are not
       // covered by the produced summary; `droppedCount` reports that blind spot.
-      let historyForModel = originalHistory;
+      // Dynamic-tool protocol context (schema messages, loadable-tools
+      // announcements) is excluded from the summarizer input entirely: it is
+      // protocol state, not conversation — summarizing it wastes tokens and
+      // risks schema text leaking into the summary. Zero information loss:
+      // the post-compaction boundary re-announces the manifest and the
+      // keep-all rebuild re-carries the schemas. Must happen before project()
+      // (which strips the origin anchor). `originalHistory` itself stays
+      // untouched for the prefix-race check and `compactedCount`.
+      let historyForModel: readonly ContextMessage[] = stripDynamicToolContext(originalHistory);
       let droppedCount = 0;
       let overflowShrinkCount = 0;
       let emptyOrTruncatedShrinkCount = 0;
@@ -525,6 +606,7 @@ export class FullCompaction {
         tokensBefore,
         droppedCount: droppedCount === 0 ? undefined : droppedCount,
       });
+      this.rebuildDynamicToolSchemas(activeDynamicToolsBefore);
 
       // Telemetry keys are snake_case, but the `context.apply_compaction`
       // record written below keeps its persisted camelCase field names
@@ -544,7 +626,15 @@ export class FullCompaction {
           ? {}
           : { input_tokens: inputTotal(usage), output_tokens: usage.output }),
       });
-      this.lastCompactedTokenCount = result.tokensAfter;
+      // Baseline the "nothing new since compaction" guard on the counter
+      // that includes the schema rebuild appended above. `result.tokensAfter`
+      // predates the rebuild (and deliberately keeps its persisted
+      // users+summary semantics — the rebuild message is accounted through
+      // the pending-estimate tail, so folding it into tokensAfter would
+      // double-count on both the live and the restore path). A baseline
+      // below the actual post-compaction floor would let checkAutoCompaction
+      // re-trigger even though the compacted shape cannot shrink further.
+      this.lastCompactedTokenCount = this.tokenCountWithPending;
       return result;
     } catch (error) {
       if (isAbortError(error)) return undefined;

packages/agent-core/src/agent/context/dynamic-tools.ts

@@ -0,0 +1,151 @@
+/**
+ * Shared predicates and shaping helpers for select_tools progressive
+ * disclosure protocol context.
+ *
+ * Two kinds of messages carry that protocol state in the history:
+ *   - dynamic tool schema messages: `role: 'system'` messages whose `tools`
+ *     field holds full tool definitions (origin
+ *     `{kind: 'injection', variant: 'dynamic_tool_schema'}` so undo keeps
+ *     them — tool loading is protocol context, not conversation);
+ *   - loadable-tools announcements: `<tools_added>/<tools_removed>` system
+ *     reminders (origin `{kind: 'system_trigger', name: 'loadable-tools'}` so
+ *     undo removes them and the next turn-boundary diff self-heals).
+ *
+ * Everything here anchors on `origin` or the `tools` field, so callers that
+ * need to filter MUST run before `project()` — projection strips `origin`.
+ */
+
+import type { Tool } from '@moonshot-ai/kosong';
+
+import type { ContextMessage } from './types';
+
+/** Origin variant of an injected dynamic tool schema message (undo keeps it). */
+export const DYNAMIC_TOOL_SCHEMA_VARIANT = 'dynamic_tool_schema';
+
+/** Origin name of the loadable-tools diff announcements (undo removes them). */
+export const LOADABLE_TOOLS_TRIGGER = 'loadable-tools';
+
+/** True for a message that loads tool definitions (`message.tools` present). */
+export function isDynamicToolSchemaMessage(message: ContextMessage): boolean {
+  return message.tools !== undefined && message.tools.length > 0;
+}
+
+/** True for a `<tools_added>/<tools_removed>` announcement reminder. */
+export function isLoadableToolsAnnouncement(message: ContextMessage): boolean {
+  return (
+    message.origin?.kind === 'system_trigger' && message.origin.name === LOADABLE_TOOLS_TRIGGER
+  );
+}
+
+/**
+ * Shape a history for a consumer that must not see dynamic-tool protocol
+ * context: drop the loadable-tools announcements and strip `message.tools`
+ * (dropping the message entirely when nothing else remains). Two callers:
+ *   - projection for a model without the `select_tools` capability (mid-session
+ *     model switch — the canonical history keeps its shape, only the outgoing
+ *     view changes; announcements would be noise and even reference a
+ *     select_tools tool the model does not have);
+ *   - the compaction summarizer input (schemas and announcements are protocol
+ *     context, not conversation — summarizing them wastes tokens and risks
+ *     leaking schema text into the summary).
+ * Returns the input array unchanged when there is nothing to strip, so the
+ * common no-dynamic-tools path costs one scan and no allocation.
+ */
+export function stripDynamicToolContext(
+  history: readonly ContextMessage[],
+): readonly ContextMessage[] {
+  if (!history.some((m) => isDynamicToolSchemaMessage(m) || isLoadableToolsAnnouncement(m))) {
+    return history;
+  }
+  const out: ContextMessage[] = [];
+  for (const message of history) {
+    if (isLoadableToolsAnnouncement(message)) continue;
+    if (isDynamicToolSchemaMessage(message)) {
+      const { tools: _tools, ...rest } = message;
+      void _tools;
+      if (rest.content.length === 0 && rest.toolCalls.length === 0) continue;
+      out.push(rest);
+      continue;
+    }
+    out.push(message);
+  }
+  return out;
+}
+
+/** Union of tool names loaded by dynamic tool schema messages in `history`. */
+export function collectLoadedDynamicToolNames(
+  history: readonly ContextMessage[],
+): Set<string> {
+  const names = new Set<string>();
+  for (const message of history) {
+    if (message.tools === undefined) continue;
+    for (const tool of message.tools) {
+      names.add(tool.name);
+    }
+  }
+  return names;
+}
+
+const TOOLS_ADDED_BLOCK = /<tools_added>\n?([\s\S]*?)\n?<\/tools_added>/g;
+const TOOLS_REMOVED_BLOCK = /<tools_removed>\n?([\s\S]*?)\n?<\/tools_removed>/g;
+
+/**
+ * Fold every loadable-tools announcement in `history`, in order, into the
+ * currently-announced name set (`tools_removed` deletes, then `tools_added`
+ * adds — last wins). The announcements are the context's own record of what
+ * the model has been told is loadable; there is deliberately no separate
+ * persisted ledger, so undo/compaction/resume all self-heal by re-folding.
+ */
+export function foldAnnouncedToolNames(history: readonly ContextMessage[]): Set<string> {
+  const announced = new Set<string>();
+  for (const message of history) {
+    if (!isLoadableToolsAnnouncement(message)) continue;
+    const text = message.content
+      .map((part) => (part.type === 'text' ? part.text : ''))
+      .join('');
+    for (const name of matchToolNameBlocks(text, TOOLS_REMOVED_BLOCK)) {
+      announced.delete(name);
+    }
+    for (const name of matchToolNameBlocks(text, TOOLS_ADDED_BLOCK)) {
+      announced.add(name);
+    }
+  }
+  return announced;
+}
+
+function matchToolNameBlocks(text: string, pattern: RegExp): string[] {
+  const names: string[] = [];
+  pattern.lastIndex = 0;
+  for (const match of text.matchAll(pattern)) {
+    const body = match[1] ?? '';
+    for (const line of body.split('\n')) {
+      const name = line.trim();
+      if (name.length > 0) names.push(name);
+    }
+  }
+  return names;
+}
+
+/**
+ * Render one diff announcement. Only the blocks with content are emitted; the
+ * guidance sentence never contains a literal block tag, so `foldAnnouncedToolNames`
+ * can anchor on the tags without tripping over prose.
+ */
+export function renderLoadableToolsAnnouncement(
+  added: readonly string[],
+  removed: readonly string[],
+): string {
+  const sections: string[] = [];
+  if (added.length > 0) {
+    sections.push(`<tools_added>\n${added.join('\n')}\n</tools_added>`);
+  }
+  if (removed.length > 0) {
+    sections.push(`<tools_removed>\n${removed.join('\n')}\n</tools_removed>`);
+  }
+  sections.push(
+    'Use the select_tools tool with exact names to load full tool definitions before calling them. ' +
+      'Names listed as removed are no longer loadable — do not select them. ' +
+      'Fold all announcements in this conversation in order to get the current list.',
+  );
+  return sections.join('\n\n');
+}

packages/agent-core/src/agent/context/index.ts

@@ -23,6 +23,7 @@ import {
   type ProjectOptions,
   trimTrailingOpenToolExchange,
 } from './projector';
+import { stripDynamicToolContext } from './dynamic-tools';
 import {
   USER_PROMPT_ORIGIN,
   type AgentContextData,
@@ -31,6 +32,7 @@ import {
 } from './types';
 
 export * from './types';
+export * from './dynamic-tools';
 
 const TOOL_ERROR_STATUS = '<system>ERROR: Tool execution failed.</system>';
 const TOOL_EMPTY_STATUS = '<system>Tool output is empty.</system>';
@@ -175,6 +177,7 @@ export class ContextMemory {
     this._lastAssistantAt = null;
     this.agent.microCompaction.reset();
     this.agent.injection.onContextClear();
+    this.agent.tools.onContextCleared();
     this.agent.emitStatusUpdated();
   }
 
@@ -351,6 +354,7 @@ export class ContextMemory {
     this.tokenCountCoveredMessageCount = this._history.length;
     this.agent.microCompaction.reset();
     this.agent.injection.onContextCompacted();
+    this.agent.tools.onContextCompacted();
     this.agent.emitStatusUpdated();
     return result;
   }
@@ -376,8 +380,15 @@ export class ContextMemory {
   }
 
   project(messages: readonly ContextMessage[], options?: ProjectOptions): Message[] {
+    // Shape for the current model BEFORE projecting: a model without the
+    // select_tools capability must not see dynamic-tool schema messages or
+    // loadable-tools announcements (the canonical history keeps them; only
+    // this outgoing view is shaped). Must run pre-projection — project()
+    // strips `origin`, the only anchor for the announcements. setModel never
+    // rewrites history, so a mid-session switch degrades/upgrades losslessly.
+    const shaped = this.agent.toolSelectEnabled ? messages : stripDynamicToolContext(messages);
     const anomalies: ProjectionAnomaly[] = [];
-    const result = project(this.agent.microCompaction.compact(messages), {
+    const result = project(this.agent.microCompaction.compact(shaped), {
       ...options,
       onAnomaly: (anomaly) => {
         anomalies.push(anomaly);

packages/agent-core/src/agent/context/projector.ts

@@ -390,6 +390,10 @@ function prepareMessageForProjection(
       },
     );
   }
+  // A message that loads tool definitions (`tools` present) is intentionally
+  // content-free — it must survive the empty-message cleanup or the loaded
+  // schemas silently vanish from every outgoing request.
+  if (next.tools !== undefined && next.tools.length > 0) return next;
   return next.content.length === 0 && next.toolCalls.length === 0 ? null : next;
 }
 
@@ -429,6 +433,7 @@ function stripContextMetadata(message: ContextMessage): Message {
     toolCalls: message.toolCalls.map((tc) => ({ ...tc })),
     toolCallId: message.toolCallId,
     partial: message.partial,
+    tools: message.tools?.map((tool) => ({ ...tool })),
   };
 }