diff --git a/.codecov.yml b/.codecov.yml index 2f09a66..d379c08 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -7,3 +7,7 @@ coverage: default: target: 70% threshold: 1% + informational: true + patch: + default: + informational: true diff --git a/.github/workflows/deploy-site-pages.yml b/.github/workflows/deploy-site-pages.yml index 36d5336..e9c0932 100644 --- a/.github/workflows/deploy-site-pages.yml +++ b/.github/workflows/deploy-site-pages.yml @@ -35,7 +35,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '22' cache: 'pnpm' cache-dependency-path: site/pnpm-lock.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8c0abb..6361744 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ jobs: version: 9 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '22' cache: 'pnpm' registry-url: 'https://registry.npmjs.org/' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94fe2e4..fa2e516 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,32 @@ on: workflow_dispatch: jobs: - # 在 Linux 上运行所有检查 + # 快速检查：格式化 + lint，与测试/构建并行 + format-lint: + name: Format & Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 9 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install + + - name: Format check + run: pnpm run format:check + + - name: Lint + run: pnpm run lint + + # 测试 + 覆盖率上报（需要 ripgrep：部分工具/测试依赖 rg） test: - name: Test & Lint + name: Test & Coverage runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -19,7 +42,7 @@ jobs: version: 9 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '22' cache: 'pnpm' - name: Install ripgrep @@ -28,11 +51,11 @@ jobs: - name: Install dependencies run: pnpm install - - name: Format check - run: pnpm run format:check + - name: Test all packages + run: pnpm run test - - name: Test all packages with coverage - run: pnpm run test:coverage + - name: Coverage report (warning only, non-blocking) + run: pnpm run test:coverage || echo "::warning::Test coverage is below the thresholds in vitest.config.ts. PR is not blocked by coverage." - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 @@ -41,5 +64,22 @@ jobs: files: ./coverage/lcov.info fail_ci_if_error: false + # 打包验证：与测试并行 + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 9 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install + - name: Test build run: pnpm run build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ff4617..d2848e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Thanks for your interest in **Memo Code**. Please read these guidelines before o ## Getting Started -- Install [Node.js](https://nodejs.org/) (>=18) and [pnpm](https://pnpm.io/). Some tools/tests depend on [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`). +- Install [Node.js](https://nodejs.org/) (>=22) and [pnpm](https://pnpm.io/). Some tools/tests depend on [ripgrep](https://github.com/BurntSushi/ripgrep) (`rg`). - Install dependencies: `pnpm install` - Run CLI: `pnpm start "your prompt" --once` or interactive mode with `pnpm start` - Build output: `pnpm run build` diff --git a/docs/cli-update.md b/docs/cli-update.md deleted file mode 100644 index c506500..0000000 --- a/docs/cli-update.md +++ /dev/null @@ -1,33 +0,0 @@ -# Historical Note: CLI Adaptation for Tool Use and Concurrency - -This document is a historical implementation note for the CLI adaptation completed on **February 1, 2026**. - -## Scope of That Update - -The update covered: - -- CLI-side compatibility for concurrent tool calls. -- Hook-level behavior compatibility between single-tool and multi-tool execution. -- TUI display compatibility without requiring immediate UI rewrites. - -## Current Relevance - -- This page is retained as migration history. -- For current architecture and behavior, use: - - `docs/core.md` - - `site/content/docs/README.md` - - `README.md` - -## Historical Summary - -At the time of the migration, the main compatibility strategy was: - -- Keep existing hook interfaces stable. -- Merge concurrent observations into a unified output payload. -- Preserve single-tool behavior to avoid regressions. - -## Status - -- Document type: historical -- Last reviewed date: February 6, 2026 -- Breaking changes introduced by this historical update: none diff --git a/docs/core.md b/docs/core.md deleted file mode 100644 index f917098..0000000 --- a/docs/core.md +++ /dev/null @@ -1,244 +0,0 @@ -# Core Implementation Notes (Current Architecture) - -Core focuses on "Tool Use API + concurrent execution + state machine". Session/Turn APIs drive tool calls and JSONL event recording. Defaults are completed from `~/.memo/config.toml` (provider, log paths, etc.), while terminal UX/entry is hosted in `packages/tui`. - -## Package Boundaries (TUI vs Core) - -- `packages/core`: runtime contracts, session state machine, tool-calling loop, config/log defaults. -- `packages/tui`: terminal runtime package (`src/cli.tsx` entry, plain-mode runtime, interactive UI state/rendering, slash commands, approval overlay). - -Core should stay UI-agnostic: do not add Ink/UI rendering details into `packages/core`. - -## Directory and Modules - -- `config/`: config and paths - - `config.ts`: reads/writes `~/.memo/config.toml`, provider selection (`name/env_api_key/model/base_url`), session path generation (`sessions/-/-.jsonl`), and session ID generation. -- `runtime/`: runtime and logging - - `prompt.md/prompt.ts`: system prompt loading (integrates Claude Code best practices + SOUL/AGENTS/skills composition). - - `history.ts`: JSONL sink and event builders. - - `defaults.ts`: fills tools, LLM, prompt, history sink, tokenizer from config. - - `session.ts`: Session/Turn state machine; runs ReAct loop, writes events, tracks tokens, fires hooks; **supports concurrent tool calls**. -- `toolRouter/`: tool routing and management - - `index.ts`: manages built-in + MCP tools, generates Tool Use API tool definitions. -- `utils/`: parsing and tokenizer wrappers (assistant output parsing, message wrappers, tiktoken wrapper). -- `types.ts`: shared types (**extended for Tool Use API support**). -- `index.ts`: package entry exporting the modules above. - -## Core Mechanism: Structured Tool Use API - -### 1. Tool Calling Protocol - -**Primary: Tool Use API** (stable and efficient) - -- Uses native Tool Use APIs from OpenAI/DeepSeek/Claude. -- Model returns structured `tool_use` blocks. -- Supports concurrent calls (`Promise.allSettled`). -- Format: `{ content: [{ type: 'tool_use', id, name, input }, ...], stop_reason: 'tool_use' }`. - -### 2. Concurrent Tool Execution - -**Concurrency scenario**: - -```typescript -// When model returns multiple tool_use blocks -if (toolUseBlocks.length > 1) { - // Execute concurrently via Promise.allSettled - const results = await Promise.allSettled(toolUseBlocks.map((block) => executeTool(block))) - // One tool failure does not affect others - // All results are merged and sent back to the model -} -``` - -**Performance gain**: - -- From 10 serial round-trips -> 2-3 concurrent round-trips -- Around **5x improvement** - -**Typical use cases**: - -- Read multiple files concurrently (`read_text_file + read_text_file + read_text_file`) -- Run multiple git commands in parallel (`exec_command + exec_command + exec_command`) -- Search and read simultaneously (`list_directory + search_files + read_text_file`) - -### 3. State Flow - -1. System prompt instructs the model to either call tools or return final response. -2. Model response is classified: - - **tool_use**: execute tools (single or concurrent), collect observation - - **end_turn**: finish and return final response - - no actionable block: end current step -3. Observation write-back: - - single tool: `{"observation":"...","tool":"name"}` - - concurrent tools: `{"observation":"[tool1]: result1\n\n[tool2]: result2"}` - -## Entry API: Session/Turn (`createAgentSession`) - -- `createAgentSession(deps, options)` returns a Session; `runTurn` runs one ReAct turn. UI controls turn count. -- Default deps can be omitted: `tools` (built-in set), `callLLM` (provider-based OpenAI client, **auto-sends tool definitions**), `loadPrompt`, `historySinks` (writes to `~/.memo/sessions/...`), `tokenCounter`. -- Config source: `~/.memo/config.toml` (overridable via `MEMO_HOME`), keys include `current_provider` and `providers` list. Missing config triggers interactive UI setup. -- Callbacks: - - `onAssistantStep` (stream-like output) - - `hooks`/`middlewares` (`onTurnStart/onAction/onObservation/onFinal`) for UI/plugin lifecycle subscription. - -Example: - -```ts -import { createAgentSession } from '@memo/core' - -const session = await createAgentSession({ onAssistantStep: console.log }, { mode: 'interactive' }) -const turn = await session.runTurn('Hello') -await session.close() -``` - -## History and Logs (`runtime/history.ts`) - -- Events: `session_start/turn_start/assistant/action/observation/final/turn_end/session_end`. -- Default output path: `~/.memo/sessions/-/-.jsonl`, with provider/model/tokenizer/token-usage metadata. -- For concurrent calls, each tool observation is logged individually, and merged observation is also recorded. - -## LLM Adapter (`runtime/defaults.ts`) - -- `withDefaultDeps` provides OpenAI SDK based invocation (selected by provider/model/base_url/env_api_key). -- **Automatically generates Tool Use API tool definitions**: `toolRouter.generateToolDefinitions()`. -- **Passes tools to LLM API**: - -```typescript -const tools = toolDefinitions.map((tool) => ({ - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: tool.input_schema, - }, -})) - -await client.chat.completions.create({ - model, - messages, - tools, - tool_choice: 'auto', -}) -``` - -- Prefers incoming `callLLM` override; otherwise reads env vars (`current provider env_api_key` / `OPENAI_API_KEY` / `DEEPSEEK_API_KEY`). - -## Tool Protocol and Registry - -- `ToolRegistry = Record` (`name/description/inputSchema/execute`). -- Default tools come from `packages/tools`, managed through `ToolRouter`. -- **`ToolRouter` responsibilities**: - - register built-in and MCP tools - - generate Tool Use API definitions - - generate prompt-format tool descriptions - - execute tool calls -- Unknown tools return `"Unknown tool: name"`. - -## Config and Path Handling (`config/config.ts`) - -- `loadMemoConfig`: reads `~/.memo/config.toml`, returns config/path + `needsSetup` flag. -- `writeMemoConfig`: writes config back. -- `buildSessionPath`: builds project-scoped JSONL path with `datetime-sessionId` filename. -- `selectProvider`: selects provider by name with fallback. - -## Key Updates (v2 Architecture) - -### Type System Extension - -Added Tool Use API support: - -```typescript -// ContentBlock types -export type ToolUseBlock = { - type: 'tool_use' - id: string - name: string - input: unknown -} - -export type TextBlock = { - type: 'text' - text: string -} - -export type ContentBlock = TextBlock | ToolUseBlock - -// LLMResponse: single structured mode -export type LLMResponse = { - content: ContentBlock[] - stop_reason: 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' - usage?: Partial -} -``` - -### Response Normalization - -`normalizeLLMResponse` extracts text/tool blocks from one structured response: - -```typescript -{ - textContent: string, - toolUseBlocks: Array<{...}>, - stopReason?: 'end_turn' | 'tool_use', - usage?: TokenUsage -} -``` - -### Concurrent Execution Logic - -`session.ts:400+` implements concurrent tool calling: - -```typescript -if (toolUseBlocks.length > 1) { - const toolResults = await Promise.allSettled( - toolUseBlocks.map(async (toolBlock) => { - const tool = this.deps.tools[toolBlock.name] - return await tool.execute(toolBlock.input) - }), - ) - const combinedObservation = observations.join('\n\n') - await runHook(this.hooks, 'onObservation', { observation: combinedObservation }) -} -``` - -## System Prompt (`runtime/prompt.md`) - -Incorporates Claude Code best practices: - -1. **Strict output control**: `< 4` lines of text (excluding tool calls/code) -2. **Concurrency requirement**: independent tools must run in parallel -3. **Plan-driven flow**: complex tasks (>=3 steps) should use `update_plan` -4. **Engineering quality**: run lint/typecheck after completion -5. **Precise references**: use `file:line` format for code references -6. **Concise refusal**: 1-2 sentence refusal, no verbosity -7. **Layered context composition**: base template (+ optional `SOUL.md` soft preference section) -> startup root `AGENTS.md` -> discovered skills -> tool descriptions - -## Compatibility Guarantees - -### Backward Compatibility - -- ✅ existing tool interfaces unchanged -- ✅ existing config format unchanged - -### Cross-model Support - -- ✅ OpenAI GPT-4/GPT-3.5 (native Tool Use) -- ✅ DeepSeek v3 (native Tool Use) -- ✅ Claude (native Tool Use) - -## Performance Metrics - -| Dimension | Before | After | Improvement | -| ------------------------- | -------------- | ----------------- | ----------------- | -| Tool-calling efficiency | 10 round-trips | 2-3 round-trips | **5x** | -| Format stability | 70% success | 95% success | **+25%** | -| Cross-model compatibility | Claude only | mainstream models | **full coverage** | - -## Summary - -Core provides a "structured Tool Use API + concurrent execution + pluggable deps" architecture, so UI can stay interaction-focused. Config/logs stay in user directories to avoid repository pollution, with support for multi-provider and token budget control. - -**Key advantages**: - -- 5x performance gain (concurrent tool calls) -- 95% format stability (native Tool Use API) -- Cross-model compatibility (native Tool Use capable models) diff --git a/docs/issue-governance.md b/docs/issue-governance.md deleted file mode 100644 index 20c045d..0000000 --- a/docs/issue-governance.md +++ /dev/null @@ -1,36 +0,0 @@ -# Issue Governance - -This repository follows a lightweight issue governance workflow for open-source maintenance. - -## Goals - -- Keep one canonical issue per topic. -- Reduce duplicate reports and fragmented discussions. -- Ensure every open issue has clear type, area, and priority context. - -## Labels - -- Type: `bug`, `enhancement`, `documentation`, `question` -- Area: `area:tui`, `area:tools`, `area:core`, `area:security`, `area:docs` -- Status: `needs-triage`, `duplicate-candidate`, `status:blocked` -- Priority: `priority:p0`, `priority:p1` - -## Triage Flow - -1. New issue gets auto-labeled by `.github/workflows/issue-governance.yml`. -2. Maintainer verifies labels and sets priority. -3. If duplicate is confirmed: - - Keep one canonical issue open. - - Close duplicates with reason `duplicate` and link canonical issue. -4. For broad roadmap topics, use one tracking issue with checklist/sub-issues. - -## Duplicate Detection - -The automation posts non-blocking duplicate hints based on title/body token overlap. -Maintainer confirmation is required before closing an issue as duplicate. - -## Contributor Expectations - -- Use issue forms instead of blank issues. -- Search existing issues before opening a new one. -- Include reproducible steps for bugs and acceptance criteria for features. diff --git a/docs/memo-architecture-design.md b/docs/memo-architecture-design.md deleted file mode 100644 index 29d59cf..0000000 --- a/docs/memo-architecture-design.md +++ /dev/null @@ -1,405 +0,0 @@ -# Memo 系统架构设计文档 - -## 概述 - -Memo 是一个轻量级的编程代理系统，采用模块化的 monorepo 架构，支持终端用户界面（TUI）交互。系统基于 Node.js 22+ 构建，使用 TypeScript 开发，遵循严格的类型安全和模块化设计原则。 - -## 整体架构 - -### 核心设计原则 - -1. **分层解耦**：通过明确的包边界分离关注点 -2. **类型安全**：使用 TypeScript 和 Zod 确保类型安全 -3. **可扩展性**：支持 MCP（Model Context Protocol）工具扩展 -4. **会话管理**：基于状态机的多轮对话管理 -5. **工具抽象**：统一的工具接口，支持内置和外部工具 - -### 包结构 - -``` -memo-code/ -├── packages/ -│ ├── core/ # 核心运行时和会话管理 -│ ├── tools/ # 工具系统和 MCP 集成 -│ ├── tui/ # 终端用户界面 -├── site/ # 文档站点 -└── dist/ # 构建输出 -``` - -## 核心模块详解 - -### 1. Core 包 (`@memo-code/core`) - -**职责**：会话状态机、提示词构建、历史管理、配置处理 - -#### 核心组件 - -**会话运行时 (`session_runtime.ts`)** -- 实现 ReAct（Reasoning and Acting）循环 -- 处理工具调度和执行 -- 管理上下文压缩和摘要 -- 事件驱动的会话日志记录 - -```typescript -// 核心会话接口 -interface AgentSession { - sessionId: string - mode: SessionMode - turn(input: string): Promise - getHistory(): ChatMessage[] - close(): Promise -} -``` - -**提示词系统 (`prompt.ts`)** -- 动态提示词构建 -- 技能（Skills）注入机制 -- 上下文窗口管理 -- 模型特定的提示词适配 - -**历史管理 (`history.ts`, `history_parser.ts`)** -- JSONL 格式的持久化存储 -- 增量历史加载和索引 -- 上下文压缩策略 -- 会话恢复机制 - -**配置系统 (`config/`)** -- TOML 格式的配置文件 -- 环境变量和 CLI 参数集成 -- 多提供商模型配置 -- 动态配置重载 - -#### 数据流 - -``` -用户输入 → 会话状态机 → 提示词构建器 → LLM 调用 → 工具执行 → 响应生成 -``` - -### 2. Tools 包 (`@memo-code/tools`) - -**职责**：工具定义、执行编排、MCP 集成、安全控制 - -#### 工具架构 - -**统一工具接口** -```typescript -interface McpTool { - name: string - description: string - source: "native" | "mcp" - inputSchema: JSONSchema - validateInput(input: unknown): ValidationResult - execute(input: unknown): Promise -} -``` - -**内置工具分类** -- **文件系统**：`read_text_file`, `write_file`, `edit_file`, `list_directory`, `search_files` -- **执行环境**：`exec_command`, `shell`, `shell_command` -- **网络操作**：`webfetch`, `read_media_file` -- **协作工具**：`spawn_agent`, `send_input`, `wait`, `close_agent`, `resume_agent` -- **高级功能**：`apply_patch`, `update_plan`, `get_memory` - -**MCP 集成** -- stdio 和 HTTP 适配器支持 -- 动态工具发现和注册 -- 连接池和生命周期管理 -- 资源模板和读取支持 - -#### 安全机制 - -**批准系统 (`approval/`)** -- 风险分类和指纹识别 -- 多级批准策略（once/session/deny） -- 批准缓存和决策历史 -- 用户交互界面集成 - -**沙盒策略** -- 文件系统访问控制 -- 可写根目录限制 -- 命令执行白名单 -- 网络访问限制 - -#### 执行编排 - -**工具编排器 (`orchestrator/`)** -- 并行工具调用支持 -- 失败策略和错误处理 -- 结果聚合和格式化 -- 执行状态跟踪 - -### 3. TUI 包 (`@memo-code/tui`) - -**职责**：终端界面、交互控制、用户输入处理 - -#### 技术栈 -- **React + Ink**：基于 React 的终端 UI 框架 -- **@inkjs/ui**：预构建的 UI 组件库 -- **状态管理**：基于 React hooks 的本地状态 - -#### 核心组件 - -**应用主体 (`App.tsx`)** -- 聊天历史渲染 -- 输入区域管理 -- 状态指示器 -- 错误处理和重试 - -**斜杠命令系统 (`slash/`)** -- 命令注册和分发 -- 参数解析和验证 -- 自动补全和帮助 -- 扩展机制 - -**控制器层 (`controllers/`)** -- 会话历史管理 -- 配置界面 -- 批准流程 -- MCP 管理 - -#### 交互模式 - -**TUI 模式** -- 全屏交互式界面 -- 实时状态更新 -- 键盘快捷键支持 -- 多面板布局 - -**纯文本模式** -- 非交互式环境支持 -- 流式输出 -- 简化的错误处理 -- CI/CD 集成 - -## 数据流和交互 - -### 典型会话流程 - -```mermaid -sequenceDiagram - participant U as 用户 - participant C as 客户端 (TUI) - participant S as 会话核心 - participant T as 工具系统 - participant L as LLM 提供商 - - U->>C: 输入消息 - C->>S: 创建/获取会话 - S->>S: 构建提示词 - S->>L: LLM API 调用 - L-->>S: 响应 (包含工具调用) - S->>T: 执行工具调用 - T-->>S: 工具执行结果 - S->>S: 处理结果并更新历史 - S-->>C: 会话响应 - C-->>U: 显示结果 -``` - -### 工具执行流程 - -```mermaid -sequenceDiagram - participant S as 会话核心 - participant O as 工具编排器 - participant R as 工具路由器 - participant A as 批准系统 - participant N as 本地工具 - participant M as MCP 工具 - - S->>O: 调度工具调用 - O->>R: 查找工具 - R-->>O: 工具定义 - O->>A: 检查批准需求 - A-->>O: 批准决策 - O->>N: 执行本地工具 - OR O->>M: 执行 MCP 工具 - N-->>O: 执行结果 - OR M-->>O: 执行结果 - O-->>S: 聚合结果 -``` - -## 配置和部署 - -### 配置管理 - -**主配置文件 (`~/.memo/config.toml`)** -```toml -[provider] -type = "openai" # openai, deepseek, ollama -api_key = "your-api-key" -base_url = "https://api.openai.com/v1" -model = "gpt-4" - -[session] -context_window = 128000 -auto_compact_threshold = 0.8 -history_file = "~/.memo/sessions/session.jsonl" - -[tools] -approval_mode = "session" # once, session, deny -parallel_execution = true -sandbox_roots = ["/tmp", "./workspace"] - -[mcp] -servers = [ - { name = "filesystem", command = "npx", args = ["@modelcontextprotocol/server-filesystem", "/tmp"] } -] -``` - -### 部署模式 - -**CLI 模式** -```bash -# 安装 -npm install -g @memo-code/memo - -# 运行 TUI -memo - -# 纯文本模式 -echo "help me debug" | memo -``` - -## 扩展机制 - -### 自定义工具开发 - -**创建本地工具** -```typescript -// src/tools/my_tool.ts -import { defineMcpTool } from '@memo/tools/tools/types' - -export const myTool = defineMcpTool({ - name: 'my_tool', - description: '自定义工具描述', - inputSchema: { - type: 'object', - properties: { - input: { type: 'string' } - } - }, - async execute({ input }) { - return { - content: [{ type: 'text', text: `处理结果: ${input}` }] - } - } -}) -``` - -**MCP 服务器集成** -```json -{ - "name": "my-mcp-server", - "command": "node", - "args": ["./my-mcp-server.js"], - "env": { - "API_KEY": "your-key" - } -} -``` - -### 技能系统 - -**SKILL.md 格式** -```markdown -# 技能名称 - -## 描述 -技能的详细描述 - -## 参数 -- param1: 参数描述 -- param2: 参数描述 - -## 示例 -使用示例和说明 -``` - -## 性能和安全 - -### 性能优化 - -**上下文管理** -- 智能上下文压缩 -- 分层历史缓存 -- 增量加载策略 -- Token 使用优化 - -**并发执行** -- 工具并行调用 -- 异步 I/O 处理 -- 连接池管理 -- 资源限制 - -### 安全措施 - -**输入验证** -- Zod 模式验证 -- 类型安全检查 -- 输入清理和过滤 -- 注入攻击防护 - -**权限控制** -- 文件系统沙盒 -- 命令执行限制 -- 网络访问控制 -- 用户权限隔离 - -**审计日志** -- 完整的操作记录 -- 工具调用追踪 -- 错误事件记录 -- 安全事件监控 - -## 监控和调试 - -### 日志系统 - -**结构化日志** -```typescript -interface LogEvent { - timestamp: string - level: 'debug' | 'info' | 'warn' | 'error' - component: string - sessionId?: string - message: string - metadata?: Record -} -``` - -**调试工具** -- 会话历史分析 -- 工具调用追踪 -- 性能指标收集 -- 错误堆栈分析 - -### 测试策略 - -**单元测试** -- 核心逻辑测试 -- 工具契约验证 -- 配置系统测试 -- 错误处理测试 - -**集成测试** -- 端到端会话测试 -- MCP 服务器集成 -- 并发场景测试 - -## 未来规划 - -### 短期目标 -- 增强多模态支持 -- 优化上下文压缩算法 -- 扩展工具生态系统 -- 改进用户界面体验 - -### 长期愿景 -- 分布式会话管理 -- 插件系统架构 -- 企业级部署支持 -- AI 驱动的自动化优化 - ---- - -本文档反映了 Memo 系统的当前架构设计，随着系统演进会持续更新。如需了解最新的实现细节，请参考各包的源代码和测试文件。 diff --git a/docs/model-agnostic-design.md b/docs/model-agnostic-design.md deleted file mode 100644 index 82f6fd4..0000000 --- a/docs/model-agnostic-design.md +++ /dev/null @@ -1,211 +0,0 @@ -# 模型无关设计：Memo CLI 如何消除模型差异 - -## 设计理念 - -Memo CLI 通过统一的接口层设计，实现对不同 LLM 服务的透明接入。上层应用（Session、TUI）无需关心底层使用的具体模型，只要该模型兼容 OpenAI API 格式即可。 - -## 四层架构 - -### 1. Provider 配置层 - -位置：`packages/core/src/config/config.ts` - -```typescript -type ProviderConfig = { - name: string // 提供商标识 - env_api_key: string // API Key 环境变量名 - model: string // 模型名称 - base_url?: string // API 基础 URL -} - -type MemoConfig = { - current_provider: string // 当前选中的 provider - providers: ProviderConfig[] // 支持的 provider 列表 - // ... -} -``` - -**配置示例**（`~/.memo/config.toml`）： - -```toml -current_provider = "deepseek" - -[[providers.deepseek]] -name = "deepseek" -env_api_key = "DEEPSEEK_API_KEY" -model = "deepseek-chat" -base_url = "https://api.deepseek.com" - -[[providers.openai]] -name = "openai" -env_api_key = "OPENAI_API_KEY" -model = "gpt-4o" -base_url = "https://api.openai.com/v1" - -[[providers.ollama]] -name = "ollama" -env_api_key = "OLLAMA_API_KEY" -model = "llama3" -base_url = "http://localhost:11434/v1" -``` - -### 2. 统一 HTTP 客户端层 - -位置：`packages/core/src/runtime/defaults.ts:147-174` - -使用 OpenAI SDK 作为统一接口： - -```typescript -const client = new OpenAI({ - apiKey, - baseURL: provider.base_url, // 关键：不同 Provider 只需配置正确的 base_url -}) -``` - -**核心逻辑**： - -- 读取配置中选中的 Provider -- 根据环境变量获取 API Key -- 使用 OpenAI SDK 发送请求（任何兼容 OpenAI API 的服务都可用） - -### 3. 消息格式转换层 - -位置：`packages/core/src/runtime/defaults.ts:34-60` - -将内部 `ChatMessage` 格式转换为 OpenAI API 格式： - -```typescript -function toOpenAIMessage(message: ChatMessage): OpenAI.ChatCompletionMessageParam { - if (message.role === 'assistant') { - return { - role: 'assistant', - content: message.content, - tool_calls: message.tool_calls?.map((toolCall) => ({ - id: toolCall.id, - type: toolCall.type, - function: { - name: toolCall.function.name, - arguments: toolCall.function.arguments, - }, - })), - } - } - if (message.role === 'tool') { - return { - role: 'tool', - content: message.content, - tool_call_id: message.tool_call_id, - } - } - return { - role: message.role, - content: message.content, - } -} -``` - -**支持的消息类型**： - -- `system`: 系统提示词 -- `user`: 用户输入 -- `assistant`: 助手输出（包含 `tool_calls`） -- `tool`: 工具执行结果 - -### 4. 响应格式归一化层 - -位置：`packages/core/src/runtime/defaults.ts:176-236` - -将模型响应转换为内部统一的 `LLMResponse` 格式： - -```typescript -type LLMResponse = { - content: ContentBlock[] // 内容块：text 或 tool_use - stop_reason: 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' - usage?: Partial // Token 使用统计 -} - -type ContentBlock = - | { type: 'text'; text: string } - | { type: 'tool_use'; id: string; name: string; input: unknown } -``` - -**转换流程**： - -1. 解析 OpenAI API 的 `tool_calls` -2. 提取文本内容和工具调用 -3. 统一 `stop_reason` 语义 -4. 归一化 token 统计数据 - -## Provider 选择机制 - -位置：`packages/core/src/config/config.ts:203-208` - -```typescript -export function selectProvider(config: MemoConfig, preferred?: string): ProviderConfig { - const name = preferred || config.current_provider - const found = config.providers.find((p) => p.name === name) - if (found) return found - return config.providers?.[0] ?? DEFAULT_CONFIG.providers[0]! -} -``` - -**切换方式**： - -- 配置文件修改 `current_provider` -- CLI 命令 `/models` 交互式切换 -- 代码指定 `providerName` 参数 - -## 支持的模型类型 - -只要兼容 OpenAI Chat Completions API 的模型均可接入： - -| Provider | Model | Base URL | -| ---------------------- | ------------------------ | ------------------------- | -| DeepSeek | deepseek-chat | https://api.deepseek.com | -| OpenAI | gpt-4o, gpt-4o-mini | https://api.openai.com/v1 | -| Anthropic | claude-3-opus (通过代理) | 代理地址 | -| Ollama | llama3, mistral | http://localhost:11434/v1 | -| Azure OpenAI | gpt-4 | Azure 端点 | -| vLLM | 各种开源模型 | vLLM 端点 | -| 其他兼容 OpenAI 的服务 | - | - | - -## 使用示例 - -### 切换 Provider - -```bash -# 交互式切换 -memo -/models -# 选择 deepseek - -# 或直接指定 -/models openai -``` - -### 自定义 Provider - -编辑 `~/.memo/config.toml`： - -```toml -[[providers.custom]] -name = "custom" -env_api_key = "CUSTOM_API_KEY" -model = "your-model-name" -base_url = "https://your-api-endpoint.com/v1" -``` - -## 架构优势 - -1. **零学习成本**：符合 OpenAI API 的服务无需额外适配 -2. **灵活切换**：运行时切换 Provider，无需重启 -3. **统一体验**：不同模型提供一致的工具调用和响应格式 -4. **易于扩展**：新增 Provider 只需配置，无需修改代码 -5. **降低依赖**：不依赖特定模型的私有 API - -## 相关文件 - -- `packages/core/src/config/config.ts` - Provider 配置管理 -- `packages/core/src/runtime/defaults.ts` - HTTP 客户端和消息转换 -- `packages/core/src/types.ts` - 统一类型定义 -- `packages/tui/src/slash/registry.ts` - CLI 命令处理 diff --git a/docs/npm-distribution-design.md b/docs/npm-distribution-design.md deleted file mode 100644 index 87c2410..0000000 --- a/docs/npm-distribution-design.md +++ /dev/null @@ -1,298 +0,0 @@ -# NPM Distribution Design - -## 1. Design Goals - -### 1.1 Core Requirements - -- **Cross-platform compatibility**: support macOS, Linux, and Windows without per-platform recompilation -- **Zero runtime dependency setup for users**: package all dependencies into one deliverable file -- **Standard Node.js runtime**: require only Node.js >=22.0.0 (no Bun-specific runtime dependency) -- **Small package size**: published package under 100KB for fast install - -### 1.2 Comparison with Binary Distribution - -| Feature | NPM Distribution | Binary Distribution | -| -------------------- | ----------------------------- | ------------------------------ | -| Cross-platform | ✅ build once, run everywhere | ❌ separate build per platform | -| Signing requirements | none | macOS/Windows signing required | -| Package size | ~38KB | ~50-100MB | -| Update workflow | `npm update` | manual download/replace | -| Runtime requirement | Node.js >=22.0.0 | none | -| Install speed | fast | slower | - -## 2. Architecture - -### 2.1 Overall Architecture - -```text -┌─────────────────────────────────────────┐ -│ @memo-code/memo │ -│ ┌─────────────────────────────────┐ │ -│ │ dist/index.js (ESM) │ │ ← single entry file -│ │ - CLI logic │ │ -│ │ - Core runtime │ │ -│ │ - Tools implementation │ │ -│ │ - UI (React/Ink) │ │ -│ └─────────────────────────────────┘ │ -│ ↓ │ -│ ┌─────────────────────────────────┐ │ -│ │ dist/prompt.md │ │ ← runtime resource -│ └─────────────────────────────────┘ │ -└─────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ Node.js Runtime (>=20) │ -└─────────────────────────────────────────┘ -``` - -### 2.2 Build Pipeline - -```text -Source Code Build Output -──────────── ──────────── -packages/tui/src/ - └─ cli.tsx ───┐ dist/ - ├─tsup──→ ├─ index.js (bundled) -packages/core/src/ ─┤ │ - React/Ink UI - ├─ runtime/ │ │ - Session management - ├─ config/ │ │ - LLM invocation - └─ ... │ │ - Token counting - │ │ -packages/tools/src/ ─┤ │ ← all dependencies inlined - ├─ exec_command.ts │ │ - ├─ read_text_file.ts │ │ - ├─ write_file.ts │ │ - ├─ search_files.ts │ │ - └─ ... │ │ - │ │ -packages/core/src/ │ │ - └─ runtime/ │ │ - └─ prompt.md ──┴──────────→├─ prompt.md -``` - -### 2.3 Dependency Strategy - -| Dependency Type | Handling | Reason | -| --------------- | ------------- | -------------------------------- | -| `react`, `ink` | bundle inline | required at runtime | -| `fast-glob` | bundle inline | avoid user-side install concerns | -| `openai` | bundle inline | API client | -| `tiktoken` | bundle inline | token counting | -| `zod` | bundle inline | schema validation | -| Node built-ins | `external` | provided by Node.js | - -## 3. Key Implementation Details - -### 3.1 Build Config (`tsup`) - -```typescript -export default defineConfig({ - entry: ['packages/tui/src/cli.tsx'], - format: ['esm'], // ESM format - target: 'node18', // minimum Node.js version - bundle: true, // bundle all dependencies - minify: true, // minify code - splitting: false, // single file output - external: [], // no external runtime deps - banner: { - js: '#!/usr/bin/env node', // shebang - }, - onSuccess() { - // copy runtime resource file - copyFileSync('packages/core/src/runtime/prompt.md', 'dist/prompt.md') - }, -}) -``` - -### 3.2 Resource File Handling - -**Problem**: `prompt.md` is a runtime-read Markdown template. - -**Solution**: - -- Copy it to `dist/prompt.md` during build -- Include it explicitly in `package.json` `files` -- Locate it with `__dirname` at runtime - -```typescript -const __dirname = dirname(fileURLToPath(import.meta.url)) -const promptPath = join(__dirname, 'prompt.md') -const prompt = await readFile(promptPath, 'utf-8') -``` - -### 3.3 Path Alias Resolution - -**During development** (`tsconfig.json`): - -```json -{ - "paths": { - "@memo/core": ["packages/core/src/index.ts"], - "@memo/core/*": ["packages/core/src/*"], - "@memo/tools": ["packages/tools/src/index.ts"], - "@memo/tools/*": ["packages/tools/src/*"] - } -} -``` - -**During build**: tsup resolves and inlines aliases automatically. - -**During test** (`vitest.config.ts`): - -```typescript -import tsconfigPaths from 'vite-tsconfig-paths' - -export default defineConfig({ - plugins: [tsconfigPaths()], -}) -``` - -## 4. Cross-platform Compatibility - -### 4.1 File Path Handling - -```typescript -import { join, normalize } from 'node:path' - -// correct -const filePath = join(process.cwd(), 'config.toml') - -// avoid hard-coded separators -const filePath2 = `${process.cwd()}/config.toml` -``` - -### 4.2 Environment Detection - -```typescript -const rgAvailable = (() => { - const result = spawnSync('rg', ['--version'], { stdio: 'ignore' }) - return !result.error && result.status === 0 -})() -``` - -### 4.3 Shell Command Execution - -```typescript -spawn('bash', ['-lc', command], { - env: process.env, - stdio: ['ignore', 'pipe', 'pipe'], -}) -``` - -## 5. Release Workflow - -### 5.1 CI/CD Flow - -```yaml -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - - uses: actions/setup-node@v4 - - - run: pnpm install - - run: pnpm run format:check - - run: pnpm run test - - run: pnpm run build -``` - -### 5.2 Manual Release Steps - -```bash -# 1) ensure everything passes -pnpm run ci - -# 2) bump version -npm version patch # or minor/major - -# 3) build and publish -npm publish --access public -``` - -### 5.3 Post-release Validation - -```bash -# 1) clear local cache -npm cache clean --force - -# 2) global install test -npm install -g @memo-code/memo - -# 3) runtime verification -memo --version -memo "test prompt" -``` - -## 6. Installation Modes - -| Mode | Command | Typical Use | -| -------------- | -------------------------------- | -------------------- | -| Global install | `npm install -g @memo-code/memo` | daily usage | -| pnpm global | `pnpm add -g @memo-code/memo` | pnpm users | -| npx run | `npx @memo-code/memo` | temporary usage | -| Local install | `npm install @memo-code/memo` | project-scoped usage | - -## 7. Troubleshooting Design - -### 7.1 Common Issues - -| Issue | Cause | Fix | -| ------------------------- | -------------------------------- | ---------------------------------------- | -| `command not found` | global bin directory not in PATH | add `$(npm bin -g)` to PATH | -| `prompt.md not found` | resource file not copied | ensure `files` includes `dist/prompt.md` | -| `ERR_MODULE_NOT_FOUND` | path aliases unresolved | ensure fully bundled build output | -| Windows execution failure | PowerShell policy | `Set-ExecutionPolicy RemoteSigned` | - -### 7.2 Debug Mode - -```bash -# verbose logs -DEBUG=* memo - -# check config -memo --config - -# run diagnostics -memo --doctor -``` - -## 8. Security Considerations - -### 8.1 Dependency Security - -- lock dependency versions at build time -- run `npm audit` regularly -- avoid dynamic `require()` where possible - -### 8.2 Runtime Security - -- approve risky tools before execution (`exec_command`, `shell`, `apply_patch`) -- enforce path allowlists -- run external commands in controlled environments - -## 9. Future Extensions - -### 9.1 Possible Optimizations - -- **Code splitting**: lazy-load large dependencies (for example tiktoken wasm) -- **Compression**: use Brotli to reduce package size further -- **Incremental updates**: support hot-update style mechanism - -### 9.2 Platform-specific Improvements - -- **macOS**: consider Notarization if distributing as an app -- **Windows**: provide PowerShell module -- **Linux**: provide snap/flatpak package - -## 10. Summary - -This design achieves efficient cross-platform distribution through: - -1. **Single entry file**: all code bundled into `dist/index.js` -2. **Bundled resource**: `prompt.md` ships with package -3. **Minimal runtime requirement**: users only need Node.js -4. **Standard toolchain**: pnpm + tsup + vitest - -Compared with binary distribution, NPM distribution gives better cross-platform compatibility and much smaller package size for Node.js-based CLI tools. diff --git a/docs/subagent-system-design-summary.md b/docs/subagent-system-design-summary.md deleted file mode 100644 index 0c480f8..0000000 --- a/docs/subagent-system-design-summary.md +++ /dev/null @@ -1,163 +0,0 @@ -# Memo 项目 Subagent 系统设计实现方案总结（架构师视角） - -## 1. 设计目标与定位 - -Subagent（子代理）系统的目标是： - -- 将主会话中的大任务拆解为可并行执行的有界子任务； -- 在不引入复杂分布式基础设施的前提下，提供“轻量级多代理协作”； -- 通过统一工具体系接入主流程，保持与现有 Tool Router / Orchestrator / Approval 机制一致； -- 通过明确的生命周期与状态模型，保证可控性、可恢复性和可观测性。 - -在当前架构中，Subagent 被实现为 **工具层的协作工具族**，而不是单独的调度服务：`spawn_agent`、`send_input`、`resume_agent`、`wait`、`close_agent`。 - ---- - -## 2. 架构分层与职责映射 - -### 2.1 系统分层映射 - -Subagent 并未破坏 Memo 既有四层工具架构，而是嵌入其中： - -1. **工具实现层（packages/tools/src/tools/collab.ts）** - - 维护子代理运行态（内存 Map）； - - 负责子进程拉起、终止、状态收敛、输出汇总； - - 提供 5 个标准工具接口。 - -2. **路由层（ToolRouter）** - - 与其他工具一致统一注册/发现/执行； - - 对上层透明，调用方无需区分 subagent 与普通工具。 - -3. **编排层（Orchestrator）** - - 复用统一输入校验与结果裁剪机制； - - 保持工具调用顺序/并行策略一致。 - -4. **审批层（Approval）** - - subagent 工具风险级别归类为 `read`； - - 且位于 `ALWAYS_AUTO_APPROVE_TOOLS`，默认不阻塞审批链路。 - -### 2.2 配置驱动与开关控制 - -- `MEMO_ENABLE_COLLAB_TOOLS=0`：整体关闭子代理工具族； -- `MEMO_SUBAGENT_COMMAND`：指定子代理进程启动命令； -- `MEMO_SUBAGENT_MAX_AGENTS`：并发运行子代理上限（默认 4）。 - -这体现了“**默认可用、显式可禁用、容量可调优**”的产品策略。 - ---- - -## 3. 核心运行模型 - -### 3.1 控制平面：Agent Record - -系统在内存中维护 `Map`，每个 AgentRecord 持有： - -- 标识与时间：`id`、`createdAt`、`updatedAt`； -- 生命周期状态：`running | completed | errored | closed`； -- 恢复语义：`statusBeforeClose`； -- 最近上下文：`lastMessage`、`lastSubmissionId`、`lastOutput`、`lastError`； -- 当前运行态：`running`（包含 process、startedAt、interrupted）。 - -该模型实现了“**轻量状态机 + 最小审计信息**”的平衡。 - -### 3.2 数据平面：Submission 执行 - -每次 `spawn_agent`/`send_input` 都会触发一次 submission： - -1. 解析并发上限； -2. 解析启动命令（环境变量 > dist fallback > memo fallback）； -3. `spawn(..., shell: true)` 拉起子进程； -4. 写入 message 到 stdin 后关闭 stdin； -5. 监听 stdout/stderr/close 汇总结果； -6. close 时根据退出码与中断标记收敛状态。 - -### 3.3 状态机语义 - -- 初始执行：`running`； -- 退出码 0：`completed`； -- 非 0 或中断：`errored`； -- 显式关闭：`closed`； -- `resume_agent`：仅恢复关闭前状态，不自动产生新 submission； -- `wait` 对未知 id 返回 `not_found`（只在 wait 结果域出现）。 - ---- - -## 4. 工具体系与协作协议 - -### 4.1 `spawn_agent` - -- 创建 agent 记录并立即启动首个 submission； -- 返回 `agent_id` + `submission_id` + 状态摘要； -- 达到并发上限时失败。 - -### 4.2 `send_input` - -- 向已有 agent 提交新任务； -- 若正在运行：默认 busy 错误；可 `interrupt=true` 先终止再提交； -- 对已关闭 agent 强制要求先 `resume_agent`。 - -### 4.3 `wait` - -- 针对 `ids` 轮询直到出现“任一最终状态”或超时； -- 超时区间限制：10s~300s，默认 30s； -- 返回 `status/details` 快照与 `timed_out` 标记。 - -### 4.4 `close_agent` / `resume_agent` - -- `close_agent`：可终止在跑 submission 并封存为 closed； -- `resume_agent`：恢复为 `statusBeforeClose`，用于续作而非自动执行。 - ---- - -## 5. 安全设计与风险边界 - -### 5.1 当前安全策略 - -- 子代理工具默认自动批准（含 strict 下白名单跳过）； -- 主风险在于子进程命令执行能力，因此系统提示明确要求“任务范围必须有边界”； -- 通过并发上限防止无限扩张； -- 通过 `close_agent` 与中断机制控制资源回收。 - -### 5.2 风险与治理建议（架构视角） - -1. **命令面风险**：当前实现通过 `spawn(..., shell: true)` 执行子代理命令，`MEMO_SUBAGENT_COMMAND` 若配置不当会放大命令注入面；建议生产环境固化可执行模板并限制可注入变量来源。 -2. **内存态风险**：当前状态仅驻留进程内存，进程重启后不可恢复；若后续面向长会话，可引入轻量持久化。 -3. **并发饥饿风险**：单全局并发阈值对复杂任务可能过紧或过松；建议演进为“全局+会话”双层限流。 - ---- - -## 6. 可观测性与运维特征 - -- `wait` 返回结构化 details，可作为主会话的最小观测面； -- 输出会做预览裁剪（防止过长污染上下文）； -- 错误语义明确（not_found / busy / interrupted / exit code）； -- 测试覆盖关键链路：成功、关闭恢复、未知 id、并发上限、参数校验。 - -整体上，该系统提供了“**可用优先、治理逐步增强**”的工程实现路径。 - ---- - -## 7. 与主会话协同的设计要点 - -主提示词已内置子代理使用规约： - -- 只用于可分解任务； -- 避免递归 spawn； -- 子任务 prompt 应简洁且有交付边界； -- `wait` 后汇总回主线程； -- 完成后 `close_agent` 释放资源。 - -这使得 Subagent 在产品行为层面形成“**工具能力 + Prompt 策略**”的双重约束。 - ---- - -## 8. 架构结论 - -Memo 的 Subagent 设计采用了“**内聚在工具层的轻量多代理方案**”： - -- 复用现有工具基础设施，集成成本低； -- 生命周期语义清晰，满足多数并行协作场景； -- 通过环境变量完成启停、命令注入和容量调优； -- 当前实现偏本地会话内协作，后续可在持久化、隔离级别、调度公平性方向继续演进。 - -从系统架构角度看，这是一个在复杂度、可维护性和交付速度之间取舍合理的 V1 设计。 diff --git a/docs/token-counting.md b/docs/token-counting.md deleted file mode 100644 index 0ae24cc..0000000 --- a/docs/token-counting.md +++ /dev/null @@ -1,41 +0,0 @@ -# Token Counting Strategy in Memo Code - -This document describes how Memo Code estimates and records tokens for prompt budgeting, context-limit protection, and usage reconciliation. - -## Counting Implementation - -- **Underlying encoder**: uses `@dqbd/tiktoken`, default encoding `cl100k_base`; override via `tokenizerModel`. -- **Plain text count**: `countText(text)` encodes a string directly and returns token length. -- **Message array count (ChatML approximation)**: `countMessages(messages)` uses a common OpenAI ChatML estimate: - - fixed overhead of 4 tokens per message (role/name wrappers, etc.) - - `content` counted via tiktoken encoding - - if `name` is supported later, adds 1 token - - adds 2 tokens at the end for assistant priming - -This is closer to actual ChatML overhead than naive text concatenation, but still an approximation. - -## Usage Scenarios - -- **Prompt budgeting**: before each step, `runTurn` estimates prompt tokens with `countMessages` and applies: - - `warnPromptTokens`: prints warning - - `maxPromptTokens`: returns early when exceeded, preventing over-limit LLM requests -- **Usage reconciliation**: each step combines local count and model-returned `usage` (if available), records into token usage and JSONL history events. - -## Precision and Limitations - -- Fixed ChatML overhead varies slightly by model. Current "4 per message + 2 ending" estimate may differ by dozens of tokens on specific models. -- Extra structural overhead for tool/function calling is not explicitly modeled yet. For exact reconciliation, model-specific constants can be added later. -- If using custom `callLLM`, pass matching model encoding or custom `tokenCounter` implementation to align with real usage. - -## How to Override - -- Pass `tokenizerModel` or inject custom `tokenCounter` when creating Session: - -```ts -import { createTokenCounter, createAgentSession } from '@memo/core' - -const tokenCounter = createTokenCounter('gpt-4o-mini') -const session = await createAgentSession({ tokenCounter }, { warnPromptTokens: 8_000 }) -``` - -- A custom counter only needs `countText`, `countMessages`, and `dispose` methods. diff --git a/docs/tool/apply_patch.md b/docs/tool/apply_patch.md deleted file mode 100644 index a18fdc8..0000000 --- a/docs/tool/apply_patch.md +++ /dev/null @@ -1,54 +0,0 @@ -# Memo CLI `apply_patch` Tool - -Applies structured patch text to local files. - -## Basic Info - -- Tool name: `apply_patch` -- Description: Apply a structured patch envelope (`*** Begin Patch` ... `*** End Patch`) with Add/Delete/Update hunks. -- File: `packages/tools/src/tools/apply_patch.ts` -- Confirmation: no - -## Parameters - -- `input` (string, required): full patch text. - -Patch format: - -``` -*** Begin Patch -*** Add File: path/to/file -+line -*** Update File: path/to/existing -*** Move to: path/to/new -@@ optional context --old line -+new line -*** Delete File: path/to/delete -*** End Patch -``` - -## Behavior - -- Supports `Add File`, `Delete File`, `Update File`, optional `Move to`, `@@` chunks, and `*** End of File`. -- Requires relative file paths (absolute paths are rejected). -- Resolves paths against runtime cwd and enforces writable-root sandbox policy. -- Computes update replacements with tolerant matching (`exact` -> `trimEnd` -> `trim` -> normalized unicode punctuation). -- Returns `isError=true` for parse failures, missing files/context, sandbox denial, or invalid input. - -## Output Example - -Success: - -``` -Success. Updated the following files: -A nested/new.txt -M src/app.ts -D obsolete.txt -``` - -Failure: - -``` -Invalid patch hunk on line 4: Expected update hunk to start with a @@ context marker, got: '...' -``` diff --git a/docs/tool/architecture.md b/docs/tool/architecture.md deleted file mode 100644 index b0352ff..0000000 --- a/docs/tool/architecture.md +++ /dev/null @@ -1,307 +0,0 @@ ---- -title: Tool Module Architecture -description: Detailed explanation of memo's tool system design ---- - -# Tool Module Architecture - -## Overview - -The tool system in memo is designed to enable AI assistants (Agents) to safely use various capabilities like executing commands, reading/writing files, making web requests, and more. This document explains the four-layer architecture that makes this possible. - -## 1. Core Design Philosophy - -**Goal**: Allow AI assistants to safely and effectively use tools to help users with software engineering tasks. - -**Key Challenges**: - -- How to manage many different tools uniformly? -- How to ensure safety (prevent dangerous operations)? -- How to let AI know what tools are available? -- How to handle both built-in and external (MCP) tools? - -## 2. Four-Layer Architecture (Onion Model) - -### Layer 1: Tool Implementation Layer (Innermost) - -``` -packages/tools/src/tools/ -├── exec_command.ts # Execute shell commands -├── read_text_file.ts # Read text file contents -├── read_media_file.ts # Read media file as base64 payload -├── read_files.ts # Read multiple text files -├── write_file.ts # Atomic file write -├── edit_file.ts # Structured text edits + diff -├── apply_patch.ts # Modify files -├── webfetch.ts # Make HTTP requests -├── list_directory.ts # List directory contents -├── search_files.ts # Search files by glob path pattern -├── update_plan.ts # Update task plans -├── get_memory.ts # Access persisted memory -└── ... -``` - -**Characteristics**: - -- Each tool is independent and self-contained -- Uses `defineMcpTool()` for consistent interface -- Implements specific functionality without dependencies on other layers -- Includes input validation and error handling - -**Example Tool Definition**: - -```typescript -export const readTextFileTool = defineMcpTool({ - name: 'read_text_file', - description: 'Read text with optional head/tail line limits', - inputSchema: z - .object({ - path: z.string().min(1), - head: z.number().int().positive().optional(), - tail: z.number().int().positive().optional(), - }) - .strict(), - execute: async (input) => { - // validatePath + read - return textResult('...') - }, -}) -``` - -### Layer 2: Routing Layer (Tool Manager) - -``` -packages/tools/src/router/ -├── index.ts # ToolRouter - Main coordinator -├── native/index.ts # Built-in tool registry -├── mcp/index.ts # External MCP tool registry -└── types.ts # Unified interface definitions -``` - -**ToolRouter Responsibilities**: - -1. **Registration**: `registerNativeTool()` - Store tools in registry -2. **Discovery**: `getTool("read_text_file")` - Find tools by name -3. **Execution**: `execute("read_text_file", {...})` - Run tools with input -4. **Documentation**: `generateToolDescriptions()` - Create tool list for AI -5. **Unified Interface**: Handle both native and MCP tools transparently - -**Key Methods**: - -- `getAllTools()`: Returns all available tools -- `generateToolDefinitions()`: Creates API-compatible tool definitions -- `hasTool(name)`: Checks if tool exists -- `dispose()`: Cleans up resources (closes MCP connections) - -### Layer 3: Orchestration Layer (Execution Scheduler) - -``` -packages/tools/src/orchestrator/ -├── index.ts # Execution orchestrator -└── types.ts # Execution-related types -``` - -**Orchestrator Responsibilities**: - -1. **Request Handling**: Receive tool execution requests from AI -2. **Safety Check**: Call approval layer for risk assessment -3. **Tool Invocation**: Use router to find and execute tools -4. **Result Processing**: Limit output size, format errors, handle timeouts -5. **Parallel Execution**: Manage concurrent tool calls when supported - -**Output Size Control**: - -- Limits tool results to prevent token overflow -- Configurable via `MEMO_TOOL_RESULT_MAX_CHARS` environment variable -- Provides clear hints when output is truncated - -### Layer 4: Approval Layer (Security Guard) - -``` -packages/tools/src/approval/ -├── classifier.ts # Risk classifier -├── fingerprint.ts # Request fingerprinting -├── manager.ts # Approval manager -├── constants.ts # Risk level constants -└── types.ts # Security types -``` - -**Security Mechanisms**: - -#### Risk Classification - -Tools are automatically classified into risk levels: - -- **read**: Low risk (e.g., `read_text_file`, `list_directory`, `webfetch`) -- **write**: Medium risk (e.g., `apply_patch`, file modifications) -- **execute**: High risk (e.g., `exec_command`, shell operations) - -#### Approval Modes - -- **auto mode**: Only `write` and `execute` tools require approval -- **strict mode**: All tools require approval -- **fingerprinting**: Unique request IDs for audit trails - -#### Default Risk Levels - -```typescript -const DEFAULT_TOOL_RISK_LEVELS: Record = { - exec_command: 'execute', - write_stdin: 'execute', - shell: 'execute', - shell_command: 'execute', - apply_patch: 'write', - write_file: 'write', - edit_file: 'write', - read_text_file: 'read', - read_media_file: 'read', - read_files: 'read', - list_directory: 'read', - search_files: 'read', - webfetch: 'read', - update_plan: 'read', - get_memory: 'read', -} -``` - -## 3. Unified Tool Interface - -All tools implement this common interface: - -```typescript -interface Tool { - name: string // Unique tool identifier - description: string // Human-readable description - source: 'native' | 'mcp' // Tool origin - inputSchema: JSONSchema // Input parameter schema - supportsParallelToolCalls?: boolean // Can run concurrently - isMutating?: boolean // Modifies external state - validateInput?: (input: unknown) => ValidationResult - execute: (input: unknown) => Promise -} -``` - -**Benefits**: - -- Consistent API for all tools -- Easy to add new tools -- Clear separation of concerns -- Type-safe input validation - -## 4. Workflow Example - -**Scenario**: AI needs to read a file and run a command - -``` -User: Help me check package.json and run tests -AI: Needs two tools: read_text_file and exec_command - -Step 1: AI sends request -→ Orchestrator receives: [read_text_file, exec_command] - -Step 2: Safety check -→ Approval manager: read_text_file(low risk) ✓, exec_command(high risk) ⚠️ -→ User approves exec_command - -Step 3: Tool execution -→ Router finds read_text_file tool -→ Executes: reads package.json successfully -→ Router finds exec_command tool -→ Executes: runs "npm test" - -Step 4: Result processing -→ Orchestrator combines both results -→ Limits output size if needed -→ Returns to AI -→ AI analyzes results and responds to user -``` - -## 5. Design Rationale - -### Why Four Layers? - -1. **Separation of Concerns**: - - Implementation layer: What tools do - - Routing layer: Where tools are and how to find them - - Orchestration layer: How tools are executed - - Approval layer: Whether tools should be executed - -2. **Safety by Design**: - - Dangerous operations require explicit approval - - Sandbox restrictions prevent file system damage - - Request fingerprinting enables audit trails - -3. **Extensibility**: - - Easy to add new tools without modifying core - - Support for external MCP tools - - Configurable security policies - -4. **AI-Friendly**: - - Automatic tool documentation generation - - Clear error messages - - Predictable behavior - -### Analogy - -- **Tool implementations** = Kitchen appliances (blender, oven, microwave) -- **Routing layer** = Appliance manuals + power outlets -- **Orchestration layer** = Smart kitchen controller -- **Approval layer** = Safety switches + child locks - -## 6. Configuration and Environment Variables - -### Tool Selection - -- `MEMO_SHELL_TOOL_TYPE`: Choose shell tool variant (`unified_exec`, `shell`, `shell_command`) -- `MEMO_ENABLE_COLLAB_TOOLS`: Disable collaborative agent tools when set to `0` (enabled by default) -- `MEMO_SUBAGENT_COMMAND`: Command executed for each subagent submission -- `MEMO_SUBAGENT_MAX_AGENTS`: Max concurrent running subagents -- `MEMO_ENABLE_MEMORY_TOOL`: Enable memory access tool -- `MEMO_FS_ALLOWED_ROOTS`: Comma-separated additional allowed filesystem roots (default root is runtime `cwd`) - -### Security Settings - -- `MEMO_SANDBOX_WRITABLE_ROOTS`: Comma-separated writable directories -- `MEMO_APPROVAL_MODE`: `auto` or `strict` approval mode - -### Performance Tuning - -- `MEMO_TOOL_RESULT_MAX_CHARS`: Maximum tool output size -- Various timeout and buffer size settings - -## 7. Adding a New Tool - -1. **Create implementation** in `packages/tools/src/tools/` -2. **Use `defineMcpTool()`** for consistent interface -3. **Add to exports** in `packages/tools/src/index.ts` -4. **Write tests** in `*.test.ts` file -5. **Update risk classification** if needed - -Example new tool structure: - -```typescript -// packages/tools/src/tools/my_tool.ts -import { defineMcpTool } from './types' - -export const myTool = defineMcpTool({ - name: 'my_tool', - description: 'Description of my tool', - inputSchema: z.object({ - /* schema */ - }), - execute: async (input) => { - // Implementation - }, -}) -``` - -## 8. Related Documentation - -- [Tools Overview](../user/tools.md) - User-facing tool documentation -- [Approval & Safety](../user/approval-safety.md) - Security features -- [MCP Integration](../user/mcp.md) - External tool support -- [Tool-specific docs](./) - Individual tool documentation - ---- - -_Last updated: 2026-02-08_ diff --git a/docs/tool/close_agent.md b/docs/tool/close_agent.md deleted file mode 100644 index 43417cc..0000000 --- a/docs/tool/close_agent.md +++ /dev/null @@ -1,22 +0,0 @@ -# Memo CLI `close_agent` Tool - -Closes an existing subagent and terminates running work. - -## Basic Info - -- Tool name: `close_agent` -- Description: close subagent by id -- File: `packages/tools/src/tools/collab.ts` -- Confirmation: no - -## Parameters - -- `id` (string, required): agent id. - -## Behavior - -- Looks up agent by id. -- If a submission is running, terminates it before returning. -- Sets status to `closed`. -- Returns JSON with `agent_id` and `status`. -- Returns `isError=true` when id is unknown. diff --git a/docs/tool/edit_file.md b/docs/tool/edit_file.md deleted file mode 100644 index d347063..0000000 --- a/docs/tool/edit_file.md +++ /dev/null @@ -1,29 +0,0 @@ -# Memo CLI `edit_file` Tool - -Applies one or more ordered text edits to a file and returns a unified diff. - -## Basic Info - -- Tool name: `edit_file` -- Description: server-aligned ordered edits with optional dry-run diff preview -- File: `packages/tools/src/tools/edit_file.ts` -- Confirmation: yes - -## Parameters - -- `path` (string, required): target file path. -- `edits` (array, required): each item is `{ oldText, newText }`. -- `dryRun` (boolean, optional, default `false`): preview only, no write. - -## Behavior - -- Executes edits in order; each next edit sees prior results. -- For each edit, replaces only the first exact match. -- If exact match fails, falls back to line-trim whitespace-tolerant matching. -- Preserves indentation using server-compatible relative indentation rules. -- If any edit does not match, stops with `Could not find exact match for edit: ...`. -- Returns Git unified diff in a fenced `diff` code block. - -## Best Practice - -Run once with `dryRun: true` to preview diff, then re-run with `dryRun: false`. diff --git a/docs/tool/exec_command.md b/docs/tool/exec_command.md deleted file mode 100644 index 5d9c799..0000000 --- a/docs/tool/exec_command.md +++ /dev/null @@ -1,44 +0,0 @@ -# Memo CLI `exec_command` Tool - -Starts a managed shell session and returns output chunks. Can continue later with `write_stdin`. - -## Basic Info - -- Tool name: `exec_command` -- Description: run command in managed session and return output or running session id -- File: `packages/tools/src/tools/exec_command.ts` -- Confirmation: no - -## Parameters - -- `cmd` (string, required): command string to run. -- `workdir` (string, optional): working directory (resolved from current cwd). -- `shell` (string, optional): shell binary override. -- `login` (boolean, optional): login shell behavior. -- `tty` (boolean, optional): accepted for compatibility. -- `yield_time_ms` (integer, optional): wait window before returning output. -- `max_output_tokens` (integer, optional): output cap (character-truncated by token estimate). -- `sandbox_permissions` / `justification` / `prefix_rule` (optional): compatibility fields for approval flows. - -## Behavior - -- Spawns a shell command process and records session state. -- Returns formatted response with chunk metadata and output. -- If process is still running after yield window, response includes session id and running status. -- If process exits, response includes exit code. - -## Output Example - -```text -Chunk ID: abc123 -Wall time: 1.2345 seconds -Process running with session ID 7 -Original token count: 42 -Output: -... -``` - -## Notes - -- Pair with `write_stdin` for interactive commands. -- Tool is execution-risk and should be approval-gated. diff --git a/docs/tool/get_memory.md b/docs/tool/get_memory.md deleted file mode 100644 index e636572..0000000 --- a/docs/tool/get_memory.md +++ /dev/null @@ -1,40 +0,0 @@ -# Memo CLI `get_memory` Tool - -Reads memory payload from local `Agents.md` and returns it in structured JSON. - -## Basic Info - -- Tool name: `get_memory` -- Description: Loads stored memory payload for a `memory_id` -- File: `packages/tools/src/tools/get_memory.ts` -- Confirmation: no - -## Parameters - -- `memory_id` (string, required): caller-provided memory key (must be non-empty). - -## Behavior - -- Resolves memory file path: - - `MEMO_HOME/Agents.md` when `MEMO_HOME` is set - - otherwise `~/.memo/Agents.md` -- Reads file as UTF-8 text. -- Returns JSON payload: - - `memory_id`: echoes input id - - `memory_summary`: full file content -- If file is missing/unreadable, returns `isError=true` with: - - `memory not found for memory_id=` - -## Output Example - -```json -{ - "memory_id": "thread-1", - "memory_summary": "## Memo Added Memories\n\n- User prefers concise output\n" -} -``` - -## Notes - -- Current implementation is read-only; it does not modify memory content. -- `memory_id` is currently used as request context/echo and does not select separate memory files. diff --git a/docs/tool/list_directory.md b/docs/tool/list_directory.md deleted file mode 100644 index 334b43c..0000000 --- a/docs/tool/list_directory.md +++ /dev/null @@ -1,29 +0,0 @@ -# Memo CLI `list_directory` Tool - -Lists direct children of a directory. - -## Basic Info - -- Tool name: `list_directory` -- Description: list one directory level with type labels -- File: `packages/tools/src/tools/list_directory.ts` -- Confirmation: no - -## Parameters - -- `path` (string, required): directory path. - -## Behavior - -- Validates directory path against allowed roots. -- Reads direct entries only (non-recursive). -- Output line format: - - `[DIR] name` - - `[FILE] name` - -## Output Example - -```text -[DIR] src -[FILE] package.json -``` diff --git a/docs/tool/list_mcp_resource_templates.md b/docs/tool/list_mcp_resource_templates.md deleted file mode 100644 index 3e1b6c9..0000000 --- a/docs/tool/list_mcp_resource_templates.md +++ /dev/null @@ -1,28 +0,0 @@ -# Memo CLI `list_mcp_resource_templates` Tool - -Lists MCP resource templates. - -## Basic Info - -- Tool name: `list_mcp_resource_templates` -- Description: list MCP resource templates globally or from one server -- File: `packages/tools/src/tools/mcp_resources.ts` -- Confirmation: no - -## Parameters - -- `server` (string, optional): target server name. -- `cursor` (string, optional): pagination cursor (only valid when `server` is set). - -## Behavior - -- Requires active MCP pool. -- With `server`: - - verifies server exists - - calls server `listResourceTemplates(cursor?)` - - returns `{ server, resourceTemplates, nextCursor }` -- Without `server`: - - rejects `cursor` - - aggregates templates from all connected servers - - returns `{ resourceTemplates: [{ server, ...template }] }` -- Returns `isError=true` on missing pool/server or call failure. diff --git a/docs/tool/list_mcp_resources.md b/docs/tool/list_mcp_resources.md deleted file mode 100644 index f2053ba..0000000 --- a/docs/tool/list_mcp_resources.md +++ /dev/null @@ -1,28 +0,0 @@ -# Memo CLI `list_mcp_resources` Tool - -Lists resources exposed by MCP servers. - -## Basic Info - -- Tool name: `list_mcp_resources` -- Description: list MCP resources globally or from one server -- File: `packages/tools/src/tools/mcp_resources.ts` -- Confirmation: no - -## Parameters - -- `server` (string, optional): target server name. -- `cursor` (string, optional): pagination cursor (only valid when `server` is set). - -## Behavior - -- Requires active MCP pool (initialized by runtime). -- With `server`: - - verifies server exists - - calls server `listResources(cursor?)` - - returns `{ server, resources, nextCursor }` -- Without `server`: - - rejects `cursor` - - aggregates resources from all connected servers - - returns `{ resources: [{ server, ...resource }] }` -- Returns `isError=true` on missing pool/server or call failure. diff --git a/docs/tool/read_files.md b/docs/tool/read_files.md deleted file mode 100644 index a5d28a5..0000000 --- a/docs/tool/read_files.md +++ /dev/null @@ -1,31 +0,0 @@ -# Memo CLI `read_files` Tool - -Reads multiple text files in one call. - -## Basic Info - -- Tool name: `read_files` -- Description: batch read text files; per-file failures do not stop the batch -- File: `packages/tools/src/tools/read_files.ts` -- Confirmation: no - -## Parameters - -- `paths` (string[], required): list of file paths. - -## Behavior - -- Iterates in input order. -- Each file is validated with shared filesystem rules. -- If one file fails, returns `: Error - ` for that item and continues. -- Results are separated by `---`. - -## Output Example - -```text -/repo/a.txt: -alpha - ---- -/repo/b.txt: Error - ENOENT: no such file or directory -``` diff --git a/docs/tool/read_mcp_resource.md b/docs/tool/read_mcp_resource.md deleted file mode 100644 index 0acb389..0000000 --- a/docs/tool/read_mcp_resource.md +++ /dev/null @@ -1,23 +0,0 @@ -# Memo CLI `read_mcp_resource` Tool - -Reads one MCP resource by `server` and `uri`. - -## Basic Info - -- Tool name: `read_mcp_resource` -- Description: read a specific MCP resource -- File: `packages/tools/src/tools/mcp_resources.ts` -- Confirmation: no - -## Parameters - -- `server` (string, required): MCP server name. -- `uri` (string, required): resource URI returned by MCP listing. - -## Behavior - -- Requires active MCP pool. -- Verifies server exists. -- Calls server `readResource({ uri })`. -- Returns merged JSON payload: `{ server, uri, ...result }`. -- Returns `isError=true` on missing pool/server or call failure. diff --git a/docs/tool/read_media_file.md b/docs/tool/read_media_file.md deleted file mode 100644 index d429a66..0000000 --- a/docs/tool/read_media_file.md +++ /dev/null @@ -1,27 +0,0 @@ -# Memo CLI `read_media_file` Tool - -Reads an image/audio file and returns base64 payload metadata. - -## Basic Info - -- Tool name: `read_media_file` -- Description: read binary media and return JSON string in text payload -- File: `packages/tools/src/tools/read_media_file.ts` -- Confirmation: no - -## Parameters - -- `path` (string, required): media file path within allowed roots. - -## Behavior - -- Uses shared filesystem validation before reading. -- Infers MIME type from extension, falls back to `application/octet-stream`. -- Returns JSON string with fixed fields: `type`, `mimeType`, `data`. -- `type` is `image`, `audio`, or `blob`. - -## Output Example - -```json -{ "type": "image", "mimeType": "image/png", "data": "iVBORw0KGgo..." } -``` diff --git a/docs/tool/read_text_file.md b/docs/tool/read_text_file.md deleted file mode 100644 index 9718595..0000000 --- a/docs/tool/read_text_file.md +++ /dev/null @@ -1,30 +0,0 @@ -# Memo CLI `read_text_file` Tool - -Reads a text file with optional head/tail line limits. - -## Basic Info - -- Tool name: `read_text_file` -- Description: read full text file content, or first/last N lines -- File: `packages/tools/src/tools/read_text_file.ts` -- Confirmation: no - -## Parameters - -- `path` (string, required): file path within allowed roots. -- `head` (integer, optional): return first N lines. -- `tail` (integer, optional): return last N lines. - -## Behavior - -- Uses shared filesystem validation before reading. -- Rejects calls that provide both `head` and `tail`. -- Returns plain text content via `textResult`. -- Returns `isError=true` on validation/read failures. - -## Output Example - -```text -line1 -line2 -``` diff --git a/docs/tool/resume_agent.md b/docs/tool/resume_agent.md deleted file mode 100644 index d0b01d3..0000000 --- a/docs/tool/resume_agent.md +++ /dev/null @@ -1,22 +0,0 @@ -# Memo CLI `resume_agent` Tool - -Reopens a previously closed subagent. - -## Basic Info - -- Tool name: `resume_agent` -- Description: resume existing agent by id -- File: `packages/tools/src/tools/collab.ts` -- Confirmation: no - -## Parameters - -- `id` (string, required): agent id. - -## Behavior - -- Looks up agent by id. -- If status is `closed`, restores the pre-close status. -- Does not start a new submission by itself; use `send_input` for new work. -- Returns JSON with `agent_id` and current `status`. -- Returns `isError=true` when id is unknown. diff --git a/docs/tool/search_files.md b/docs/tool/search_files.md deleted file mode 100644 index e006535..0000000 --- a/docs/tool/search_files.md +++ /dev/null @@ -1,30 +0,0 @@ -# Memo CLI `search_files` Tool - -Recursively searches under a root path using glob pattern matching. - -## Basic Info - -- Tool name: `search_files` -- Description: recursive glob path match with optional excludes -- File: `packages/tools/src/tools/search_files.ts` -- Confirmation: no - -## Parameters - -- `path` (string, required): search root path. -- `pattern` (string, required): glob pattern matched against relative paths. -- `excludePatterns` (string[], optional): glob patterns to exclude. - -## Behavior - -- Validates root path and each traversed path with shared filesystem security. -- Matches against paths relative to input `path`. -- Returns matching absolute paths, one per line. -- Returns `No matches found` when nothing matches. - -## Output Example - -```text -/repo/src/main.ts -/repo/src/utils/fs.ts -``` diff --git a/docs/tool/send_input.md b/docs/tool/send_input.md deleted file mode 100644 index d2d6535..0000000 --- a/docs/tool/send_input.md +++ /dev/null @@ -1,24 +0,0 @@ -# Memo CLI `send_input` Tool - -Sends a follow-up message to an existing subagent and starts a new submission. - -## Basic Info - -- Tool name: `send_input` -- Description: submit input to existing subagent -- File: `packages/tools/src/tools/collab.ts` -- Confirmation: no - -## Parameters - -- `id` (string, required): agent id. -- `message` (string, required): new message. -- `interrupt` (boolean, optional): when true, interrupts current running submission first. - -## Behavior - -- Looks up agent by id. -- If agent is running and `interrupt` is not true, returns busy error. -- If `interrupt=true`, terminates current submission, then starts the new one. -- Returns JSON with `agent_id`, `status`, and new `submission_id`. -- Returns `isError=true` when id is unknown. diff --git a/docs/tool/shell.md b/docs/tool/shell.md deleted file mode 100644 index 18a9977..0000000 --- a/docs/tool/shell.md +++ /dev/null @@ -1,28 +0,0 @@ -# Memo CLI `shell` Tool - -Executes shell commands using argv form. - -## Basic Info - -- Tool name: `shell` -- Description: run shell command from argv array -- File: `packages/tools/src/tools/shell.ts` -- Confirmation: no - -## Parameters - -- `command` (string array, required): argv-style command. -- `workdir` (string, optional): working directory. -- `timeout_ms` (integer, optional): mapped to output wait window. -- `sandbox_permissions` / `justification` / `prefix_rule` (optional): compatibility fields. - -## Behavior - -- Escapes argv into a command string. -- Runs through managed exec runtime (`login=false`). -- Returns same chunk metadata format as `exec_command`. -- Returns `isError=true` on runtime errors. - -## Notes - -- Enabled when `MEMO_SHELL_TOOL_TYPE=shell`. diff --git a/docs/tool/shell_command.md b/docs/tool/shell_command.md deleted file mode 100644 index d806730..0000000 --- a/docs/tool/shell_command.md +++ /dev/null @@ -1,28 +0,0 @@ -# Memo CLI `shell_command` Tool - -Executes shell commands using string form. - -## Basic Info - -- Tool name: `shell_command` -- Description: run shell command string and return output -- File: `packages/tools/src/tools/shell_command.ts` -- Confirmation: no - -## Parameters - -- `command` (string, required): command string. -- `workdir` (string, optional): working directory. -- `login` (boolean, optional): login shell mode. -- `timeout_ms` (integer, optional): mapped to output wait window. -- `sandbox_permissions` / `justification` / `prefix_rule` (optional): compatibility fields. - -## Behavior - -- Runs command through managed exec runtime. -- Returns same chunk metadata format as `exec_command`. -- Returns `isError=true` on runtime errors. - -## Notes - -- Enabled when `MEMO_SHELL_TOOL_TYPE=shell_command`. diff --git a/docs/tool/spawn_agent.md b/docs/tool/spawn_agent.md deleted file mode 100644 index 13a0642..0000000 --- a/docs/tool/spawn_agent.md +++ /dev/null @@ -1,29 +0,0 @@ -# Memo CLI `spawn_agent` Tool - -Creates a real subagent task process and returns its id. - -## Basic Info - -- Tool name: `spawn_agent` -- Description: spawn subagent task and return `agent_id` / `submission_id` -- File: `packages/tools/src/tools/collab.ts` -- Confirmation: no - -## Parameters - -- `message` (string, required): initial task message. -- `agent_type` (string, optional): reserved compatibility field. - -## Behavior - -- Creates/starts a subagent submission immediately. -- Returns JSON like: - - `agent_id`: stable id for the agent - - `submission_id`: current run id - - `status`: initial status (`running`) -- Fails when concurrent running agents exceed `MEMO_SUBAGENT_MAX_AGENTS`. - -## Notes - -- Tool is enabled by default; set `MEMO_ENABLE_COLLAB_TOOLS=0` to disable collab tools. -- Runtime command is controlled by `MEMO_SUBAGENT_COMMAND` (default uses `memo --dangerous` fallback). diff --git a/docs/tool/update_plan.md b/docs/tool/update_plan.md deleted file mode 100644 index f26a8dc..0000000 --- a/docs/tool/update_plan.md +++ /dev/null @@ -1,41 +0,0 @@ -# Memo CLI `update_plan` Tool - -Stores and updates an in-session plan state. - -## Basic Info - -- Tool name: `update_plan` -- Description: update structured plan with pending/in_progress/completed statuses -- File: `packages/tools/src/tools/update_plan.ts` -- Confirmation: no - -## Parameters - -- `explanation` (string, optional): context text for current plan revision. -- `plan` (array, required): list of plan items: - - `step` (string, required) - - `status` (`pending` | `in_progress` | `completed`, required) - -## Behavior - -- Validates there is at most one `in_progress` item. -- Replaces current in-memory plan with provided list. -- Returns JSON payload containing message, explanation, and plan. -- Returns `isError=true` on validation failure. - -## Output Example - -```json -{ - "message": "Plan updated", - "explanation": "Implement parser then tests", - "plan": [ - { "step": "Implement parser", "status": "in_progress" }, - { "step": "Add tests", "status": "pending" } - ] -} -``` - -## Notes - -- Plan state is process-local and non-persistent. diff --git a/docs/tool/wait.md b/docs/tool/wait.md deleted file mode 100644 index 0d6568b..0000000 --- a/docs/tool/wait.md +++ /dev/null @@ -1,31 +0,0 @@ -# Memo CLI `wait` Tool - -Waits for subagents to reach a final state. - -## Basic Info - -- Tool name: `wait` -- Description: wait for final subagent statuses -- File: `packages/tools/src/tools/collab.ts` -- Confirmation: no - -## Parameters - -- `ids` (string array, required): one or more agent ids. -- `timeout_ms` (integer, optional): wait timeout in ms. - -## Behavior - -- Polls listed ids until at least one becomes final, or timeout. -- Timeout is clamped to `[10000, 300000]` ms; default is `30000` ms. -- Response shape: - - `status`: map of `id -> final_status` (only final entries included) - - `details`: map of `id -> detail` (same ids as `status`) - - `status`: final status - - `last_message`: latest message sent to the agent - - `last_output`: final stdout/stderr summary for the latest submission - - `last_error`: error summary (if any) - - `last_submission_id`: latest submission id - - `updated_at`: record update time - - `timed_out`: whether timeout happened before any final status -- Final statuses: `completed`, `errored`, `closed`, `not_found`. diff --git a/docs/tool/webfetch.md b/docs/tool/webfetch.md deleted file mode 100644 index 40ddd1a..0000000 --- a/docs/tool/webfetch.md +++ /dev/null @@ -1,60 +0,0 @@ -# Memo CLI `webfetch` Tool - -Fetches web content with pagination, optional HTML-to-Markdown extraction, robots policy checks, and private-network protection. - -## Basic Info - -- Tool name: `webfetch` -- Description: Fetch URL content and return paged text; HTML can be simplified into Markdown. -- File: `packages/tools/src/tools/webfetch.ts` -- Confirmation: no - -## Parameters - -- `url` (string, required): URL to fetch. Supported schemes: `http:`, `https:`. -- `max_length` (number, optional): max returned characters for this call. Default `5000`, range `1..999999`. -- `start_index` (number, optional): start offset for paged reads. Default `0`. -- `raw` (boolean, optional): return raw response body instead of simplifying HTML. Default `false`. -- `proxy_url` (string, optional): HTTP(S) proxy URL. - -## Behavior - -- Validates URL/proxy URL and requires HTTP(S) protocols. -- Applies private-network guard by default: - - blocks `localhost`, loopback, link-local, RFC1918, ULA, and related reserved ranges - - checks both IP literals and DNS-resolved addresses -- Applies robots.txt policy by default for autonomous fetches: - - robots URL is `{scheme}://{host}/robots.txt` - - `401/403` blocks fetch - - other `4xx` allows fetch - - robots network failures are treated as errors -- Follows redirects (up to 10 hops), and enforces timeout and response-byte limits. -- For HTML (when `raw=false`), extracts readable article content with Readability and converts to Markdown. -- For non-HTML or `raw=true`, returns raw body with a prefix note. -- Supports pagination with `start_index` + `max_length`: - - appends continuation hint when truncated - - returns `No more content available.` when offset is out of range - -## Output Example - -Success (simplified HTML): - -`Contents of https://example.com/article:` -`# Title` -`...` - -Success (raw / non-HTML): - -`Content type application/json cannot be simplified to markdown, but here is the raw content:` -`Contents of https://example.com/data:` -`{"ok":true}` - -## Notes - -- All failures return `isError=true` with a readable message. -- Default environment settings: - - `MEMO_WEBFETCH_USER_AGENT` - - `MEMO_WEBFETCH_IGNORE_ROBOTS_TXT=0` - - `MEMO_WEBFETCH_TIMEOUT_MS=30000` - - `MEMO_WEBFETCH_MAX_BODY_BYTES=5000000` - - `MEMO_WEBFETCH_BLOCK_PRIVATE_NET=1` diff --git a/docs/tool/write_file.md b/docs/tool/write_file.md deleted file mode 100644 index 2780a9b..0000000 --- a/docs/tool/write_file.md +++ /dev/null @@ -1,28 +0,0 @@ -# Memo CLI `write_file` Tool - -Creates or overwrites a text file. - -## Basic Info - -- Tool name: `write_file` -- Description: atomically write UTF-8 content -- File: `packages/tools/src/tools/write_file.ts` -- Confirmation: yes - -## Parameters - -- `path` (string, required): target file path. -- `content` (string, required): UTF-8 content. - -## Behavior - -- Validates target path against allowed roots first. -- Uses temporary file + rename to preserve atomic replace semantics. -- Returns success text on completion. -- Returns `isError=true` on validation or write failures. - -## Output Example - -```text -Successfully wrote to /repo/notes/todo.txt -``` diff --git a/docs/tool/write_stdin.md b/docs/tool/write_stdin.md deleted file mode 100644 index cd60d3f..0000000 --- a/docs/tool/write_stdin.md +++ /dev/null @@ -1,40 +0,0 @@ -# Memo CLI `write_stdin` Tool - -Continues an existing `exec_command` session by sending stdin bytes and collecting new output. - -## Basic Info - -- Tool name: `write_stdin` -- Description: write stdin to managed exec session and fetch recent output -- File: `packages/tools/src/tools/write_stdin.ts` -- Confirmation: no - -## Parameters - -- `session_id` (integer, required): target exec session id. -- `chars` (string, optional): text to write to stdin. -- `yield_time_ms` (integer, optional): wait window before reading output. -- `max_output_tokens` (integer, optional): output cap. - -## Behavior - -- Looks up session by `session_id`. -- Writes `chars` when session is still running. -- Waits for output/exit window and returns formatted chunk. -- Returns `isError=true` if session does not exist. - -## Output Example - -```text -Chunk ID: def456 -Wall time: 2.0100 seconds -Process exited with code 0 -Original token count: 20 -Output: -interactive response -``` - -## Notes - -- Use empty `chars` to poll output only. -- Session lifecycle is managed in-memory. diff --git a/package.json b/package.json index b9bd6bf..df224cc 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dist/prompt.md", "dist/commands/**", "dist/task-prompts/*.md", + "dist/skills/builtin/**", "README.md", "LICENSE" ], @@ -31,7 +32,6 @@ "test": "vitest run", "test:coverage": "vitest run --coverage", "test:core": "vitest run packages/core", - "test:tools": "vitest run packages/tools", "test:tui": "vitest run packages/tui", "ci": "pnpm run format:check && pnpm run test:coverage && pnpm run build", "prepublishOnly": "pnpm run build && chmod +x dist/index.js", @@ -53,16 +53,13 @@ "vitest": "^2.1.8" }, "dependencies": { - "@dqbd/tiktoken": "^1.0.22", "@inkjs/ui": "^2.0.0", - "@modelcontextprotocol/sdk": "^1.24.3", "@mozilla/readability": "^0.6.0", "fast-glob": "^3.3.3", "ink": "^6.7.0", "ipaddr.js": "^2.3.0", "jsdom": "^28.1.0", "marked": "^17.0.1", - "openai": "^6.10.0", "pastel": "^4.0.1", "react": "^19.2.4", "react-reconciler": "^0.33.0", @@ -71,7 +68,6 @@ "toml": "^3.0.0", "turndown": "^7.2.2", "undici": "^6.23.0", - "zod": "^4.3.6", - "zod-to-json-schema": "^3.25.1" + "zod": "^4.3.6" } } diff --git a/packages/core/README.md b/packages/core/README.md index 1375ee3..c30922a 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -6,12 +6,32 @@ Core provides the central capabilities of **Memo Code**: the ReAct loop, session - `config/` - `config.ts`: Reads `~/.memo/config.toml` (providers, sessions path), handles provider selection, session path building, and config writes. -- `runtime/` - - `prompt.ts/xml`: System prompt loading. - - `history.ts`: JSONL history sink and event construction. - - `defaults.ts`: Default dependency completion (toolset, LLM, prompt, history sink, tokenizer). - - `session.ts`: Session/Turn runtime, executes ReAct loop, writes events, tracks token usage. -- `types.ts`: Shared types (`AgentDeps`, `Session/Turn`, `TokenUsage`, `HistoryEvent`, etc.). +- `llm/` + - `ai_provider.ts`: AI SDK provider factory registry (dispatch by provider name; openai-compatible default). + - `ai_stream.ts`: Default streaming LLM call via AI SDK `streamText`. + - `model_profile.ts`: Model capability resolution (parallel tool calls, reasoning, context window). +- `agent/` — the agent loop kernel (minimal, readable, replaceable) + - `loop.ts`: ReAct loop (observe → think → act → record), session state, token usage, permissions, abort handling. + - `messages.ts`: Message construction and LLM result normalization (AI SDK `ModelMessage`/`GenerateTextResult`). + - `sdk_tools.ts`: Adapter from the memo Tool registry to AI SDK tools — execute wrappers run approval (white-list → classifier → fingerprint), truncation, and deny handling inside `streamText`. + - `step_gate.ts`: Per-step concurrency gate (serializes mutating tools, skips pending tools after denial). + - `session.ts`: `createAgentSession` factory. + - `defaults.ts`: Composition root — default dependency completion (toolset, LLM, prompt, history sink, tokenizer). + - `hooks.ts`: Hook/middleware runners and history snapshotting. + - `compact_prompt.ts`: Context compaction prompt building. +- `tools/` — tool registry, approval, and the 24 built-in tools (merged back from the former tools package) + - `router/`: ToolRouter (native + MCP registries); MCP clients via `@ai-sdk/mcp` (`router/mcp/pool.ts`), disk cache and OAuth credentials kept. + - `approval/`: Approval manager (risk classifier, fingerprints, once/session/deny caches). + - `tools/`: Built-in tool implementations (`defineMcpTool` zod adapter). +- `features/` — user-facing capabilities built on the contracts (not part of the loop); one directory per module, exports via `index.ts` + - `slash/`: Slash command specs and registry. + - `file_suggestions/`: File suggestion helpers for the composer. + - `history/`: Complete session-history module — JSONL sink (write side, injected by the composition root), parser and index (read side for resume/viewing). +- `prompt/` + - `prompt.ts` + `prompt.md`: System prompt loading (runtime context, AGENTS.md/skills injection). +- `skills/` / `mcp/` + - Skill management and MCP server admin. +- `types.ts`: Shared types (`AgentDeps`, `Session/Turn`, `LanguageModelUsage`, `HistoryEvent`, etc.). - `utils/` - Utility functions (assistant output parsing, message wrappers). - `tokenizer.ts`: tiktoken-based tokenizer helpers. diff --git a/packages/core/package.json b/packages/core/package.json index 83cd87a..9df47cf 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,11 +15,18 @@ "version": "0.1.0", "private": true, "scripts": { - "build": "tsup --config tsup.config.ts && node -e \"const { copyFileSync } = require('node:fs'); copyFileSync('src/runtime/prompt.md', 'dist/prompt.md');\"", + "build": "tsup --config tsup.config.ts && node -e \"const { copyFileSync } = require('node:fs'); copyFileSync('src/prompt/prompt.md', 'dist/prompt.md');\"", "test": "vitest run" }, "dependencies": { + "@ai-sdk/mcp": "1.0.66", + "@ai-sdk/openai-compatible": "^2.0.0", + "@ai-sdk/provider-utils": "4.0.41", + "ai": "^6.0.0", + "diff": "^8.0.3", "ignore": "^7.0.5", + "js-tiktoken": "^1.0.21", + "minimatch": "^10.0.1", "zod": "^4.3.6" }, "devDependencies": { diff --git a/packages/core/src/agent/compact_prompt.test.ts b/packages/core/src/agent/compact_prompt.test.ts new file mode 100644 index 0000000..7f9fafa --- /dev/null +++ b/packages/core/src/agent/compact_prompt.test.ts @@ -0,0 +1,130 @@ +import assert from 'node:assert' +import { describe, test } from 'vitest' +import type { ChatMessage } from '@memo/core/types' +import { + buildCompactionUserPrompt, + CONTEXT_SUMMARY_PREFIX, + isContextSummaryMessage, + selectCompactionMessages, +} from '@memo/core/agent/compact_prompt' + +describe('compact_prompt', () => { + test('buildCompactionUserPrompt formats assistant tool calls and tool messages', () => { + const longToolOutput = 'x'.repeat(4_005) + const messages: ChatMessage[] = [ + { + role: 'assistant', + content: [ + { type: 'text', text: 'planning' }, + { type: 'tool-call', toolCallId: 'call-1', toolName: 'exec_command', input: {} }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'exec_command', + output: { type: 'text', value: longToolOutput }, + }, + ], + }, + ] + + const prompt = buildCompactionUserPrompt(messages) + assert.ok(prompt.includes('[0] ASSISTANT (tool_calls: exec_command)')) + assert.ok(prompt.includes('[1] TOOL (exec_command)')) + // Truncation keeps the tail (tool result/error at the end carries the info). + assert.ok(prompt.includes(`...${'x'.repeat(4_000)}`)) + assert.ok(prompt.includes('Return only the summary body in plain text. Do not add markdown fences.')) + }) + + test('buildCompactionUserPrompt renders empty transcript fallback', () => { + const prompt = buildCompactionUserPrompt([]) + assert.ok(prompt.includes('(empty)')) + }) + + test('buildCompactionUserPrompt normalizes tool content and handles unnamed tool message', () => { + const messages: ChatMessage[] = [ + { + role: 'assistant', + content: 'plain assistant text', + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-2', + toolName: '', + output: { type: 'text', value: ' \r\nresult line\r\n ' }, + }, + ], + }, + ] + + const prompt = buildCompactionUserPrompt(messages) + assert.ok(prompt.includes('[0] ASSISTANT\nplain assistant text')) + assert.ok(prompt.includes('[1] TOOL\nresult line')) + assert.strictEqual(prompt.includes('(undefined)'), false) + }) + + test('selectCompactionMessages returns all messages when budget is sufficient', () => { + const messages: ChatMessage[] = [ + { role: 'user', content: 'first' }, + { role: 'assistant', content: 'second' }, + { role: 'user', content: 'third' }, + ] + const selected = selectCompactionMessages(messages, 100, (text) => text.length) + assert.deepStrictEqual(selected, messages) + }) + + test('selectCompactionMessages drops oldest messages under a tight budget', () => { + const messages: ChatMessage[] = [ + { role: 'user', content: 'oldest' }, + { role: 'assistant', content: 'middle' }, + { role: 'user', content: 'newest' }, + ] + const selected = selectCompactionMessages(messages, 20, (text) => text.length) + assert.deepStrictEqual( + selected.map((m) => m.content), + ['newest'], + ) + }) + + test('selectCompactionMessages keeps the newest message even when it exceeds the budget', () => { + const messages: ChatMessage[] = [ + { role: 'user', content: 'old' }, + { role: 'assistant', content: 'x'.repeat(500) }, + ] + const selected = selectCompactionMessages(messages, 10, (text) => text.length) + assert.deepStrictEqual(selected, [messages[1]]) + }) + + test('selectCompactionMessages returns empty for an empty array', () => { + assert.deepStrictEqual( + selectCompactionMessages([], 100, (text) => text.length), + [], + ) + }) + + test('isContextSummaryMessage only matches user summary prefix with newline', () => { + const summaryUserMessage: ChatMessage = { + role: 'user', + content: `${CONTEXT_SUMMARY_PREFIX}\nsummary body`, + } + const missingNewlineUserMessage: ChatMessage = { + role: 'user', + content: CONTEXT_SUMMARY_PREFIX, + } + const assistantMessage: ChatMessage = { + role: 'assistant', + content: `${CONTEXT_SUMMARY_PREFIX}\nsummary body`, + } + + assert.strictEqual(isContextSummaryMessage(summaryUserMessage), true) + assert.strictEqual(isContextSummaryMessage(missingNewlineUserMessage), false) + assert.strictEqual(isContextSummaryMessage(assistantMessage), false) + }) +}) diff --git a/packages/core/src/agent/compact_prompt.ts b/packages/core/src/agent/compact_prompt.ts new file mode 100644 index 0000000..e7555d2 --- /dev/null +++ b/packages/core/src/agent/compact_prompt.ts @@ -0,0 +1,108 @@ +import type { ChatMessage } from '@memo/core/types' + +const MAX_MESSAGE_CONTENT_CHARS = 4_000 + +export const CONTEXT_COMPACTION_SYSTEM_PROMPT = `You are performing a CONTEXT CHECKPOINT COMPACTION. Create a handoff summary for another LLM that will resume the task. + +Include: +- Current progress and key decisions made +- Important context, constraints, or user preferences +- What remains to be done (clear next steps) +- Any critical data, examples, or references needed to continue + +Be concise, structured, and focused on helping the next LLM seamlessly continue the work.` + +export const CONTEXT_SUMMARY_PREFIX = + 'Another language model started to solve this problem and produced a summary of its thinking process. You also have access to the state of the tools that were used by that language model. Use this to build on the work that has already been done and avoid duplicating work. Here is the summary produced by the other language model, use the information in this summary to assist with your own analysis:' + +function normalizeContent(content: string): string { + const compact = content.replace(/\r\n/g, '\n').trim() + if (compact.length <= MAX_MESSAGE_CONTENT_CHARS) { + return compact + } + // Keep the tail: tool output carries its result/error at the end, while the + // head is usually command echoes and noise. + return `...${compact.slice(-MAX_MESSAGE_CONTENT_CHARS)}` +} + +function messageToTranscriptLine(message: ChatMessage, index: number): string { + const role = message.role.toUpperCase() + if (message.role === 'assistant') { + const parts = Array.isArray(message.content) ? message.content : [] + const toolCalls = parts.filter((part) => part.type === 'tool-call') + const text = + typeof message.content === 'string' + ? message.content + : parts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('') + if (toolCalls.length) { + const toolNames = toolCalls.map((part) => part.toolName).join(', ') + return `[${index}] ${role} (tool_calls: ${toolNames})\n${normalizeContent(text)}` + } + return `[${index}] ${role}\n${normalizeContent(text)}` + } + if (message.role === 'tool') { + const part = Array.isArray(message.content) ? message.content[0] : undefined + const toolName = part?.type === 'tool-result' ? part.toolName : '' + const text = part?.type === 'tool-result' && part.output.type === 'text' ? part.output.value : '' + return `[${index}] ${role}${toolName ? ` (${toolName})` : ''}\n${normalizeContent(text)}` + } + const content = typeof message.content === 'string' ? message.content : '' + return `[${index}] ${role}\n${normalizeContent(content)}` +} + +export function isContextSummaryMessage(message: ChatMessage): boolean { + if (message.role !== 'user') return false + return typeof message.content === 'string' && message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`) +} + +/** + * Drop the oldest messages so the serialized transcript fits within + * budgetTokens, keeping the newest message unconditionally. Returns the + * selected messages in their original order (indices are preserved — gaps + * mark the dropped messages). + */ +export function selectCompactionMessages( + messages: ChatMessage[], + budgetTokens: number, + countTokens: (text: string) => number, +): ChatMessage[] { + if (!messages.length) { + return [] + } + + const selected: ChatMessage[] = [] + let used = 0 + for (let i = messages.length - 1; i >= 0; i -= 1) { + const message = messages[i] + if (!message) { + continue + } + const tokens = countTokens(messageToTranscriptLine(message, i)) + 1 // +1 for the '\n\n' separator + if (selected.length > 0 && used + tokens > budgetTokens) { + break + } + selected.push(message) + used += tokens + if (used >= budgetTokens) { + break + } + } + selected.reverse() + return selected +} + +export function buildCompactionUserPrompt(messages: ChatMessage[]): string { + const transcript = messages.length + ? messages.map((message, index) => messageToTranscriptLine(message, index)).join('\n\n') + : '(empty)' + + return [ + 'Conversation history to summarize:', + transcript, + '', + 'Return only the summary body in plain text. Do not add markdown fences.', + ].join('\n') +} diff --git a/packages/core/src/agent/constants.ts b/packages/core/src/agent/constants.ts new file mode 100644 index 0000000..acfde3b --- /dev/null +++ b/packages/core/src/agent/constants.ts @@ -0,0 +1,9 @@ +/** @file Agent loop constants shared across agent modules. */ +import type { SessionMode } from '@memo/core/types' +import type { ToolActionStatus } from '@memo/core/tools/approval' + +export const DEFAULT_SESSION_MODE: SessionMode = 'interactive' +export const DEFAULT_CONTEXT_WINDOW = 120_000 +export const TOOL_ACTION_SUCCESS_STATUS: ToolActionStatus = 'success' +export const TOOL_DISABLED_ERROR_MESSAGE = + 'Tool usage is disabled in the current permission mode. Switch to /core/tools once or /core/tools full to enable tools.' diff --git a/packages/core/src/runtime/defaults.test.ts b/packages/core/src/agent/defaults.test.ts similarity index 59% rename from packages/core/src/runtime/defaults.test.ts rename to packages/core/src/agent/defaults.test.ts index 26bdbb8..6561c5d 100644 --- a/packages/core/src/runtime/defaults.test.ts +++ b/packages/core/src/agent/defaults.test.ts @@ -1,24 +1,5 @@ import { describe, expect, test } from 'vitest' -import { parseToolArguments, filterMcpServersBySelection } from '@memo/core/runtime/defaults' - -describe('parseToolArguments', () => { - test('parses valid JSON string', () => { - const res = parseToolArguments('{"a":1}') - expect(res.ok).toBe(true) - if (res.ok) { - expect(res.data).toEqual({ a: 1 }) - } - }) - - test('returns error when JSON invalid', () => { - const res = parseToolArguments('这不是json') - expect(res.ok).toBe(false) - if (!res.ok) { - expect(res.raw).toBe('这不是json') - expect(res.error.length).toBeGreaterThan(0) - } - }) -}) +import { filterMcpServersBySelection } from '@memo/core/agent/defaults' describe('filterMcpServersBySelection', () => { const servers = { diff --git a/packages/core/src/agent/defaults.ts b/packages/core/src/agent/defaults.ts new file mode 100644 index 0000000..c8ab5c1 --- /dev/null +++ b/packages/core/src/agent/defaults.ts @@ -0,0 +1,150 @@ +/** @file Session default dependency assembly: toolset, LLM, history sinks, tokenizer, etc. */ +import type { ToolSet } from 'ai' +import { NATIVE_TOOLS } from '@memo/core/tools' +import { createTokenCounter } from '@memo/core/utils/tokenizer' +import { buildSessionPath, getSessionsDir, loadMemoConfig, selectProvider } from '@memo/core/config/config' +import { JsonlHistorySink } from '@memo/core/features/history' +import { resolveModelProfile } from '@memo/core/llm/model_profile' +import { streamCallLLM } from '@memo/core/llm/ai_stream' +import { getProviderFactory } from '@memo/core/llm/ai_provider' +import { loadSystemPrompt as defaultLoadPrompt } from '@memo/core/prompt/prompt' +import { McpToolRegistry } from '@memo/core/tools/router' +import { wrapToolSetWithRuntime } from '@memo/core/tools/sdk_tools' +import { buildSkillIndex, filterActiveSkills, loadSkills } from '@memo/core/skills/skills' +import type { SkillIndex } from '@memo/core/skills/skills' +import { installBuiltinSkills } from '@memo/core/skills/builtin_skills' +import type { AgentSessionDeps, AgentSessionOptions, CallLLM, HistorySink, TokenCounter } from '@memo/core/types' +import type { MCPServerConfig } from '@memo/core/config/config' + +export function filterMcpServersBySelection( + servers: Record | undefined, + activeNames: string[] | undefined, +): Record | undefined { + if (!servers) return servers + if (!activeNames) return servers + + const selected = new Set(activeNames.map((name) => name.trim()).filter(Boolean)) + if (selected.size === 0) return {} + + const filtered: Record = {} + for (const [name, config] of Object.entries(servers)) { + if (selected.has(name)) { + filtered[name] = config + } + } + return filtered +} + +/** + * Complete dependencies with default strategy (tools, callLLM, prompt, history sinks, tokenizer). + * Caller can provide only callbacks/overrides, rest use default implementations. + */ +export async function withDefaultDeps( + deps: AgentSessionDeps, + options: AgentSessionOptions, + sessionId: string, +): Promise<{ + tools: ToolSet + callLLM: CallLLM + loadPrompt: () => Promise + historySinks: HistorySink[] + tokenCounter: TokenCounter + dispose: () => Promise + historyFilePath?: string + skillIndex: SkillIndex +}> { + const loaded = await loadMemoConfig() + const config = loaded.config + + // 1. Load external MCP tools (follows MEMO_HOME) + const mcpRegistry = new McpToolRegistry() + await mcpRegistry.loadServersWithOptions( + filterMcpServersBySelection(config.mcp_servers, options.activeMcpServers), + { + memoHome: loaded.home, + storeMode: config.mcp_oauth_credentials_store_mode, + callbackPort: config.mcp_oauth_callback_port, + }, + ) + + // 2. Merge user custom tools (deps.tools has highest priority, keys are tool names) + const combinedTools: ToolSet = { + ...NATIVE_TOOLS, + ...mcpRegistry.toToolSet(), + ...deps.tools, + } + + // 3. Wrap every tool execute with the runtime gate (approval / skip / truncation). + // Context flows in per call via streamText experimental_context. + const runtimeTools = wrapToolSetWithRuntime(combinedTools) ?? combinedTools + + // 4.5 Built-in skills: idempotent install into $MEMO_HOME/skills before the + // scan, so a fresh install is picked up by this very session. Failure is + // non-fatal - memo still works, just without builtin skills. + await installBuiltinSkills({ memoHome: loaded.home }).catch((error: unknown) => { + console.warn(`[memo] failed to install builtin skills: ${(error as Error).message}`) + }) + + // 5. Skills: one scan per session, shared by the system prompt directory + // and the read_skill tool (keeps both consistent). + const skillSnapshot = await loadSkills({ cwd: options.cwd, memoHome: loaded.home }) + const activeSkills = filterActiveSkills(skillSnapshot, config.active_skills) + const skillIndex = buildSkillIndex(activeSkills) + + // 6. Build loadPrompt (tool exposure is handled by the AI SDK tools schema, not prompt text) + const loadPrompt = async () => { + if (deps.loadPrompt) { + return deps.loadPrompt() + } + return defaultLoadPrompt({ + cwd: options.cwd, + memoHome: loaded.home, + activeSkillPaths: config.active_skills, + skills: activeSkills, + }) + } + + const sessionsDir = getSessionsDir(loaded, options) + const historyFilePath = buildSessionPath(sessionsDir, sessionId) + const defaultHistorySink = new JsonlHistorySink(historyFilePath) + + return { + tools: runtimeTools, + dispose: async () => { + if (deps.dispose) await deps.dispose() + await mcpRegistry.dispose() + }, + callLLM: + deps.callLLM ?? + (async (messages, onChunk, callOptions) => { + const selectedProvider = selectProvider(config, options.providerName) + const modelName = options.modelName?.trim() + const provider = modelName ? { ...selectedProvider, model: modelName } : selectedProvider + const apiKey = + process.env[provider.env_api_key] ?? process.env.OPENAI_API_KEY ?? process.env.DEEPSEEK_API_KEY + if (!apiKey) { + throw new Error(`Missing env var ${provider.env_api_key} (or OPENAI_API_KEY/DEEPSEEK_API_KEY)`) + } + const { profile: modelProfile } = resolveModelProfile(provider, config.model_profiles) + return streamCallLLM({ + provider, + apiKey, + messages, + // toolContext absent (compaction) disables tools in streamCallLLM. + tools: runtimeTools, + profile: modelProfile, + factory: getProviderFactory(provider), + toolContext: callOptions?.toolContext, + thinking: callOptions?.thinking, + onChunk, + onReasoningChunk: callOptions?.onReasoningChunk, + signal: callOptions?.signal, + }) + }), + loadPrompt, + historySinks: deps.historySinks ?? [defaultHistorySink], + tokenCounter: deps.tokenCounter ?? createTokenCounter(), + historyFilePath: historyFilePath, + skillIndex, + } +} diff --git a/packages/core/src/agent/defaults.with_default_deps.test.ts b/packages/core/src/agent/defaults.with_default_deps.test.ts new file mode 100644 index 0000000..f11a73e --- /dev/null +++ b/packages/core/src/agent/defaults.with_default_deps.test.ts @@ -0,0 +1,411 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import { tool, type Tool, type ToolSet } from 'ai' +import { z } from 'zod' +import type { AgentSessionDeps, AgentSessionOptions, ChatMessage, LLMResult } from '@memo/core/types' +import type { MCPServerConfig } from '@memo/core/config/config' +import type { AIProviderFactory } from '@memo/core/llm/ai_provider' +import { emptyUsage } from '@memo/core/utils/usage' + +const state = vi.hoisted(() => ({ + loadedConfig: { + home: '/tmp/memo-home', + path: '/tmp/memo-home/config.toml', + config: { + current_provider: 'mock', + providers: [ + { + name: 'mock', + env_api_key: 'MOCK_API_KEY', + model: 'mock-model', + base_url: 'https://mock.local/v1', + }, + ], + model_profiles: {}, + mcp_servers: { + alpha: { command: 'node', args: ['alpha.js'] } as MCPServerConfig, + beta: { command: 'node', args: ['beta.js'] } as MCPServerConfig, + }, + }, + }, + selectedProvider: { + name: 'mock', + env_api_key: 'MOCK_API_KEY', + model: 'mock-model', + base_url: 'https://mock.local/v1', + }, + sessionsDir: '/tmp/memo-sessions', + sessionPath: '/tmp/memo-sessions/session-1.jsonl', + registry: { + mock_tool: { + description: 'mock tool', + inputSchema: { type: 'object' }, + execute: async () => ({ type: 'text', value: 'ok' }), + } as unknown as Tool, + } as ToolSet, + loadMcpServersCalls: [] as unknown[], + historySinkPaths: [] as string[], + routerDisposed: 0, + createTokenCounterCalls: [] as Array, + promptText: 'SYSTEM_PROMPT', + streamCalls: [] as unknown[], + factoryLookups: [] as unknown[], + factory: { + kind: 'openai-compatible', + build: vi.fn(), + buildProviderOptions: vi.fn(() => undefined), + } as unknown as AIProviderFactory, + llmResponse: { + text: 'ok', + toolCalls: [] as LLMResult['toolCalls'], + toolResults: [] as LLMResult['toolResults'], + usage: { + inputTokens: 11, + outputTokens: 7, + totalTokens: 18, + inputTokenDetails: { noCacheTokens: undefined, cacheReadTokens: undefined, cacheWriteTokens: undefined }, + outputTokenDetails: { reasoningTokens: undefined }, + }, + finishReason: 'stop', + } as LLMResult, +})) + +vi.mock('@memo/core/tools', () => ({ + NATIVE_TOOLS: [], +})) + +// Builtin skill installation writes into the mocked home (/tmp/memo-home); +// tests here don't need real files on disk. +vi.mock('@memo/core/skills/builtin_skills', () => ({ + installBuiltinSkills: vi.fn(async () => {}), +})) + +vi.mock('@memo/core/config/config', () => ({ + loadMemoConfig: vi.fn(async () => state.loadedConfig), + selectProvider: vi.fn(() => state.selectedProvider), + getSessionsDir: vi.fn(() => state.sessionsDir), + buildSessionPath: vi.fn(() => state.sessionPath), +})) + +vi.mock('@memo/core/features/history', () => ({ + JsonlHistorySink: class JsonlHistorySink { + constructor(path: string) { + state.historySinkPaths.push(path) + } + }, +})) + +vi.mock('@memo/core/llm/model_profile', () => ({ + resolveModelProfile: vi.fn(() => ({ profile: { supportsParallelToolCalls: true } })), +})) + +vi.mock('@memo/core/llm/ai_stream', () => ({ + streamCallLLM: vi.fn(async (params: unknown) => { + state.streamCalls.push(params) + return state.llmResponse + }), +})) + +vi.mock('@memo/core/llm/ai_provider', () => ({ + getProviderFactory: vi.fn((provider: unknown) => { + state.factoryLookups.push(provider) + return state.factory + }), +})) + +vi.mock('@memo/core/prompt/prompt', () => ({ + loadSystemPrompt: vi.fn(async () => state.promptText), +})) + +vi.mock('@memo/core/utils/tokenizer', () => ({ + createTokenCounter: vi.fn(() => { + state.createTokenCounterCalls.push(undefined) + return { + countText: (text: string) => text.length, + countMessages: (messages: Array<{ content: string }>) => + messages.reduce((sum, message) => sum + message.content.length, 0), + } + }), +})) + +vi.mock('@memo/core/tools/router', () => ({ + McpToolRegistry: class McpToolRegistry { + async loadServersWithOptions(servers: unknown, options: unknown) { + state.loadMcpServersCalls.push([servers, options]) + } + + toToolSet() { + return state.registry + } + + async dispose() { + state.routerDisposed += 1 + } + }, +})) + +describe('withDefaultDeps (default path)', () => { + beforeEach(() => { + state.loadMcpServersCalls = [] + state.historySinkPaths = [] + state.routerDisposed = 0 + state.createTokenCounterCalls = [] + state.promptText = 'SYSTEM_PROMPT' + state.streamCalls = [] + state.factoryLookups = [] + state.llmResponse = { + text: 'ok', + toolCalls: [] as LLMResult['toolCalls'], + toolResults: [] as LLMResult['toolResults'], + usage: { ...emptyUsage(), inputTokens: 11, outputTokens: 7, totalTokens: 18 }, + finishReason: 'stop', + } as LLMResult + delete process.env.MOCK_API_KEY + delete process.env.OPENAI_API_KEY + delete process.env.DEEPSEEK_API_KEY + }) + + afterEach(() => { + delete process.env.MOCK_API_KEY + delete process.env.OPENAI_API_KEY + delete process.env.DEEPSEEK_API_KEY + }) + + test('passes user custom tools through with full definition (schema/isMutating/parallel flags)', async () => { + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const customTool: Tool = tool({ + description: 'custom tool', + inputSchema: z.object({ value: z.string() }), + metadata: { memo: { isMutating: true, supportsParallelToolCalls: false } }, + execute: async () => ({ type: 'text', value: 'done' }), + }) + + const resolved = await withDefaultDeps( + { tools: { my_tool: customTool } } as AgentSessionDeps, + {} as AgentSessionOptions, + 'session-custom', + ) + + const wrapped = resolved.tools.my_tool + expect(wrapped).toBeDefined() + expect(wrapped?.description).toBe(customTool.description) + expect(wrapped?.inputSchema).toBe(customTool.inputSchema) + expect(wrapped?.metadata).toEqual(customTool.metadata) + expect(typeof wrapped?.execute).toBe('function') + }) + + test('builds default deps with default sinks and prompt without tool description injection', async () => { + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-1') + + expect(state.loadMcpServersCalls).toHaveLength(1) + expect(state.historySinkPaths).toEqual([state.sessionPath]) + expect(state.createTokenCounterCalls).toEqual([undefined]) + expect(resolved.historyFilePath).toBe(state.sessionPath) + + const prompt = await resolved.loadPrompt() + expect(prompt).toBe('SYSTEM_PROMPT') + }) + + test('respects provided deps overrides (callLLM/historySinks/tokenCounter/loadPrompt/dispose)', async () => { + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const callLLM = vi.fn(async () => ({ + text: 'override', + toolCalls: [] as LLMResult['toolCalls'], + toolResults: [] as LLMResult['toolResults'], + usage: emptyUsage(), + finishReason: 'stop' as const, + })) + const historySinks = [{ append: vi.fn() }] + const tokenCounter = { + countText: (text: string) => text.length, + countMessages: (messages: Array<{ content: string }>) => + messages.reduce((sum, message) => sum + message.content.length, 0), + } + const dispose = vi.fn(async () => {}) + + const resolved = await withDefaultDeps( + { + callLLM, + historySinks, + tokenCounter, + loadPrompt: async () => 'CUSTOM_PROMPT', + dispose, + } as AgentSessionDeps, + {} as AgentSessionOptions, + 'session-2', + ) + + expect(await resolved.loadPrompt()).toContain('CUSTOM_PROMPT') + expect(resolved.callLLM).toBe(callLLM) + expect(resolved.historySinks).toBe(historySinks) + expect(resolved.tokenCounter).toBe(tokenCounter) + + await resolved.dispose() + expect(dispose).toHaveBeenCalledTimes(1) + expect(state.routerDisposed).toBe(1) + }) + + test('throws when provider api key is missing', async () => { + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-3') + + await expect(resolved.callLLM([{ role: 'user', content: 'hello' } as ChatMessage])).rejects.toThrow( + 'Missing env var MOCK_API_KEY', + ) + }) + + test('falls back to OPENAI_API_KEY and delegates to streamCallLLM', async () => { + process.env.OPENAI_API_KEY = 'openai-fallback-key' + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-3b') + const messages = [{ role: 'user', content: 'hello' } as ChatMessage] + + const response = await resolved.callLLM(messages) + + expect(response).toEqual(state.llmResponse) + expect(state.factoryLookups).toEqual([state.selectedProvider]) + const call = state.streamCalls[0] as { + provider: typeof state.selectedProvider + apiKey: string + messages: unknown[] + toolDefinitions: unknown[] + factory: unknown + } + expect(call.apiKey).toBe('openai-fallback-key') + expect(call.provider).toEqual({ + name: 'mock', + env_api_key: 'MOCK_API_KEY', + model: 'mock-model', + base_url: 'https://mock.local/v1', + }) + expect(call.messages).toEqual(messages) + expect(call.factory).toBe(state.factory) + }) + + test('uses the session model override for resumed sessions', async () => { + process.env.MOCK_API_KEY = 'test-key' + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const resolved = await withDefaultDeps( + {}, + { providerName: 'mock', modelName: 'historic-model' }, + 'session-resume', + ) + + await resolved.callLLM([{ role: 'user', content: 'continue' } as ChatMessage]) + + expect((state.streamCalls[0] as { provider: { model: string } }).provider.model).toBe('historic-model') + }) + + test('passes call options (tools/signal) and forwards structured LLM response', async () => { + process.env.MOCK_API_KEY = 'test-key' + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const signal = new AbortController().signal + + state.llmResponse = { + text: 'assistant text', + reasoning: 'reasoned', + toolCalls: [{ type: 'tool-call', toolCallId: 'call-ok', toolName: 'echo', input: { value: 1 } }], + toolResults: [] as LLMResult['toolResults'], + usage: { ...emptyUsage(), inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + finishReason: 'tool-calls', + } + + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-4') + const response = await resolved.callLLM( + [ + { + role: 'assistant', + content: [ + { type: 'reasoning' as const, text: 'reasoning content' }, + { + type: 'tool-call', + toolCallId: 'prev-call', + toolName: 'read_file', + input: {}, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'prev-call', + toolName: 'read_file', + output: { type: 'text', value: 'observation' }, + }, + ], + }, + { role: 'user', content: 'continue' }, + ], + undefined, + { signal }, + ) + + expect(response).toEqual(state.llmResponse) + + const call = state.streamCalls[0] as { + provider: typeof state.selectedProvider + apiKey: string + messages: Array> + profile: unknown + factory: unknown + signal: AbortSignal + } + expect(call.apiKey).toBe('test-key') + expect(call.signal).toBe(signal) + expect(call.profile).toEqual({ supportsParallelToolCalls: true }) + expect(call.factory).toBe(state.factory) + expect( + (call.messages[0] as { content: Array<{ type: string }> }).content.some( + (part) => part.type === 'reasoning', + ), + ).toBe(true) + expect( + (call.messages[0] as { content: Array<{ type: string }> }).content.some( + (part) => part.type === 'tool-call', + ), + ).toBe(true) + expect( + (call.messages[1] as { content: Array<{ type: string }> }).content.some( + (part) => part.type === 'tool-result', + ), + ).toBe(true) + }) + + test('forwards plain text response with usage', async () => { + process.env.MOCK_API_KEY = 'test-key' + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + + state.llmResponse = { + text: 'plain assistant answer', + reasoning: 'concise reason', + toolCalls: [] as LLMResult['toolCalls'], + toolResults: [] as LLMResult['toolResults'], + usage: { ...emptyUsage(), inputTokens: 3, outputTokens: 4, totalTokens: 7 }, + finishReason: 'stop', + } + + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-5b') + const response = await resolved.callLLM([{ role: 'user', content: 'x' } as ChatMessage]) + expect(response.finishReason).toBe('stop') + expect(response.reasoning).toBe('concise reason') + expect(response.text).toBe('plain assistant answer') + expect(response.usage.inputTokens).toBe(3) + expect(response.usage.outputTokens).toBe(4) + expect(response.usage.totalTokens).toBe(7) + }) + + test('propagates streamCallLLM errors (e.g. empty content)', async () => { + process.env.MOCK_API_KEY = 'test-key' + const { withDefaultDeps } = await import('@memo/core/agent/defaults') + const { streamCallLLM } = await import('@memo/core/llm/ai_stream') + vi.mocked(streamCallLLM).mockRejectedValueOnce(new Error('OpenAI-compatible API returned empty content')) + + const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-6') + await expect(resolved.callLLM([{ role: 'user', content: 'x' } as ChatMessage])).rejects.toThrow( + 'OpenAI-compatible API returned empty content', + ) + }) +}) diff --git a/packages/core/src/runtime/hooks.test.ts b/packages/core/src/agent/hooks.test.ts similarity index 78% rename from packages/core/src/runtime/hooks.test.ts rename to packages/core/src/agent/hooks.test.ts index f26099a..05830d5 100644 --- a/packages/core/src/runtime/hooks.test.ts +++ b/packages/core/src/agent/hooks.test.ts @@ -6,9 +6,9 @@ import type { ObservationHookPayload, FinalHookPayload, ChatMessage, - AssistantToolCall, } from '@memo/core/types' -import { buildHookRunners, runHook, snapshotHistory } from '@memo/core/runtime/hooks' +import { buildHookRunners, runHook, snapshotHistory } from '@memo/core/agent/hooks' +import { emptyUsage } from '@memo/core/utils/usage' describe('buildHookRunners', () => { test('creates empty hook map when no hooks provided', () => { @@ -186,7 +186,7 @@ describe('runHook', () => { sessionId: 's1', turn: 1, step: 1, - action: { tool: 'test', input: {} }, + action: { toolCallId: 'call-1', tool: 'test', input: {} }, history: [], } @@ -219,7 +219,7 @@ describe('runHook', () => { turn: 1, finalText: 'done', status: 'ok', - turnUsage: { prompt: 10, completion: 5, total: 15 }, + turnUsage: { ...emptyUsage(), inputTokens: 10, outputTokens: 5, totalTokens: 15 }, steps: [], } @@ -252,6 +252,14 @@ describe('runHook', () => { step: 1, tool: 'test', observation: 'result', + results: [ + { + toolCallId: 'call-1', + tool: 'test', + observation: 'result', + status: 'success', + }, + ], history: [], } @@ -295,62 +303,56 @@ describe('snapshotHistory', () => { expect(snapshot[0]).toEqual(history[0]) }) - test('deeply copies tool_calls function objects', () => { - const toolCall: AssistantToolCall = { - id: 'call-1', - type: 'function', - function: { - name: 'test_tool', - arguments: '{"arg": "value"}', + test('deeply copies tool-call part inputs', () => { + const history: ChatMessage[] = [ + { + role: 'assistant', + content: [{ type: 'tool-call', toolCallId: 'call-1', toolName: 'test_tool', input: { arg: 'value' } }], }, - } - const history: ChatMessage[] = [{ role: 'assistant', content: '', tool_calls: [toolCall] }] + ] const snapshot = snapshotHistory(history) expect(snapshot).not.toBe(history) - const histMsg0 = history[0] as { tool_calls: AssistantToolCall[] } - const snapMsg0 = snapshot[0] as { tool_calls: AssistantToolCall[] } - expect(snapMsg0.tool_calls).not.toBe(histMsg0.tool_calls) - expect(snapMsg0.tool_calls[0]).not.toBe(histMsg0.tool_calls[0]) - expect(snapMsg0.tool_calls[0]?.function).not.toBe(histMsg0.tool_calls[0]?.function) - expect(snapMsg0.tool_calls[0]?.function).toEqual(histMsg0.tool_calls[0]?.function) + const histMsg0 = history[0] as { content: Array<{ type: string; input: unknown }> } + const snapMsg0 = snapshot[0] as { content: Array<{ type: string; input: unknown }> } + expect(snapMsg0.content).not.toBe(histMsg0.content) + expect(snapMsg0.content[0]).not.toBe(histMsg0.content[0]) + expect(snapMsg0.content[0]?.input).not.toBe(histMsg0.content[0]?.input) + expect(snapMsg0.content[0]?.input).toEqual(histMsg0.content[0]?.input) }) - test('handles multiple tool_calls', () => { + test('handles multiple tool-call parts', () => { const history: ChatMessage[] = [ { role: 'assistant', - content: 'using tools', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { name: 'tool1', arguments: '{}' }, - }, - { - id: 'call-2', - type: 'function', - function: { name: 'tool2', arguments: '{}' }, - }, + content: [ + { type: 'text', text: 'using tools' }, + { type: 'tool-call', toolCallId: 'call-1', toolName: 'tool1', input: {} }, + { type: 'tool-call', toolCallId: 'call-2', toolName: 'tool2', input: {} }, ], }, ] const snapshot = snapshotHistory(history) - const snapMsg0 = snapshot[0] as { tool_calls: AssistantToolCall[] } - expect(snapMsg0.tool_calls).toHaveLength(2) - const histMsg0 = history[0] as { tool_calls: AssistantToolCall[] } - expect(snapMsg0.tool_calls[0]).not.toBe(histMsg0.tool_calls[0]) - expect(snapMsg0.tool_calls[1]).not.toBe(histMsg0.tool_calls[1]) + const snapMsg0 = snapshot[0] as { content: Array<{ type: string }> } + expect(snapMsg0.content.filter((part) => part.type === 'tool-call')).toHaveLength(2) + const histMsg0 = history[0] as { content: Array<{ type: string }> } + expect(snapMsg0.content[1]).not.toBe(histMsg0.content[1]) + expect(snapMsg0.content[2]).not.toBe(histMsg0.content[2]) }) test('creates deep copy of tool messages', () => { const history: ChatMessage[] = [ { role: 'tool', - content: 'tool result', - tool_call_id: 'call-1', - name: 'test_tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'test_tool', + output: { type: 'text', value: 'tool result' }, + }, + ], }, ] const snapshot = snapshotHistory(history) @@ -365,19 +367,21 @@ describe('snapshotHistory', () => { { role: 'user', content: 'hello' }, { role: 'assistant', - content: 'response', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { name: 'tool', arguments: '{}' }, - }, + content: [ + { type: 'text', text: 'response' }, + { type: 'tool-call', toolCallId: 'call-1', toolName: 'tool', input: {} }, ], }, { role: 'tool', - content: 'result', - tool_call_id: 'call-1', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'tool', + output: { type: 'text', value: 'result' }, + }, + ], }, ] const snapshot = snapshotHistory(history) @@ -387,8 +391,8 @@ describe('snapshotHistory', () => { expect(snapshot[1]?.role).toBe('user') expect(snapshot[2]?.role).toBe('assistant') expect(snapshot[3]?.role).toBe('tool') - const histMsg2 = history[2] as { tool_calls: AssistantToolCall[] } - const snapMsg2 = snapshot[2] as { tool_calls: AssistantToolCall[] } - expect(snapMsg2.tool_calls[0]?.function).not.toBe(histMsg2.tool_calls[0]?.function) + const histMsg2 = history[2] as { content: Array<{ type: string; input: unknown }> } + const snapMsg2 = snapshot[2] as { content: Array<{ type: string; input: unknown }> } + expect(snapMsg2.content[1]?.input).not.toBe(histMsg2.content[1]?.input) }) }) diff --git a/packages/core/src/runtime/hooks.ts b/packages/core/src/agent/hooks.ts similarity index 93% rename from packages/core/src/runtime/hooks.ts rename to packages/core/src/agent/hooks.ts index ea9b611..9a664f1 100644 --- a/packages/core/src/runtime/hooks.ts +++ b/packages/core/src/agent/hooks.ts @@ -96,13 +96,12 @@ export async function runHook(map: HookRunnerMap, name: K, p export function snapshotHistory(history: ChatMessage[]): ChatMessage[] { return history.map((msg) => { - if (msg.role === 'assistant' && msg.tool_calls?.length) { + if (msg.role === 'assistant' && Array.isArray(msg.content)) { return { ...msg, - tool_calls: msg.tool_calls.map((toolCall) => ({ - ...toolCall, - function: { ...toolCall.function }, - })), + content: msg.content.map((part) => + part.type === 'tool-call' ? { ...part, input: structuredClone(part.input) } : part, + ), } } return { ...msg } diff --git a/packages/core/src/agent/loop.test.ts b/packages/core/src/agent/loop.test.ts new file mode 100644 index 0000000..d19835e --- /dev/null +++ b/packages/core/src/agent/loop.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test, vi } from 'vitest' +import type { HistorySink } from '@memo/core/types' +import { emitEventToSinks } from '@memo/core/agent/loop' +import { parseTextToolCall, toToolHistoryMessage } from '@memo/core/agent/messages' + +describe('emitEventToSinks', () => { + test('writes structured error payload to stderr when sink append fails', async () => { + const writes: string[] = [] + const writeSpy = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + writes.push(String(chunk)) + return true + }) as typeof process.stderr.write) + + const failingSink: HistorySink = { + append: async () => { + throw new Error('disk full') + }, + } + + try { + await emitEventToSinks( + { + ts: '2026-01-01T00:00:00.000Z', + sessionId: 's-1', + type: 'assistant', + content: 'hello', + }, + [failingSink], + ) + } finally { + writeSpy.mockRestore() + } + + expect(writes.length).toBeGreaterThan(0) + const parsed = JSON.parse(writes.join('').trim()) as Record + expect(parsed.level).toBe('error') + expect(parsed.event).toBe('history_sink_append_failed') + expect(parsed.message).toBe('disk full') + expect(parsed.sink).toBe('Object') + }) +}) + +describe('parseTextToolCall', () => { + const tools = { + read_file: {} as never, + exec_command: {} as never, + } + + test('parses plain json tool call', () => { + const parsed = parseTextToolCall('{"tool":"read_file","input":{"path":"a.txt"}}', tools) + expect(parsed).toEqual({ + tool: 'read_file', + input: { path: 'a.txt' }, + }) + }) + + test('parses fenced json tool call', () => { + const parsed = parseTextToolCall('```json\n{"tool":"exec_command","input":{"cmd":"ls"}}\n```', tools) + expect(parsed).toEqual({ + tool: 'exec_command', + input: { cmd: 'ls' }, + }) + }) + + test('returns null for unknown or invalid tool payload', () => { + expect(parseTextToolCall('{"tool":"unknown","input":{}}', tools)).toBeNull() + expect(parseTextToolCall('{"tool":"read_file"', tools)).toBeNull() + expect(parseTextToolCall('not-json', tools)).toBeNull() + expect(parseTextToolCall(' ', tools)).toBeNull() + }) +}) + +describe('tool result helpers', () => { + test('toToolHistoryMessage maps tool result part into tool chat message', () => { + const message = toToolHistoryMessage({ + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'read_file', + output: { type: 'text', value: 'content' }, + }) + expect(message).toEqual({ + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'read_file', + output: { type: 'text', value: 'content' }, + }, + ], + }) + }) +}) diff --git a/packages/core/src/runtime/session_runtime.ts b/packages/core/src/agent/loop.ts similarity index 65% rename from packages/core/src/runtime/session_runtime.ts rename to packages/core/src/agent/loop.ts index 887b631..91e624d 100644 --- a/packages/core/src/runtime/session_runtime.ts +++ b/packages/core/src/agent/loop.ts @@ -1,13 +1,12 @@ /** @file Session/Turn runtime core: handles ReAct loop, tool scheduling, and event logging. */ import { randomUUID } from 'node:crypto' -import { createHistoryEvent } from '@memo/core/runtime/history' -import { buildThinking } from '@memo/core/utils/utils' import { buildCompactionUserPrompt, CONTEXT_COMPACTION_SYSTEM_PROMPT, CONTEXT_SUMMARY_PREFIX, isContextSummaryMessage, -} from '@memo/core/runtime/compact_prompt' + selectCompactionMessages, +} from '@memo/core/agent/compact_prompt' import type { ChatMessage, AgentSession, @@ -19,46 +18,72 @@ import type { HistoryEvent, HistorySink, ParsedAssistant, + Role, SessionMode, ToolPermissionMode, TokenCounter, - TokenUsage, ToolRegistry, TurnResult, TurnStatus, } from '@memo/core/types' -import { buildHookRunners, runHook, snapshotHistory, type HookRunnerMap } from '@memo/core/runtime/hooks' -import { - createToolOrchestrator, - type ToolApprovalHooks, - type ToolOrchestrator, - type ToolActionResult, -} from '@memo/tools/orchestrator' -import { runWithRuntimeContext } from '@memo/tools/runtime/context' +import type { LanguageModelUsage, ToolCallPart, ToolResultPart } from 'ai' +import { buildHookRunners, runHook, snapshotHistory, type HookRunnerMap } from '@memo/core/agent/hooks' import { DEFAULT_CONTEXT_WINDOW, DEFAULT_SESSION_MODE, TOOL_ACTION_SUCCESS_STATUS, TOOL_DISABLED_ERROR_MESSAGE, - TOOL_SKIPPED_DISABLED_MESSAGE, - accumulateUsage, - buildAssistantToolCalls, - completeToolResultsForProtocol, - emitEventToSinks, - emptyUsage, - fallbackSessionTitleFromPrompt, - isAbortError, +} from '@memo/core/agent/constants' +import { accumulateUsage, emptyUsage } from '@memo/core/utils/usage' +import { isAbortError } from '@memo/core/utils/errors' +import { stableStringify } from '@memo/core/utils/serialize' +import { fallbackSessionTitleFromPrompt } from '@memo/core/utils/title' +import { + createApprovalManager, + type ApprovalManager, + type ApprovalRequest, + type ApprovalDecision, + type ToolActionStatus, +} from '@memo/core/tools/approval' +import type { ToolApprovalHooks } from '@memo/core/tools/sdk_tools' +import { runWithRuntimeContext } from '@memo/core/tools/runtime/context' +import type { ToolExecutionContext } from '@memo/core/tools/sdk_tools' +import { createStepGate } from '@memo/core/tools/runtime/step_gate' +import { + isToolSkippedOutput, + mapOutputStatus, normalizeLLMResponse, + outputToObservation, parseTextToolCall, - resolveToolPermission, - stableStringify, toToolHistoryMessage, -} from '@memo/core/runtime/session_runtime_helpers' -import type { ApprovalRequest, ApprovalDecision } from '@memo/tools/approval' +} from './messages' const DEFAULT_AUTO_COMPACT_THRESHOLD_PERCENT = 80 const COMPACT_USER_MESSAGE_MAX_TOKENS = 20_000 +export type SessionOperationKind = 'turn' | 'compact' + +export class SessionBusyError extends Error { + override name = 'SessionBusyError' + + constructor(activeOperation: SessionOperationKind, requestedOperation: SessionOperationKind) { + super(`Session is busy with ${activeOperation}; cannot start ${requestedOperation}.`) + } +} + +export class SessionClosedError extends Error { + override name = 'SessionClosedError' + + constructor() { + super('Session is closed.') + } +} + +type ActiveSessionOperation = { + kind: SessionOperationKind + done: Promise +} + /** In-process conversation Session, implements multi-turn execution and log writing. */ export class AgentSessionImpl implements AgentSession { public title?: string @@ -70,18 +95,22 @@ export class AgentSessionImpl implements AgentSession { private turnIndex = 0 private tokenCounter: TokenCounter private sinks: HistorySink[] - private sessionUsage: TokenUsage = emptyUsage() + private sessionUsage: LanguageModelUsage = emptyUsage() private startedAt = Date.now() private hooks: HookRunnerMap private closed = false + private activeOperation: ActiveSessionOperation | null = null + private closePromise: Promise | null = null private sessionStartEmitted = false private currentAbortController: AbortController | null = null private cancelling = false private lastActionSignature: string | null = null private repeatedActionCount = 0 - private toolOrchestrator: ToolOrchestrator + private approvalManager: ApprovalManager private toolsDisabled = false private toolPermissionMode: ToolPermissionMode | 'auto' = 'auto' + /** Thinking override; undefined follows the provider model profile. */ + private thinkingOverride: boolean | undefined constructor( private deps: AgentSessionDeps & { @@ -103,13 +132,16 @@ export class AgentSessionImpl implements AgentSession { const resolvedPermission = resolveToolPermission(options) this.toolsDisabled = resolvedPermission.toolsDisabled this.toolPermissionMode = resolvedPermission.mode - this.toolOrchestrator = createToolOrchestrator({ - tools: deps.tools, - approval: { - dangerous: resolvedPermission.dangerous, - mode: resolvedPermission.approvalMode, - }, + this.approvalManager = createApprovalManager({ + dangerous: resolvedPermission.dangerous, + mode: resolvedPermission.approvalMode, }) + this.thinkingOverride = options.thinking + } + + /** 运行时切换思考模式（undefined 恢复为跟随模型 profile）。 */ + setThinking(enabled: boolean): void { + this.thinkingOverride = enabled } /** 初始化：延迟写入 session_start，避免空会话落盘。 */ @@ -117,6 +149,26 @@ export class AgentSessionImpl implements AgentSession { // 留空，等第一次 runTurn 时再写 session_start 事件 } + private async runExclusiveOperation(kind: SessionOperationKind, operation: () => Promise): Promise { + if (this.closed) throw new SessionClosedError() + if (this.activeOperation) throw new SessionBusyError(this.activeOperation.kind, kind) + + let resolveDone!: () => void + const done = new Promise((resolve) => { + resolveDone = resolve + }) + this.activeOperation = { kind, done } + + try { + return await operation() + } finally { + if (this.activeOperation?.done === done) { + this.activeOperation = null + } + resolveDone() + } + } + private resetActionRepetition() { this.lastActionSignature = null this.repeatedActionCount = 0 @@ -174,7 +226,8 @@ export class AgentSessionImpl implements AgentSession { } private calculateUsagePercent(promptTokens: number, contextWindow: number): number { - if (promptTokens <= 0 || contextWindow <= 0) return 0 + // contextWindow 可被外部配置解析为 0（Math.floor(0 < n < 1)），此时避免 Infinity。 + if (contextWindow <= 0) return 0 return Math.round((promptTokens / contextWindow) * 10_000) / 100 } @@ -248,7 +301,7 @@ export class AgentSessionImpl implements AgentSession { (message): message is ChatMessage & { role: 'user' } => message.role === 'user' && !isContextSummaryMessage(message), ) - .map((message) => message.content) + .map((message) => (typeof message.content === 'string' ? message.content : '')) const retainedUserMessages = this.selectCompactionUserMessages(userMessages).map( (content) => ({ role: 'user', content }) as ChatMessage, ) @@ -287,7 +340,8 @@ export class AgentSessionImpl implements AgentSession { } if (remaining > 0) { - selected.push(message.slice(0, remaining)) + // 4 chars ≈ 1 token (ASCII); CJK overshoots to ~3x the byte budget, which is acceptable for the compaction request. + selected.push(message.slice(0, remaining * 4)) } break } @@ -322,54 +376,76 @@ export class AgentSessionImpl implements AgentSession { return skipped } - try { - const response = await this.deps.callLLM( - [ - { role: 'system', content: CONTEXT_COMPACTION_SYSTEM_PROMPT }, - { role: 'user', content: buildCompactionUserPrompt(historyWithoutSystem) }, - ], - undefined, - { tools: [] }, - ) - const normalized = normalizeLLMResponse(response) - const summary = this.normalizeCompactionSummary(normalized.textContent) - if (!summary) { - throw new Error('Compaction model returned an empty summary.') - } - - const compactedHistory = this.buildCompactedHistory(summary) - const afterTokens = this.tokenCounter.countMessages(compactedHistory) - this.history.splice(0, this.history.length, ...compactedHistory) - - const reductionPercent = - beforeTokens > 0 - ? Math.max(0, Math.round(((beforeTokens - afterTokens) / beforeTokens) * 10_000) / 100) - : 0 + // Budget for the compaction request (system instruction + scaffold + + // transcript): capped at the trigger threshold, so the request itself + // always stays well below the window. Halved on retry. + let requestBudget = thresholdTokens + let lastError: Error | null = null - const result: CompactResult = { - reason, - status: 'success', - beforeTokens, - afterTokens, - thresholdTokens, - reductionPercent, - summary, + for (let attempt = 0; attempt < 2; attempt += 1) { + if (attempt > 0) { + requestBudget = Math.max(1, Math.floor(requestBudget / 2)) } - await this.emitContextCompacted(turn, step, result) - return result - } catch (err) { - const result: CompactResult = { - reason, - status: 'failed', - beforeTokens, - afterTokens: beforeTokens, - thresholdTokens, - reductionPercent: 0, - errorMessage: (err as Error).message, + try { + const transcriptMessages = selectCompactionMessages(historyWithoutSystem, requestBudget, (text) => + this.tokenCounter.countText(text), + ) + // The compaction model sees the original system prompt alongside + // the compaction instructions, so the summary keeps the global + // constraints (agent role, AGENTS.md rules) in mind. + const compactionSystemPrompt = systemMessage + ? `${typeof systemMessage.content === 'string' ? systemMessage.content : ''}\n\n${CONTEXT_COMPACTION_SYSTEM_PROMPT}` + : CONTEXT_COMPACTION_SYSTEM_PROMPT + const response = await this.deps.callLLM( + [ + { role: 'system', content: compactionSystemPrompt }, + { role: 'user', content: buildCompactionUserPrompt(transcriptMessages) }, + ], + undefined, + {}, + ) + const normalized = normalizeLLMResponse(response) + const summary = this.normalizeCompactionSummary(normalized.textContent) + if (!summary) { + throw new Error('Compaction model returned an empty summary.') + } + + const compactedHistory = this.buildCompactedHistory(summary) + const afterTokens = this.tokenCounter.countMessages(compactedHistory) + this.history.splice(0, this.history.length, ...compactedHistory) + + const reductionPercent = + beforeTokens > 0 + ? Math.max(0, Math.round(((beforeTokens - afterTokens) / beforeTokens) * 10_000) / 100) + : 0 + + const result: CompactResult = { + reason, + status: 'success', + beforeTokens, + afterTokens, + thresholdTokens, + reductionPercent, + summary, + } + await this.emitContextCompacted(turn, step, result) + return result + } catch (err) { + lastError = err as Error } - await this.emitContextCompacted(turn, step, result) - return result } + + const result: CompactResult = { + reason, + status: 'failed', + beforeTokens, + afterTokens: beforeTokens, + thresholdTokens, + reductionPercent: 0, + errorMessage: lastError?.message, + } + await this.emitContextCompacted(turn, step, result) + return result } private buildToolApprovalHooks(turn: number, step: number): ToolApprovalHooks { @@ -400,20 +476,6 @@ export class AgentSessionImpl implements AgentSession { } } - /** 通过工具编排器执行工具调用。 */ - private async executeToolAction( - actionId: string, - toolName: string, - toolInput: unknown, - turn: number, - step: number, - ): Promise { - return this.toolOrchestrator.executeAction( - { id: actionId, name: toolName, input: toolInput }, - this.buildToolApprovalHooks(turn, step), - ) - } - private async maybeGenerateSessionTitle(turn: number, originalPrompt: string) { if (turn !== 1 || this.title) return @@ -437,6 +499,10 @@ export class AgentSessionImpl implements AgentSession { /** 执行一次 Turn：接受用户输入，走 ReAct 循环，返回最终结果与步骤轨迹。 */ async runTurn(input: string): Promise { + return this.runExclusiveOperation('turn', () => this.runTurnInternal(input)) + } + + private async runTurnInternal(input: string): Promise { return runWithRuntimeContext({ cwd: this.resolveSessionCwd() }, async () => { const abortController = new AbortController() this.currentAbortController = abortController @@ -459,11 +525,13 @@ export class AgentSessionImpl implements AgentSession { meta: { mode: this.mode, cwd: this.resolveSessionCwd(), - tokenizer: this.tokenCounter.model, warnPromptTokens: this.options.warnPromptTokens, contextWindow, autoCompactThresholdPercent, toolPermissionMode: this.toolPermissionMode, + providerName: this.options.providerName, + modelName: this.options.modelName, + thinking: this.thinkingOverride, }, }) this.sessionStartEmitted = true @@ -477,7 +545,7 @@ export class AgentSessionImpl implements AgentSession { await this.emitEvent('turn_start', { turn, content: input, - meta: { tokens: { prompt: promptTokens } }, + meta: { tokens: { prompt: promptTokens }, thinking: this.thinkingOverride }, }) await runHook(this.hooks, 'onTurnStart', { sessionId: this.id, @@ -552,11 +620,18 @@ export class AgentSessionImpl implements AgentSession { } let assistantText = '' - let toolUseBlocks: Array<{ id: string; name: string; input: unknown }> = [] - let usageFromLLM: Partial | undefined - let stopReason: string | undefined + let toolUseBlocks: ToolCallPart[] = [] + let toolResults: ToolResultPart[] = [] + let usageFromLLM: Partial | undefined let reasoningContent: string | undefined let receivedAssistantChunk = false + const toolContext: ToolExecutionContext = { + approvalManager: this.approvalManager, + approvalHooks: this.buildToolApprovalHooks(turn, step), + toolsDisabled: this.toolsDisabled, + gate: createStepGate(), + skillIndex: this.deps.skillIndex, + } try { const llmResult = await this.deps.callLLM( this.history, @@ -566,12 +641,17 @@ export class AgentSessionImpl implements AgentSession { } this.deps.onAssistantStep?.(chunk, step) }, - { signal: abortController.signal }, + { + signal: abortController.signal, + toolContext, + thinking: this.thinkingOverride, + onReasoningChunk: (chunk) => this.deps.onReasoningChunk?.(chunk, step), + }, ) const normalized = normalizeLLMResponse(llmResult) assistantText = normalized.textContent toolUseBlocks = normalized.toolUseBlocks - stopReason = normalized.stopReason + toolResults = normalized.toolResults usageFromLLM = normalized.usage reasoningContent = normalized.reasoningContent if (assistantText.trim().length > 0) { @@ -638,29 +718,36 @@ export class AgentSessionImpl implements AgentSession { // parsed.action 复用单 action 结构，取首个工具作为主 action 语义。 const firstTool = toolUseBlocks[0] if (firstTool) { - const thinking = assistantText ? buildThinking([assistantText]) : undefined + // Reasoning is already separated by the AI SDK; no think-tag extraction needed. + const thinking = reasoningContent parsed = { action: { - tool: firstTool.name, + tool: firstTool.toolName, input: firstTool.input, }, thinking, } assistantHistoryMessage = { role: 'assistant', - content: assistantText, - reasoning_content: reasoningContent, - tool_calls: buildAssistantToolCalls(toolUseBlocks), + content: [ + ...(assistantText ? [{ type: 'text' as const, text: assistantText }] : []), + ...(reasoningContent + ? [{ type: 'reasoning' as const, text: reasoningContent }] + : []), + ...toolUseBlocks, + ], } } else { parsed = {} } } else if (assistantText) { - parsed = { final: assistantText } + parsed = { final: assistantText, thinking: reasoningContent } assistantHistoryMessage = { role: 'assistant', - content: assistantText, - reasoning_content: reasoningContent, + content: [ + ...(assistantText ? [{ type: 'text' as const, text: assistantText }] : []), + ...(reasoningContent ? [{ type: 'reasoning' as const, text: reasoningContent }] : []), + ], } } else { // 没有内容，视为空响应 @@ -668,22 +755,24 @@ export class AgentSessionImpl implements AgentSession { } // 使用 LLM 返回的 usage 作为用量记录。本地 tokenizer 仅用于预估（压缩触发、上下文超限检查），不作为用量上报的 fallback。 - const stepUsage: TokenUsage = usageFromLLM + const stepUsage: LanguageModelUsage = usageFromLLM ? { - prompt: usageFromLLM.prompt ?? 0, - completion: usageFromLLM.completion ?? 0, - total: usageFromLLM.total ?? 0, + ...emptyUsage(), + inputTokens: usageFromLLM.inputTokens ?? 0, + outputTokens: usageFromLLM.outputTokens ?? 0, + totalTokens: usageFromLLM.totalTokens ?? 0, } - : { prompt: 0, completion: 0, total: 0 } + : emptyUsage() accumulateUsage(turnUsage, stepUsage) accumulateUsage(this.sessionUsage, stepUsage) - steps.push({ + const stepTrace: AgentStepTrace = { index: step, assistantText, parsed, tokenUsage: stepUsage, - }) + } + steps.push(stepTrace) await this.emitEvent('assistant', { turn, @@ -696,6 +785,7 @@ export class AgentSessionImpl implements AgentSession { protocol_violation_count: textToolCall ? protocolViolationCount + 1 : protocolViolationCount || undefined, + thinking: reasoningContent, }, }) @@ -738,180 +828,31 @@ export class AgentSessionImpl implements AgentSession { this.history.push(assistantHistoryMessage) } - if (toolUseBlocks.length > 0 && this.toolsDisabled) { - for (const block of toolUseBlocks) { + // 工具调用已由 AI SDK 在 streamText 内执行（execute 包装器：审批/截断/禁用跳过）。 + if (toolUseBlocks.length > 0) { + // 工具禁用模式：全部跳过 → 按工具禁用错误终止 + const disabledSkipped = toolResults.some((tr) => isToolSkippedOutput(tr.output)) + if (disabledSkipped) { + status = 'error' + finalText = TOOL_DISABLED_ERROR_MESSAGE + errorMessage = TOOL_DISABLED_ERROR_MESSAGE + for (const tr of toolResults) { + this.history.push(toToolHistoryMessage(tr)) + } this.history.push({ - role: 'tool', - content: TOOL_SKIPPED_DISABLED_MESSAGE, - tool_call_id: block.id, - name: block.name, - }) - } - status = 'error' - finalText = TOOL_DISABLED_ERROR_MESSAGE - errorMessage = TOOL_DISABLED_ERROR_MESSAGE - this.history.push({ - role: 'assistant', - content: TOOL_DISABLED_ERROR_MESSAGE, - }) - await this.emitEvent('final', { - turn, - step, - content: TOOL_DISABLED_ERROR_MESSAGE, - role: 'assistant', - meta: { - error_type: 'tool_disabled', - tool_count: toolUseBlocks.length, - tools: toolUseBlocks.map((block) => block.name).join(','), - tokens: stepUsage, - }, - }) - await runHook(this.hooks, 'onFinal', { - sessionId: this.id, - turn, - step, - finalText: TOOL_DISABLED_ERROR_MESSAGE, - status, - errorMessage, - tokenUsage: stepUsage, - turnUsage: { ...turnUsage }, - steps, - }) - break - } - - // 处理工具调用（支持并发执行多个工具） - if (toolUseBlocks.length > 1) { - // 重复调用防呆：对每个工具调用记录签名 - for (const block of toolUseBlocks) { - this.maybeWarnRepeatedAction(block.name, block.input) - } - - // 触发 action hooks（action 字段取首个工具，parallelActions 包含全量） - await this.emitEvent('action', { - turn, - step, - meta: { - tools: toolUseBlocks.map((b) => b.name), - action_ids: toolUseBlocks.map((b) => b.id), - action_id: toolUseBlocks[0]?.id, - parallel: true, - phase: 'dispatch', - thinking: parsed.thinking, - // 保存所有工具的完整信息 - toolBlocks: toolUseBlocks.map((b) => ({ - id: b.id, - name: b.name, - input: b.input, - })), - }, - }) - const firstTool = toolUseBlocks[0] - if (firstTool) { - await runHook(this.hooks, 'onAction', { - sessionId: this.id, - turn, - step, - action: { - tool: firstTool.name, - input: firstTool.input, - }, - parallelActions: toolUseBlocks.map((block) => ({ - tool: block.name, - input: block.input, - })), - thinking: parsed.thinking, - history: snapshotHistory(this.history), - }) - } - - const allSupportParallel = toolUseBlocks.every((block) => { - const tool = this.deps.tools[block.name] - return Boolean(tool?.supportsParallelToolCalls) - }) - const hasMutatingTool = toolUseBlocks.some((block) => { - const tool = this.deps.tools[block.name] - return Boolean(tool?.isMutating) - }) - const executionMode = allSupportParallel && !hasMutatingTool ? 'parallel' : 'sequential' - - const execution = await this.toolOrchestrator.executeActions( - toolUseBlocks.map((block) => ({ - id: block.id, - name: block.name, - input: block.input, - })), - { - ...this.buildToolApprovalHooks(turn, step), - executionMode, - failurePolicy: 'fail_fast', - }, - ) - - const protocolResults = completeToolResultsForProtocol( - toolUseBlocks, - execution.results, - execution.hasRejection, - ) - - for (const [idx, result] of protocolResults.entries()) { - this.history.push(toToolHistoryMessage(result)) - await this.emitEvent('observation', { - turn, - step, - content: result.observation, - meta: { - tool: result.tool, - index: idx, - action_id: result.actionId, - phase: 'result', - status: result.status, - error_type: result.errorType, - duration_ms: result.durationMs, - execution_mode: executionMode, - }, + role: 'assistant', + content: TOOL_DISABLED_ERROR_MESSAGE, }) - } - - const combinedObservation = protocolResults - .map((result) => `[${result.tool}]: ${result.observation}`) - .join('\n\n') - const parallelResultStatuses = protocolResults.map((result) => result.status) - const resultStatus = - parallelResultStatuses.find((candidate) => candidate !== TOOL_ACTION_SUCCESS_STATUS) ?? - TOOL_ACTION_SUCCESS_STATUS - const lastStep = steps[steps.length - 1] - if (lastStep) { - lastStep.observation = combinedObservation - } - // 触发 observation hook（使用合并后的结果） - await runHook(this.hooks, 'onObservation', { - sessionId: this.id, - turn, - step, - tool: toolUseBlocks.map((b) => b.name).join(', '), - observation: combinedObservation, - resultStatus, - parallelResultStatuses, - history: snapshotHistory(this.history), - }) - - // 如果被拒绝，停止本轮次 - if (execution.hasRejection) { - const rejectionResult = protocolResults.find((result) => result.rejected) - status = 'cancelled' - finalText = '用户拒绝了工具执行，已停止当前操作。' await this.emitEvent('final', { turn, step, - content: finalText, + content: TOOL_DISABLED_ERROR_MESSAGE, role: 'assistant', meta: { - rejected: true, - phase: 'result', - action_id: rejectionResult?.actionId, - error_type: rejectionResult?.errorType ?? 'approval_denied', - duration_ms: rejectionResult?.durationMs, + error_type: 'tool_disabled', + tool_count: toolUseBlocks.length, + tools: toolUseBlocks.map((block) => block.toolName).join(','), + tokens: stepUsage, }, }) await runHook(this.hooks, 'onFinal', { @@ -920,27 +861,35 @@ export class AgentSessionImpl implements AgentSession { step, finalText, status, + errorMessage, tokenUsage: stepUsage, turnUsage: { ...turnUsage }, steps, }) break } - continue - } - // 单个工具调用 - // 注意：当 toolUseBlocks.length > 1 时，已在上面处理，这里跳过 - else if (parsed.action) { - this.maybeWarnRepeatedAction(parsed.action.tool, parsed.action.input) - const actionId = toolUseBlocks[0]?.id ?? `${turn}:${step}:single:${parsed.action.tool}` + // 重复调用防呆 + for (const block of toolUseBlocks) { + this.maybeWarnRepeatedAction(block.toolName, block.input) + } + + // action 事件（批次级） await this.emitEvent('action', { turn, step, meta: { - tool: parsed.action.tool, - input: parsed.action.input, - action_id: actionId, + tools: toolUseBlocks.map((b) => b.toolName), + action_ids: toolUseBlocks.map((b) => b.toolCallId), + action_id: toolUseBlocks[0]?.toolCallId, + tool: toolUseBlocks[0]?.toolName, + input: toolUseBlocks[0]?.input, + toolBlocks: toolUseBlocks.map((block) => ({ + id: block.toolCallId, + name: block.toolName, + input: block.input, + })), + parallel: toolUseBlocks.length > 1, phase: 'dispatch', thinking: parsed.thinking, }, @@ -949,29 +898,75 @@ export class AgentSessionImpl implements AgentSession { sessionId: this.id, turn, step, - action: parsed.action, + action: { + toolCallId: toolUseBlocks[0]?.toolCallId ?? '', + tool: toolUseBlocks[0]?.toolName ?? '', + input: toolUseBlocks[0]?.input, + }, + parallelActions: toolUseBlocks.map((block) => ({ + toolCallId: block.toolCallId, + tool: block.toolName, + input: block.input, + })), thinking: parsed.thinking, history: snapshotHistory(this.history), }) - // 使用审批流程执行工具 - const result = await this.executeToolAction( - actionId, - parsed.action.tool, - parsed.action.input, + // 逐结果回填历史 + observation 事件 + const observations: string[] = [] + const resultStatuses: ToolActionStatus[] = [] + const observationResults = [] + let denied = false + for (const [idx, tr] of toolResults.entries()) { + const observation = outputToObservation(tr) + const status = mapOutputStatus(tr) + observations.push(observation) + resultStatuses.push(status) + observationResults.push({ + toolCallId: tr.toolCallId, + tool: tr.toolName, + observation, + status, + }) + this.history.push(toToolHistoryMessage(tr)) + await this.emitEvent('observation', { + turn, + step, + content: observation, + meta: { + tool: tr.toolName, + index: idx, + action_id: tr.toolCallId, + phase: 'result', + status, + error_type: status === 'success' ? undefined : status, + }, + }) + if (tr.output.type === 'execution-denied') denied = true + } + const combinedObservation = observations + .map((obs, i) => `[${toolResults[i]?.toolName ?? ''}]: ${obs}`) + .join('\n\n') + const hookObservation = toolResults.length > 1 ? combinedObservation : (observations[0] ?? '') + stepTrace.observation = hookObservation + const resultStatus = + resultStatuses.find((candidate) => candidate !== TOOL_ACTION_SUCCESS_STATUS) ?? + TOOL_ACTION_SUCCESS_STATUS + await runHook(this.hooks, 'onObservation', { + sessionId: this.id, turn, step, - ) + tool: toolUseBlocks.map((b) => b.toolName).join(', '), + observation: hookObservation, + resultStatus, + parallelResultStatuses: resultStatuses, + results: observationResults, + history: snapshotHistory(this.history), + }) - // 如果被拒绝，停止本轮次 - if (result.rejected) { - this.history.push( - toToolHistoryMessage({ - ...result, - observation: - result.observation || `User denied tool execution: ${parsed.action.tool}`, - }), - ) + // 拒绝 → 终止本轮（保持现状语义） + if (denied) { + const deniedResult = toolResults.find((tr) => tr.output.type === 'execution-denied') status = 'cancelled' finalText = '用户拒绝了工具执行，已停止当前操作。' await this.emitEvent('final', { @@ -982,9 +977,8 @@ export class AgentSessionImpl implements AgentSession { meta: { rejected: true, phase: 'result', - action_id: result.actionId, - error_type: result.errorType ?? 'approval_denied', - duration_ms: result.durationMs, + action_id: deniedResult?.toolCallId, + error_type: 'approval_denied', }, }) await runHook(this.hooks, 'onFinal', { @@ -999,49 +993,14 @@ export class AgentSessionImpl implements AgentSession { }) break } - - const observation = result.observation - - this.history.push({ - role: 'tool', - content: observation, - tool_call_id: result.actionId, - name: parsed.action.tool, - }) - const lastStep = steps[steps.length - 1] - if (lastStep) { - lastStep.observation = observation - } - await this.emitEvent('observation', { - turn, - step, - content: observation, - meta: { - tool: parsed.action.tool, - action_id: result.actionId, - phase: 'result', - status: result.status, - error_type: result.errorType, - duration_ms: result.durationMs, - }, - }) - await runHook(this.hooks, 'onObservation', { - sessionId: this.id, - turn, - step, - tool: parsed.action.tool, - observation, - resultStatus: result.status, - history: snapshotHistory(this.history), - }) continue } - // 检查是否是最终回复（end_turn 或有 final 字段） - if (stopReason === 'end_turn' || parsed.final) { + // 无工具调用：文本即最终回复 + if (toolUseBlocks.length === 0) { this.resetActionRepetition() const shouldFallbackFromPreviousText = - stopReason === 'end_turn' && + toolUseBlocks.length === 0 && !parsed.final && assistantText.trim().length === 0 && Boolean(lastNonEmptyAssistantText) && @@ -1061,6 +1020,7 @@ export class AgentSessionImpl implements AgentSession { meta: { tokens: stepUsage, fallback_from_previous_text: shouldFallbackFromPreviousText || undefined, + thinking: reasoningContent, }, }) await runHook(this.hooks, 'onFinal', { @@ -1072,13 +1032,10 @@ export class AgentSessionImpl implements AgentSession { tokenUsage: stepUsage, turnUsage: { ...turnUsage }, steps, + thinking: reasoningContent, }) break } - - // 无动作且未结束时，重置重复计数（保持“连续”语义） - this.resetActionRepetition() - break } if (!finalText && status !== 'cancelled') { @@ -1126,7 +1083,7 @@ export class AgentSessionImpl implements AgentSession { this.currentAbortController = null this.cancelling = false // 清除单次授权（每次 turn 结束后） - this.toolOrchestrator.clearOnceApprovals() + this.approvalManager.clearOnceApprovals() } }) } @@ -1139,39 +1096,45 @@ export class AgentSessionImpl implements AgentSession { } async compactHistory(reason: CompactReason = 'manual'): Promise { - return this.compactHistoryInternal(reason, this.turnIndex, 0) + return this.runExclusiveOperation('compact', () => this.compactHistoryInternal(reason, this.turnIndex, 0)) } listToolNames() { return Object.keys(this.deps.tools) } - async close() { - if (this.closed) return + close(): Promise { + if (this.closePromise) return this.closePromise this.closed = true - const hasContent = this.sessionStartEmitted || this.turnIndex >= 0 - if (hasContent) { + this.cancelCurrentTurn() + this.closePromise = this.closeInternal() + return this.closePromise + } + + private async closeInternal() { + await this.activeOperation?.done + // 空会话（从未 runTurn）不写 session_end，避免空会话落盘；sink 清理始终执行。 + if (this.sessionStartEmitted) { await this.emitEvent('session_end', { meta: { durationMs: Date.now() - this.startedAt, tokens: this.sessionUsage, }, }) - for (const sink of this.sinks) { - try { - if (sink.close) { - await sink.close() - } else if (sink.flush) { - await sink.flush() - } - } catch (err) { - console.error(`History flush failed: ${(err as Error).message}`) + } + for (const sink of this.sinks) { + try { + if (sink.close) { + await sink.close() + } else if (sink.flush) { + await sink.flush() } + } catch (err) { + console.error(`History flush failed: ${(err as Error).message}`) } } - this.tokenCounter.dispose() // 清理所有授权 - this.toolOrchestrator.dispose() + this.approvalManager.dispose() if (this.deps.dispose) { await this.deps.dispose() } @@ -1192,3 +1155,90 @@ export class AgentSessionImpl implements AgentSession { await emitEventToSinks(event, this.sinks) } } + +/** Helper to generate structured history events. */ +export function createHistoryEvent(params: { + sessionId: string + type: HistoryEvent['type'] + turn?: number + step?: number + content?: string + role?: Role + meta?: Record +}): HistoryEvent { + return { + ts: new Date().toISOString(), + sessionId: params.sessionId, + turn: params.turn, + step: params.step, + type: params.type, + content: params.content, + role: params.role, + meta: params.meta, + } +} + +// --- Agent loop constants and helpers --------------------------------------------- + +export type ResolvedToolPermission = { + mode: ToolPermissionMode | 'auto' + toolsDisabled: boolean + dangerous: boolean + approvalMode: 'auto' | 'strict' +} + +function writeStructuredError(payload: Record) { + process.stderr.write(`${JSON.stringify(payload)}\n`) +} + +export function resolveToolPermission(options: AgentSessionOptions): ResolvedToolPermission { + if (options.toolPermissionMode === 'none') { + return { + mode: 'none', + toolsDisabled: true, + dangerous: false, + approvalMode: 'auto', + } + } + + if (options.toolPermissionMode === 'once') { + return { + mode: 'once', + toolsDisabled: false, + dangerous: false, + approvalMode: 'auto', + } + } + + if (options.toolPermissionMode === 'full') { + return { + mode: 'full', + toolsDisabled: false, + dangerous: true, + approvalMode: 'auto', + } + } + + const dangerous = options.dangerous ?? false + return { + mode: dangerous ? 'full' : 'auto', + toolsDisabled: false, + dangerous, + approvalMode: 'auto', + } +} + +export async function emitEventToSinks(event: HistoryEvent, sinks: HistorySink[]) { + for (const sink of sinks) { + try { + await sink.append(event) + } catch (err) { + writeStructuredError({ + level: 'error', + event: 'history_sink_append_failed', + sink: sink.constructor?.name || 'anonymous_sink', + message: (err as Error).message, + }) + } + } +} diff --git a/packages/core/src/agent/messages.ts b/packages/core/src/agent/messages.ts new file mode 100644 index 0000000..992448a --- /dev/null +++ b/packages/core/src/agent/messages.ts @@ -0,0 +1,93 @@ +/** @file Message construction and LLM result normalization for the agent loop. */ +import type { LanguageModelUsage, ToolCallPart, ToolResultPart } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ChatMessage, LLMResult, ToolRegistry } from '@memo/core/types' +import type { ToolActionStatus } from '@memo/core/tools/approval' +import { TOOL_SKIPPED_DISABLED_MESSAGE } from '@memo/core/tools/sdk_tools' + +export function parseToolArguments( + raw: string, +): { ok: true; data: unknown } | { ok: false; raw: string; error: string } { + try { + return { ok: true, data: JSON.parse(raw) } + } catch (err) { + return { ok: false, raw, error: (err as Error).message } + } +} + +/** Extract session-level fields from an AI SDK GenerateTextResult. */ +export function normalizeLLMResponse(raw: LLMResult): { + textContent: string + /** Tool calls (AI SDK ToolCallPart[]; inputs are already parsed objects). */ + toolUseBlocks: ToolCallPart[] + reasoningContent?: string + usage?: Partial + /** Executed tool results (AI SDK executed the tools inside streamText). */ + toolResults: ToolResultPart[] +} { + return { + textContent: raw.text, + toolUseBlocks: raw.toolCalls, + reasoningContent: + typeof raw.reasoning === 'string' && raw.reasoning.trim().length > 0 ? raw.reasoning : undefined, + usage: raw.usage, + toolResults: raw.toolResults, + } +} + +/** Parse a plain-text tool call (legacy text protocol fallback). */ +export function parseTextToolCall(text: string, tools: ToolRegistry): { tool: string; input: unknown } | null { + const trimmed = text.trim() + if (!trimmed) return null + + const candidates = [trimmed] + const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i) + if (fenced?.[1]) { + candidates.push(fenced[1].trim()) + } + + for (const candidate of candidates) { + if (!candidate.startsWith('{') || !candidate.endsWith('}')) continue + try { + const parsed = JSON.parse(candidate) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue + const obj = parsed as Record + const tool = typeof obj.tool === 'string' ? obj.tool.trim() : '' + if (!tool || !Object.prototype.hasOwnProperty.call(tools, tool)) continue + return { tool, input: obj.input ?? {} } + } catch { + // Ignore invalid json + } + } + + return null +} + +/** AI SDK ToolResultPart → tool history message (CoreMessage shape, passthrough). */ +export function toToolHistoryMessage(result: ToolResultPart): ChatMessage { + return { + role: 'tool', + content: [result], + } +} + +/** Detect the tools-disabled skip output (non-standard 'skipped' replaced by an error-text sentinel). */ +export function isToolSkippedOutput(output: ToolResultOutput): boolean { + return output.type === 'error-text' && output.value === TOOL_SKIPPED_DISABLED_MESSAGE +} + +/** ToolResultPart → observation display text. */ +export function outputToObservation(result: ToolResultPart): string { + const output = result.output + if (output.type === 'text' || output.type === 'error-text') return output.value + if (output.type === 'json') return JSON.stringify(output.value) + if (output.type === 'execution-denied') return output.reason ?? 'User denied tool execution' + return '(no tool output)' +} + +/** ToolResultPart → memo status ('success' | error type). */ +export function mapOutputStatus(result: ToolResultPart): ToolActionStatus { + if (result.output.type === 'execution-denied') return 'approval_denied' + if (result.output.type === 'error-text') return 'execution_failed' + return 'success' +} diff --git a/packages/core/src/runtime/session.ts b/packages/core/src/agent/session.ts similarity index 75% rename from packages/core/src/runtime/session.ts rename to packages/core/src/agent/session.ts index 48d4579..3f7974d 100644 --- a/packages/core/src/runtime/session.ts +++ b/packages/core/src/agent/session.ts @@ -1,9 +1,11 @@ import { randomUUID } from 'node:crypto' -import { withDefaultDeps } from '@memo/core/runtime/defaults' -import { DEFAULT_SESSION_MODE } from '@memo/core/runtime/session_runtime_helpers' -import { AgentSessionImpl } from '@memo/core/runtime/session_runtime' +import { withDefaultDeps } from '@memo/core/agent/defaults' +import { DEFAULT_SESSION_MODE } from '@memo/core/agent/constants' +import { AgentSessionImpl } from '@memo/core/agent/loop' import type { AgentSession, AgentSessionDeps, AgentSessionOptions } from '@memo/core/types' +export { SessionBusyError, SessionClosedError, type SessionOperationKind } from '@memo/core/agent/loop' + /** * 创建一个 Agent Session，支持多轮对话与 JSONL 事件记录。 */ diff --git a/packages/core/src/agent/session_concurrency.test.ts b/packages/core/src/agent/session_concurrency.test.ts new file mode 100644 index 0000000..b4e4d0f --- /dev/null +++ b/packages/core/src/agent/session_concurrency.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from 'vitest' +import { + createAgentSession, + createTokenCounter, + SessionBusyError, + SessionClosedError, + type LLMResult, +} from '@memo/core' +import { emptyUsage } from '@memo/core/utils/usage' + +function response(text: string): LLMResult { + return { + text, + toolCalls: [], + toolResults: [], + usage: emptyUsage(), + finishReason: 'stop', + } +} + +function deferredResponse() { + let resolve!: (value: LLMResult) => void + const promise = new Promise((next) => { + resolve = next + }) + return { promise, resolve } +} + +describe('AgentSession operation boundaries', () => { + test('rejects a second turn and manual compaction while a turn is running', async () => { + const pending = deferredResponse() + const session = await createAgentSession({ + callLLM: async () => pending.promise, + historySinks: [], + loadPrompt: async () => 'system', + tokenCounter: createTokenCounter(), + tools: {}, + }) + + const running = session.runTurn('first') + await expect(session.runTurn('second')).rejects.toBeInstanceOf(SessionBusyError) + await expect(session.compactHistory('manual')).rejects.toBeInstanceOf(SessionBusyError) + + pending.resolve(response('done')) + await expect(running).resolves.toMatchObject({ finalText: 'done' }) + await session.close() + }) + + test('rejects a turn while manual compaction is running', async () => { + const pending = deferredResponse() + let callCount = 0 + const session = await createAgentSession({ + callLLM: async () => { + callCount += 1 + return callCount === 1 ? response('seeded') : pending.promise + }, + historySinks: [], + loadPrompt: async () => 'system', + tokenCounter: createTokenCounter(), + tools: {}, + }) + + await session.runTurn('seed') + const compacting = session.compactHistory('manual') + await expect(session.runTurn('overlap')).rejects.toBeInstanceOf(SessionBusyError) + + pending.resolve(response('summary')) + await expect(compacting).resolves.toMatchObject({ status: 'success' }) + await session.close() + }) + + test('close waits for the active operation and rejects new work', async () => { + const pending = deferredResponse() + const session = await createAgentSession({ + callLLM: async () => pending.promise, + historySinks: [], + loadPrompt: async () => 'system', + tokenCounter: createTokenCounter(), + tools: {}, + }) + + const running = session.runTurn('first') + const closing = session.close() + + await expect(session.runTurn('second')).rejects.toBeInstanceOf(SessionClosedError) + await expect(session.compactHistory('manual')).rejects.toBeInstanceOf(SessionClosedError) + + pending.resolve(response('done')) + await running + await closing + }) +}) diff --git a/packages/core/src/runtime/session_hooks.test.ts b/packages/core/src/agent/session_hooks.test.ts similarity index 66% rename from packages/core/src/runtime/session_hooks.test.ts rename to packages/core/src/agent/session_hooks.test.ts index 19f6e86..00a9060 100644 --- a/packages/core/src/runtime/session_hooks.test.ts +++ b/packages/core/src/agent/session_hooks.test.ts @@ -2,79 +2,138 @@ import assert from 'node:assert' import { describe, test } from 'vitest' import { createAgentSession, createTokenCounter } from '@memo/core' -import type { ChatMessage, HistoryEvent, LLMResponse, TokenCounter } from '@memo/core' -import type { Tool } from '@memo/tools/router' -import { CONTEXT_COMPACTION_SYSTEM_PROMPT, CONTEXT_SUMMARY_PREFIX } from '@memo/core/runtime/compact_prompt' +import type { ChatMessage, HistoryEvent, LLMResult, TokenCounter } from '@memo/core' +import { jsonSchema, tool, type Tool, type ToolResultPart } from 'ai' +import { CONTEXT_COMPACTION_SYSTEM_PROMPT, CONTEXT_SUMMARY_PREFIX } from '@memo/core/agent/compact_prompt' +import { emptyUsage } from '@memo/core/utils/usage' -const echoTool: Tool = { - name: 'echo', +const echoTool: Tool = tool({ description: 'echo input', - source: 'native', - inputSchema: { type: 'object', properties: { text: { type: 'string' } } }, + inputSchema: jsonSchema({ type: 'object', properties: { text: { type: 'string' } } }), execute: async (input: unknown) => { const { text } = input as { text: string } - return { - content: [{ type: 'text' as const, text: `echo:${text}` }], - } + return { type: 'text', value: `echo:${text}` } }, -} +}) -const readNoteTool: Tool = { - name: 'read_note', +const readNoteTool: Tool = tool({ description: 'read note', - source: 'native', - inputSchema: { type: 'object', properties: { topic: { type: 'string' } } }, + inputSchema: jsonSchema({ type: 'object', properties: { topic: { type: 'string' } } }), execute: async (input: unknown) => { const { topic } = input as { topic: string } + return { type: 'text', value: `note:${topic}` } + }, +}) + +type MockToolOpts = { denied?: boolean; skipped?: boolean; skippedDisabled?: boolean; invalid?: boolean } + +const TOOL_SKIPPED_DISABLED_TEXT = 'Tool execution skipped: tools are disabled in current permission mode.' +const TOOL_SKIPPED_AFTER_REJECTION_TEXT = 'Skipped tool execution after previous rejection.' + +/** Simulate the AI SDK execute wrapper output for a tool. */ +function mockToolResult(id: string, name: string, input: unknown, opts: MockToolOpts = {}): ToolResultPart { + if (opts.denied) { return { - content: [{ type: 'text' as const, text: `note:${topic}` }], + type: 'tool-result', + toolCallId: id, + toolName: name, + output: { type: 'execution-denied', reason: `User denied tool execution: ${name}` }, } - }, + } + if (opts.skippedDisabled) { + return { + type: 'tool-result', + toolCallId: id, + toolName: name, + output: { type: 'error-text', value: TOOL_SKIPPED_DISABLED_TEXT }, + } + } + if (opts.skipped) { + return { + type: 'tool-result', + toolCallId: id, + toolName: name, + output: { type: 'text', value: TOOL_SKIPPED_AFTER_REJECTION_TEXT }, + } + } + if (opts.invalid) { + return { + type: 'tool-result', + toolCallId: id, + toolName: name, + output: { type: 'error-text', value: `${name} invalid input: bad` }, + } + } + const params = input as { text?: string; topic?: string } + const value = + name === 'echo' && typeof params.text === 'string' + ? `echo:${params.text}` + : name === 'read_note' && typeof params.topic === 'string' + ? `note:${params.topic}` + : `${name} done` + return { type: 'tool-result', toolCallId: id, toolName: name, output: { type: 'text', value } } } -function toolUseResponse(id: string, name: string, input: unknown, text?: string): LLMResponse { +function toolUseResponse(id: string, name: string, input: unknown, text?: string, opts: MockToolOpts = {}): LLMResult { return { - content: [ - ...(text ? [{ type: 'text' as const, text }] : []), - { - type: 'tool_use' as const, - id, - name, - input, - }, - ], - stop_reason: 'tool_use', + text: text ?? '', + toolCalls: [{ type: 'tool-call', toolCallId: id, toolName: name, input }], + toolResults: [mockToolResult(id, name, input, opts)], + usage: emptyUsage(), + finishReason: 'tool-calls', } } -function multiToolUseResponse(calls: Array<{ id: string; name: string; input: unknown }>, text?: string): LLMResponse { +function multiToolUseResponse( + calls: Array<{ id: string; name: string; input: unknown; opts?: MockToolOpts }>, + text?: string, +): LLMResult { return { - content: [ - ...(text ? [{ type: 'text' as const, text }] : []), - ...calls.map((call) => ({ - type: 'tool_use' as const, - id: call.id, - name: call.name, - input: call.input, - })), - ], - stop_reason: 'tool_use', + text: text ?? '', + toolCalls: calls.map((call) => ({ + type: 'tool-call', + toolCallId: call.id, + toolName: call.name, + input: call.input, + })), + toolResults: calls.map((call) => mockToolResult(call.id, call.name, call.input, call.opts)), + usage: emptyUsage(), + finishReason: 'tool-calls', } } -function endTurnResponse(text: string = 'done'): LLMResponse { +function endTurnResponse(text: string = 'done'): LLMResult { return { - content: [{ type: 'text' as const, text }], - stop_reason: 'end_turn', + text, + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], } } function createLengthTokenCounter(): TokenCounter { return { - model: 'test-length-counter', countText: (text: string) => text.length, countMessages: (messages) => messages.reduce((sum, message) => sum + message.content.length, 0), - dispose: () => {}, + } +} + +/** Extract tool-call ids from an assistant message's parts (CoreMessage shape). */ +function assistantToolCallIds(message: ChatMessage): string[] { + if (message.role !== 'assistant' || !Array.isArray(message.content)) return [] + return message.content.filter((part) => part.type === 'tool-call').map((part) => part.toolCallId) +} + +/** Extract tool-result details from a tool message's parts (CoreMessage shape). */ +function toolMessageDetails(message: ChatMessage): { toolCallId: string; toolName: string; text: string } | null { + if (message.role !== 'tool') return null + const part = message.content[0] + if (!part || part.type !== 'tool-result') return null + return { + toolCallId: part.toolCallId, + toolName: part.toolName, + text: part.output.type === 'text' || part.output.type === 'error-text' ? part.output.value : '', } } @@ -85,15 +144,17 @@ function hasInvalidToolProtocol(messages: ChatMessage[]): boolean { if (message.role !== 'tool') { return true } - if (!pendingToolCallIds.has(message.tool_call_id)) { + const details = toolMessageDetails(message) + if (!details || !pendingToolCallIds.has(details.toolCallId)) { return true } - pendingToolCallIds.delete(message.tool_call_id) + pendingToolCallIds.delete(details.toolCallId) continue } - if (message.role === 'assistant' && message.tool_calls?.length) { - pendingToolCallIds = new Set(message.tool_calls.map((toolCall) => toolCall.id)) + const toolCallIds = assistantToolCallIds(message) + if (toolCallIds.length) { + pendingToolCallIds = new Set(toolCallIds) continue } @@ -106,14 +167,14 @@ function hasInvalidToolProtocol(messages: ChatMessage[]): boolean { describe('session hooks & middleware', () => { test('invokes hooks and middlewares in order', async () => { - const outputs: LLMResponse[] = [toolUseResponse('action-1', 'echo', { text: 'foo' }), endTurnResponse('done')] + const outputs: LLMResult[] = [toolUseResponse('action-1', 'echo', { text: 'foo' }), endTurnResponse('done')] const hookLog: string[] = [] const session = await createAgentSession( { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), // 自动批准所有工具调用 requestApproval: async () => 'once', hooks: { @@ -168,7 +229,7 @@ describe('session hooks & middleware', () => { }) test('executes action from structured tool_use with accompanying text', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ toolUseResponse('action-1', 'echo', { text: 'hi' }, 'demo'), endTurnResponse('done'), ] @@ -178,7 +239,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), // 自动批准所有工具调用 requestApproval: async () => 'once', hooks: { @@ -201,8 +262,53 @@ describe('session hooks & middleware', () => { } }) + test('emits structured results for parallel tool calls', async () => { + const outputs: LLMResult[] = [ + multiToolUseResponse([ + { id: 'list-call', name: 'echo', input: { text: 'first' } }, + { id: 'search-call', name: 'read_note', input: { topic: 'second' } }, + ]), + endTurnResponse('done'), + ] + let actionIds: string[] = [] + let resultPayload: Array<{ toolCallId: string; tool: string; observation: string }> = [] + const session = await createAgentSession( + { + tools: { echo: echoTool, read_note: readNoteTool }, + callLLM: async () => outputs.shift() ?? endTurnResponse('done'), + historySinks: [], + tokenCounter: createTokenCounter(), + requestApproval: async () => 'once', + hooks: { + onAction: ({ parallelActions }) => { + actionIds = (parallelActions ?? []).map((action) => action.toolCallId) + }, + onObservation: ({ results }) => { + resultPayload = results.map(({ toolCallId, tool, observation }) => ({ + toolCallId, + tool, + observation, + })) + }, + }, + }, + {}, + ) + + try { + await session.runTurn('parallel') + assert.deepStrictEqual(actionIds, ['list-call', 'search-call']) + assert.deepStrictEqual(resultPayload, [ + { toolCallId: 'list-call', tool: 'echo', observation: 'echo:first' }, + { toolCallId: 'search-call', tool: 'read_note', observation: 'note:second' }, + ]) + } finally { + await session.close() + } + }) + test('reuses previous assistant text when end_turn arrives empty after tool call', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ toolUseResponse('action-1', 'echo', { text: 'x' }, '这是最终答案'), endTurnResponse(''), ] @@ -211,7 +317,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -226,7 +332,7 @@ describe('session hooks & middleware', () => { }) test('warns after three identical tool calls', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ toolUseResponse('loop-1', 'echo', { text: 'loop' }), toolUseResponse('loop-2', 'echo', { text: 'loop' }), toolUseResponse('loop-3', 'echo', { text: 'loop' }), @@ -237,7 +343,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), // 自动批准所有工具调用 requestApproval: async () => 'session', }, @@ -256,13 +362,13 @@ describe('session hooks & middleware', () => { }) test('bypasses approval in dangerous mode', async () => { - const outputs: LLMResponse[] = [toolUseResponse('action-1', 'echo', { text: 'safe' }), endTurnResponse('done')] + const outputs: LLMResult[] = [toolUseResponse('action-1', 'echo', { text: 'safe' }), endTurnResponse('done')] const session = await createAgentSession( { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'deny', }, { dangerous: true }, @@ -277,7 +383,7 @@ describe('session hooks & middleware', () => { }) test('uses risk-based approvals in once tool permission mode', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ toolUseResponse('action-1', 'read_note', { topic: 'memo' }), endTurnResponse('done'), ] @@ -287,7 +393,7 @@ describe('session hooks & middleware', () => { tools: { read_note: readNoteTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => { approvalAsked = true return 'deny' @@ -306,8 +412,8 @@ describe('session hooks & middleware', () => { }) test('blocks tool calls when tool permission mode is none', async () => { - const outputs: LLMResponse[] = [ - toolUseResponse('action-1', 'echo', { text: 'blocked' }), + const outputs: LLMResult[] = [ + toolUseResponse('action-1', 'echo', { text: 'blocked' }, undefined, { skippedDisabled: true }), endTurnResponse('done'), ] const session = await createAgentSession( @@ -315,7 +421,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, { toolPermissionMode: 'none' }, @@ -325,13 +431,13 @@ describe('session hooks & middleware', () => { assert.strictEqual(result.status, 'error') assert.ok(result.finalText.includes('Tool usage is disabled')) assert.strictEqual(result.steps[0]?.observation, undefined) - const deniedToolMessage = session.history.find( - (message) => message.role === 'tool' && message.tool_call_id === 'action-1', - ) + const deniedToolMessage = session.history + .map(toolMessageDetails) + .find((details) => details?.toolCallId === 'action-1') assert.ok(deniedToolMessage, 'tool message should be recorded for denied tool_call_id') - if (deniedToolMessage?.role === 'tool') { + if (deniedToolMessage) { assert.ok( - deniedToolMessage.content.includes('tools are disabled'), + deniedToolMessage.text.includes('tools are disabled'), 'tool message should explain why execution was skipped', ) } @@ -341,12 +447,15 @@ describe('session hooks & middleware', () => { }) test('rejects native tool input via validateInput before execute', async () => { - const outputs: LLMResponse[] = [toolUseResponse('action-1', 'read_text_file', {}), endTurnResponse('done')] + const outputs: LLMResult[] = [ + toolUseResponse('action-1', 'read_text_file', {}, undefined, { invalid: true }), + endTurnResponse('done'), + ] const session = await createAgentSession( { callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -362,23 +471,7 @@ describe('session hooks & middleware', () => { test('emits structured tool execution metadata in history events', async () => { const events: HistoryEvent[] = [] - const outputs = [ - { - content: [ - { - type: 'tool_use' as const, - id: 'action-1', - name: 'echo', - input: { text: 'x' }, - }, - ], - stop_reason: 'tool_use' as const, - }, - { - content: [{ type: 'text' as const, text: 'done' }], - stop_reason: 'end_turn' as const, - }, - ] + const outputs = [toolUseResponse('action-1', 'echo', { text: 'x' }), endTurnResponse('done')] const session = await createAgentSession( { tools: { echo: echoTool }, @@ -390,10 +483,16 @@ describe('session hooks & middleware', () => { }, }, ], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, - {}, + { + providerName: 'mock', + modelName: 'mock-model', + contextWindow: 64_000, + toolPermissionMode: 'once', + thinking: true, + }, ) try { const result = await session.runTurn('meta') @@ -402,6 +501,12 @@ describe('session hooks & middleware', () => { const sessionStart = events.find((event) => event.type === 'session_start') assert.ok(sessionStart, 'session_start should exist') assert.strictEqual(sessionStart?.meta?.cwd, process.cwd()) + assert.strictEqual(sessionStart?.meta?.providerName, 'mock') + assert.strictEqual(sessionStart?.meta?.modelName, 'mock-model') + assert.strictEqual(sessionStart?.meta?.contextWindow, 64_000) + assert.strictEqual(sessionStart?.meta?.toolPermissionMode, 'once') + assert.strictEqual(sessionStart?.meta?.thinking, true) + assert.strictEqual(events.find((event) => event.type === 'turn_start')?.meta?.thinking, true) assert.strictEqual(sessionStart?.role, 'system') assert.ok( typeof sessionStart?.content === 'string' && sessionStart.content.length > 0, @@ -420,20 +525,20 @@ describe('session hooks & middleware', () => { assert.strictEqual(observationEvent.meta?.phase, 'result') assert.strictEqual(observationEvent.meta?.status, 'success') assert.strictEqual(observationEvent.meta?.error_type, undefined) - assert.strictEqual(typeof observationEvent.meta?.duration_ms, 'number') + // duration_ms no longer available: SDK ToolResultPart carries no timing. } finally { await session.close() } }) test('records structured tool call/result messages without json fallback payloads', async () => { - const outputs: LLMResponse[] = [toolUseResponse('action-1', 'echo', { text: 'x' }), endTurnResponse('done')] + const outputs: LLMResult[] = [toolUseResponse('action-1', 'echo', { text: 'x' }), endTurnResponse('done')] const session = await createAgentSession( { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -442,30 +547,36 @@ describe('session hooks & middleware', () => { const result = await session.runTurn('meta') assert.strictEqual(result.finalText, 'done') - const assistantToolMessage = session.history.find( - (message) => - message.role === 'assistant' && message.tool_calls?.some((toolCall) => toolCall.id === 'action-1'), + const assistantToolMessage = session.history.find((message) => + assistantToolCallIds(message).includes('action-1'), ) assert.ok(assistantToolMessage, 'assistant tool_calls message should exist') - const toolResultMessage = session.history.find( - (message) => message.role === 'tool' && message.tool_call_id === 'action-1', - ) + const toolResultMessage = session.history + .map(toolMessageDetails) + .find((details) => details?.toolCallId === 'action-1') assert.ok(toolResultMessage, 'tool result message should exist') - if (toolResultMessage?.role === 'tool') { - assert.strictEqual(toolResultMessage.content, 'echo:x') - assert.strictEqual(toolResultMessage.name, 'echo') + if (toolResultMessage) { + assert.strictEqual(toolResultMessage.text, 'echo:x') + assert.strictEqual(toolResultMessage.toolName, 'echo') } assert.ok( !session.history.some( - (message) => message.role === 'assistant' && message.content.startsWith('{"tool":'), + (message) => + message.role === 'assistant' && + typeof message.content === 'string' && + typeof message.content === 'string' && + message.content.startsWith('{"tool":'), ), 'assistant history should not contain plain-text tool json payloads', ) assert.ok( !session.history.some( - (message) => message.role === 'user' && message.content.includes('"observation"'), + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.includes('"observation"'), ), 'history should not inject observation json through user messages', ) @@ -476,19 +587,7 @@ describe('session hooks & middleware', () => { test('emits structured rejection metadata in final event', async () => { const events: HistoryEvent[] = [] - const outputs = [ - { - content: [ - { - type: 'tool_use' as const, - id: 'reject-1', - name: 'echo', - input: { text: 'x' }, - }, - ], - stop_reason: 'tool_use' as const, - }, - ] + const outputs = [toolUseResponse('reject-1', 'echo', { text: 'x' }, undefined, { denied: true })] const session = await createAgentSession( { tools: { echo: echoTool }, @@ -500,7 +599,7 @@ describe('session hooks & middleware', () => { }, }, ], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'deny', }, {}, @@ -508,9 +607,9 @@ describe('session hooks & middleware', () => { try { const result = await session.runTurn('meta') assert.strictEqual(result.status, 'cancelled') - const toolMessage = session.history.find( - (message) => message.role === 'tool' && message.tool_call_id === 'reject-1', - ) + const toolMessage = session.history + .map(toolMessageDetails) + .find((details) => details?.toolCallId === 'reject-1') assert.ok(toolMessage, 'tool message should exist for rejected tool_call_id') const finalEvent = [...events].reverse().find((event) => event.type === 'final') assert.ok(finalEvent, 'final event should exist') @@ -518,17 +617,17 @@ describe('session hooks & middleware', () => { assert.strictEqual(finalEvent?.meta?.phase, 'result') assert.strictEqual(finalEvent?.meta?.error_type, 'approval_denied') assert.strictEqual(finalEvent?.meta?.action_id, 'reject-1') - assert.strictEqual(typeof finalEvent?.meta?.duration_ms, 'number') + // duration_ms no longer available: SDK ToolResultPart carries no timing. } finally { await session.close() } }) test('records tool messages for all tool_call_ids on fail_fast rejection', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ multiToolUseResponse([ - { id: 'reject-1', name: 'echo', input: { text: 'a' } }, - { id: 'reject-2', name: 'echo', input: { text: 'b' } }, + { id: 'reject-1', name: 'echo', input: { text: 'a' }, opts: { denied: true } }, + { id: 'reject-2', name: 'echo', input: { text: 'b' }, opts: { skipped: true } }, ]), ] const session = await createAgentSession( @@ -536,7 +635,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'deny', }, {}, @@ -546,17 +645,13 @@ describe('session hooks & middleware', () => { const result = await session.runTurn('meta') assert.strictEqual(result.status, 'cancelled') - const first = session.history.find( - (message) => message.role === 'tool' && message.tool_call_id === 'reject-1', - ) - const second = session.history.find( - (message) => message.role === 'tool' && message.tool_call_id === 'reject-2', - ) + const first = session.history.map(toolMessageDetails).find((details) => details?.toolCallId === 'reject-1') + const second = session.history.map(toolMessageDetails).find((details) => details?.toolCallId === 'reject-2') assert.ok(first, 'first tool_call_id should have a matching tool message') assert.ok(second, 'second tool_call_id should have a matching tool message') - if (second?.role === 'tool') { + if (second) { assert.ok( - second.content.includes('Skipped tool execution after previous rejection'), + second.text.includes('Skipped tool execution after previous rejection'), 'missing tool execution should be represented as skipped observation', ) } @@ -567,7 +662,7 @@ describe('session hooks & middleware', () => { test('fails with model_protocol_error when model emits plain-text tool json', async () => { const events: HistoryEvent[] = [] - const outputs: LLMResponse[] = [endTurnResponse('{"tool":"echo","input":{"text":"x"}}')] + const outputs: LLMResult[] = [endTurnResponse('{"tool":"echo","input":{"text":"x"}}')] const session = await createAgentSession( { tools: { echo: echoTool }, @@ -579,7 +674,7 @@ describe('session hooks & middleware', () => { }, }, ], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -604,17 +699,20 @@ describe('session hooks & middleware', () => { }) test('falls back to generic final error when model returns no actionable content', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ { - content: [], - stop_reason: 'stop_sequence', + text: '', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }, ] const session = await createAgentSession( { callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -633,13 +731,13 @@ describe('session hooks & middleware', () => { }) test('does not treat unknown tool json text as protocol violation', async () => { - const outputs: LLMResponse[] = [endTurnResponse('{"tool":"unknown","input":{}}')] + const outputs: LLMResult[] = [endTurnResponse('{"tool":"unknown","input":{}}')] const session = await createAgentSession( { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', }, {}, @@ -654,7 +752,7 @@ describe('session hooks & middleware', () => { }) test('emits context usage hooks at turn start and each step', async () => { - const outputs: LLMResponse[] = [toolUseResponse('action-1', 'echo', { text: 'x' }), endTurnResponse('done')] + const outputs: LLMResult[] = [toolUseResponse('action-1', 'echo', { text: 'x' }), endTurnResponse('done')] const phases: string[] = [] const session = await createAgentSession( @@ -662,7 +760,7 @@ describe('session hooks & middleware', () => { tools: { echo: echoTool }, callLLM: async () => outputs.shift() ?? endTurnResponse('done'), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), requestApproval: async () => 'once', hooks: { onContextUsage: ({ phase, step }) => { @@ -684,7 +782,7 @@ describe('session hooks & middleware', () => { }) test('auto compaction is triggered at threshold and runs at most once per turn', async () => { - const outputs: LLMResponse[] = [ + const outputs: LLMResult[] = [ toolUseResponse('action-1', 'echo', { text: 'x' }), toolUseResponse('action-2', 'echo', { text: 'y' }), endTurnResponse('done'), @@ -697,9 +795,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { autoCompactionCalls += 1 return endTurnResponse('checkpoint') @@ -717,8 +814,9 @@ describe('session hooks & middleware', () => { }, ) try { - const result = await session.runTurn('auto compact please '.repeat(20)) - assert.strictEqual(result.status, 'prompt_limit') + const result = await session.runTurn('auto compact please '.repeat(30)) + assert.strictEqual(result.status, 'ok') + assert.strictEqual(result.finalText, 'done') assert.strictEqual(autoCompactionCalls, 1) } finally { await session.close() @@ -734,9 +832,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { compactionCalls += 1 return endTurnResponse(`summary-${compactionCalls}`) @@ -759,8 +856,8 @@ describe('session hooks & middleware', () => { ) try { - const runResult = await session.runTurn('trigger auto compaction '.repeat(20)) - assert.strictEqual(runResult.status, 'prompt_limit') + const runResult = await session.runTurn('trigger auto compaction '.repeat(25)) + assert.strictEqual(runResult.status, 'ok') const manualResult = await session.compactHistory('manual') assert.strictEqual(manualResult.status, 'success') @@ -782,9 +879,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { throw new Error('compaction failed') } @@ -814,7 +910,10 @@ describe('session hooks & middleware', () => { assert.strictEqual(regularLLMCalls, 0) assert.strictEqual( session.history.some( - (message) => message.role === 'user' && message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), ), false, ) @@ -823,6 +922,152 @@ describe('session hooks & middleware', () => { } }) + test('compaction retries once with a halved budget after a failure', async () => { + const compactedUserPrompts: string[] = [] + let compactionCalls = 0 + + const session = await createAgentSession( + { + callLLM: async (messages, _onChunk, options) => { + const isCompactionCall = + messages[0]?.role === 'system' && + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext + if (isCompactionCall) { + compactionCalls += 1 + compactedUserPrompts.push(String(messages[1]?.content)) + if (compactionCalls === 1) { + throw new Error('first attempt fails') + } + return endTurnResponse('checkpoint') + } + return endTurnResponse('done') + }, + loadPrompt: async () => 'sys', + historySinks: [], + tokenCounter: createLengthTokenCounter(), + requestApproval: async () => 'once', + }, + { + contextWindow: 10_000, + autoCompactThresholdPercent: 5, + }, + ) + + try { + const result = await session.runTurn('trigger auto compaction '.repeat(25)) + assert.strictEqual(result.status, 'ok') + assert.strictEqual(compactionCalls, 2, 'exactly one retry') + const retryPrompt = compactedUserPrompts[1] ?? '' + const firstPrompt = compactedUserPrompts[0] ?? '' + assert.ok(retryPrompt.length <= firstPrompt.length, 'retry prompt must not be larger') + assert.ok( + session.history.some( + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\ncheckpoint`), + ), + ) + } finally { + await session.close() + } + }) + + test('compaction failure after retry keeps history intact', async () => { + const compactStatuses: string[] = [] + let compactionCalls = 0 + + const session = await createAgentSession( + { + callLLM: async (messages, _onChunk, options) => { + const isCompactionCall = + messages[0]?.role === 'system' && + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext + if (isCompactionCall) { + compactionCalls += 1 + throw new Error('always fails') + } + return endTurnResponse('unexpected') + }, + loadPrompt: async () => 'sys', + historySinks: [], + tokenCounter: createLengthTokenCounter(), + hooks: { + onContextCompacted: ({ status }) => { + compactStatuses.push(status) + }, + }, + }, + { + contextWindow: 200, + autoCompactThresholdPercent: 50, + }, + ) + + try { + const result = await session.runTurn('this input is intentionally long enough '.repeat(8)) + assert.strictEqual(result.status, 'prompt_limit') + assert.strictEqual(compactionCalls, 2, 'original attempt plus one retry') + assert.strictEqual(compactStatuses.filter((status) => status === 'failed').length, 1) + assert.ok( + !session.history.some( + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), + ), + ) + } finally { + await session.close() + } + }) + + test('compaction request honors the transcript budget and drops oldest messages', async () => { + let compactionUserPrompt = '' + + const session = await createAgentSession( + { + callLLM: async (messages, _onChunk, options) => { + const isCompactionCall = + messages[0]?.role === 'system' && + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext + if (isCompactionCall) { + compactionUserPrompt = String(messages[1]?.content) + return endTurnResponse('checkpoint') + } + return endTurnResponse('done') + }, + loadPrompt: async () => 'sys', + historySinks: [], + tokenCounter: createLengthTokenCounter(), + requestApproval: async () => 'once', + }, + { + contextWindow: 1000, + autoCompactThresholdPercent: 5, + }, + ) + + try { + // threshold = 5% of 1000 = 50 tokens (char-count counter). + // First turn leaves a long old user message; the second turn + // pushes history over the threshold and triggers compaction. + const first = await session.runTurn(`oldest-marker ${'x'.repeat(35)}`) + assert.strictEqual(first.status, 'ok') + const second = await session.runTurn('newest-marker tail') + assert.strictEqual(second.status, 'ok') + // Budget 50 keeps the newest message (plus the short assistant + // reply) and drops the oldest one. + assert.ok(compactionUserPrompt.includes('newest-marker tail'), 'newest message must be retained') + assert.ok(!compactionUserPrompt.includes('oldest-marker'), 'oldest messages must be dropped') + } finally { + await session.close() + } + }) + test('manual compaction skips when there is no non-system history', async () => { let llmCalls = 0 const compactStatuses: string[] = [] @@ -858,12 +1103,9 @@ describe('session hooks & middleware', () => { test('manual compaction rebuilds history with user-only context and summary', async () => { const assistantToolCall = { - id: 'call_function_l5suo7l5etii_1', - type: 'function' as const, - function: { - name: 'exec_command', - arguments: '{}', - }, + toolCallId: 'call_function_l5suo7l5etii_1', + toolName: 'exec_command', + input: {}, } let sawCompactionCall = false @@ -872,9 +1114,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { sawCompactionCall = true return endTurnResponse('compacted summary') @@ -893,12 +1134,20 @@ describe('session hooks & middleware', () => { try { session.history.push( { role: 'user', content: 'u1' }, - { role: 'assistant', content: '', tool_calls: [assistantToolCall] }, + { + role: 'assistant', + content: [{ type: 'tool-call', ...assistantToolCall }], + }, { role: 'tool', - content: 'tool-result', - tool_call_id: assistantToolCall.id, - name: 'exec_command', + content: [ + { + type: 'tool-result', + toolCallId: assistantToolCall.toolCallId, + toolName: assistantToolCall.toolName, + output: { type: 'text', value: 'tool-result' }, + }, + ], }, { role: 'assistant', content: 'a1' }, { role: 'user', content: 'u2' }, @@ -919,7 +1168,10 @@ describe('session hooks & middleware', () => { assert.strictEqual(hasInvalidToolProtocol(session.history), false) assert.ok( session.history.some( - (message) => message.role === 'user' && message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), ), 'summary message should be preserved in compacted history', ) @@ -929,12 +1181,7 @@ describe('session hooks & middleware', () => { 'compacted history should drop tool result messages', ) assert.strictEqual( - session.history.some( - (message) => - message.role === 'assistant' && - Array.isArray(message.tool_calls) && - message.tool_calls.length > 0, - ), + session.history.some((message) => assistantToolCallIds(message).length > 0), false, 'compacted history should drop assistant tool-call messages', ) @@ -948,15 +1195,15 @@ describe('session hooks & middleware', () => { }) test('manual compaction keeps recent user context within token budget', async () => { - const hugeUserMessage = 'a'.repeat(25_000) + // 90_000 ASCII chars ≈ 22_500 tokens (4 chars/token estimate), exceeding the 20_000 budget. + const hugeUserMessage = 'a'.repeat(90_000) const session = await createAgentSession( { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { return endTurnResponse('summary-budget') } @@ -980,12 +1227,15 @@ describe('session hooks & middleware', () => { const retainedUserMessage = session.history[1] assert.strictEqual(retainedUserMessage?.role, 'user') - assert.strictEqual(retainedUserMessage?.content.length, 20_000) - assert.ok(retainedUserMessage?.content.startsWith('a')) + // Truncated to the 20_000-token budget (× 4 chars/token). + assert.strictEqual(retainedUserMessage?.content.length, 80_000) + const retainedContent = retainedUserMessage?.content + assert.ok(typeof retainedContent === 'string' && retainedContent.startsWith('a')) const summaryMessage = session.history[2] assert.strictEqual(summaryMessage?.role, 'user') - assert.ok(summaryMessage?.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`)) + const summaryContent = summaryMessage?.content + assert.ok(typeof summaryContent === 'string' && summaryContent.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`)) } finally { await session.close() } @@ -997,9 +1247,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { return endTurnResponse('internal\n\nsummary\n\n\nnext') } @@ -1021,7 +1270,8 @@ describe('session hooks & middleware', () => { const summaryMessage = session.history[session.history.length - 1] assert.strictEqual(summaryMessage?.role, 'user') - assert.ok(summaryMessage?.content.endsWith('summary\n\nnext')) + const summaryContent = summaryMessage?.content + assert.ok(typeof summaryContent === 'string' && summaryContent.endsWith('summary\n\nnext')) } finally { await session.close() } @@ -1034,9 +1284,8 @@ describe('session hooks & middleware', () => { callLLM: async (messages, _onChunk, options) => { const isCompactionCall = messages[0]?.role === 'system' && - messages[0].content === CONTEXT_COMPACTION_SYSTEM_PROMPT && - Array.isArray(options?.tools) && - options.tools.length === 0 + String(messages[0].content).includes(CONTEXT_COMPACTION_SYSTEM_PROMPT) && + !options?.toolContext if (isCompactionCall) { return endTurnResponse('new summary') } @@ -1057,10 +1306,14 @@ describe('session hooks & middleware', () => { assert.strictEqual(result.status, 'success') const summaryMessages = session.history.filter( - (message) => message.role === 'user' && message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), + (message) => + message.role === 'user' && + typeof message.content === 'string' && + message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`), ) assert.strictEqual(summaryMessages.length, 1) - assert.ok(summaryMessages[0]?.content.endsWith('new summary')) + const lastSummary = summaryMessages[0]?.content + assert.ok(typeof lastSummary === 'string' && lastSummary.endsWith('new summary')) assert.strictEqual( session.history.some((message) => message.content === oldSummary), false, @@ -1233,7 +1486,7 @@ describe('session hooks & middleware', () => { const events: HistoryEvent[] = [] const generatedTitles: string[] = [] const calls: Array<{ options: unknown }> = [] - const outputs: LLMResponse[] = [endTurnResponse('done')] + const outputs: LLMResult[] = [endTurnResponse('done')] const session = await createAgentSession( { @@ -1248,7 +1501,7 @@ describe('session hooks & middleware', () => { }, }, ], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), hooks: { onTitleGenerated: ({ title }) => { generatedTitles.push(title) @@ -1275,7 +1528,7 @@ describe('session hooks & middleware', () => { test('emits session title only once across multiple turns', async () => { const events: HistoryEvent[] = [] - const outputs: LLMResponse[] = [endTurnResponse('done'), endTurnResponse('done-again')] + const outputs: LLMResult[] = [endTurnResponse('done'), endTurnResponse('done-again')] const session = await createAgentSession( { @@ -1287,7 +1540,7 @@ describe('session hooks & middleware', () => { }, }, ], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, {}, ) diff --git a/packages/core/src/api_types.ts b/packages/core/src/api_types.ts index cb5164a..aeb47a5 100644 --- a/packages/core/src/api_types.ts +++ b/packages/core/src/api_types.ts @@ -1,35 +1,6 @@ -export type ApiSuccessMeta = { - requestId: string - timestamp: string -} - -export type ApiErrorInfo = { - code: string - message: string - details?: unknown -} +import type { LanguageModelUsage } from 'ai' -export type ApiErrorMeta = ApiSuccessMeta & { - path?: string -} - -export type ApiEnvelope = - | { - success: true - data: T - meta: ApiSuccessMeta - } - | { - success: false - error: ApiErrorInfo - meta: ApiErrorMeta - } - -export type TokenUsageSummary = { - prompt: number - completion: number - total: number -} +export type TokenUsageSummary = LanguageModelUsage export type ToolUsageSummary = { total: number @@ -78,15 +49,23 @@ export type SessionTurnStep = { assistantText?: string thinking?: string action?: { + toolCallId?: string tool: string input: unknown } parallelActions?: Array<{ + toolCallId?: string tool: string input: unknown }> observation?: string resultStatus?: string + toolResults?: Array<{ + toolCallId?: string + tool: string + observation: string + resultStatus?: string + }> } export type SessionTurnDetail = { @@ -104,6 +83,18 @@ export type SessionDetail = SessionListItem & { summary: string turns: SessionTurnDetail[] events: SessionEventItem[] + /** Provider name recorded at session start, if any. */ + providerName?: string + /** Model name recorded at session start, if any. */ + modelName?: string + /** Context window (tokens) recorded at session start. */ + contextWindow?: number + /** Tool permission mode recorded at session start. */ + toolPermissionMode?: string + /** Most recently recorded thinking override (undefined follows the model profile). */ + thinking?: boolean + /** Latest successful context_compact summary, for --prev restore injection. */ + compactionSummary?: string } export type SessionListResponse = { @@ -119,122 +110,6 @@ export type SessionEventsResponse = { nextCursor: string | null } -export type QueuedInputItem = { - id: string - input: string - createdAt: string -} - -export type LiveSessionState = { - id: string - title: string - workspaceId: string - projectName: string - providerName: string - model: string - cwd: string - startedAt: string - status: 'idle' | 'running' | 'closed' - pendingApproval?: { - fingerprint: string - toolName: string - reason: string - riskLevel: string - params: unknown - } - activeMcpServers: string[] - toolPermissionMode: 'none' | 'once' | 'full' - queuedInputs: QueuedInputItem[] - currentContextTokens?: number - contextWindow?: number -} - -export type WsServerEvent = - | { type: 'session.snapshot'; payload: LiveSessionState } - | { - type: 'turn.start' - payload: { turn: number; input: string; promptTokens?: number } - } - | { - type: 'assistant.chunk' - payload: { turn: number; step: number; chunk: string } - } - | { - type: 'context.usage' - payload: { - turn: number - step: number - phase: 'turn_start' | 'step_start' | 'post_compact' - promptTokens: number - contextWindow: number - thresholdTokens: number - usagePercent: number - } - } - | { - type: 'tool.action' - payload: { - turn: number - step: number - action: { tool: string; input: unknown } - parallelActions?: Array<{ tool: string; input: unknown }> - thinking?: string - } - } - | { - type: 'tool.observation' - payload: { - turn: number - step: number - observation: string - resultStatus?: string - parallelResultStatuses?: string[] - } - } - | { - type: 'turn.final' - payload: { - turn: number - step?: number - finalText: string - status: string - errorMessage?: string - turnUsage?: TokenUsageSummary - tokenUsage?: TokenUsageSummary - } - } - | { - type: 'approval.request' - payload: { - fingerprint: string - toolName: string - reason: string - riskLevel: string - params: unknown - } - } - | { - type: 'session.status' - payload: { - status: 'idle' | 'running' | 'closed' - } - } - | { - type: 'system.message' - payload: { - title: string - content: string - tone?: 'info' | 'warning' | 'error' - } - } - | { - type: 'error' - payload: { - code: string - message: string - } - } - export type SkillRecord = { id: string name: string @@ -250,31 +125,3 @@ export type McpServerRecord = { authStatus: 'unsupported' | 'not_logged_in' | 'bearer_token' | 'oauth' active: boolean } - -export type WorkspaceRecord = { - id: string - name: string - cwd: string - createdAt: string - lastUsedAt: string -} - -export type WorkspaceDirEntry = { - name: string - path: string - kind: 'dir' - readable: boolean -} - -export type WorkspaceFsListResult = { - path: string - parentPath: string | null - items: WorkspaceDirEntry[] -} - -export type SessionRuntimeBadge = { - sessionId: string - status: 'idle' | 'running' | 'closed' - workspaceId: string - updatedAt: string -} diff --git a/packages/core/src/runtime/file_suggestions.test.ts b/packages/core/src/features/file_suggestions/file_suggestions.test.ts similarity index 100% rename from packages/core/src/runtime/file_suggestions.test.ts rename to packages/core/src/features/file_suggestions/file_suggestions.test.ts diff --git a/packages/core/src/runtime/file_suggestions.ts b/packages/core/src/features/file_suggestions/file_suggestions.ts similarity index 100% rename from packages/core/src/runtime/file_suggestions.ts rename to packages/core/src/features/file_suggestions/file_suggestions.ts diff --git a/packages/core/src/features/file_suggestions/index.ts b/packages/core/src/features/file_suggestions/index.ts new file mode 100644 index 0000000..42979c8 --- /dev/null +++ b/packages/core/src/features/file_suggestions/index.ts @@ -0,0 +1 @@ +export * from './file_suggestions' diff --git a/packages/core/src/runtime/history_index.test.ts b/packages/core/src/features/history/history_index.test.ts similarity index 100% rename from packages/core/src/runtime/history_index.test.ts rename to packages/core/src/features/history/history_index.test.ts diff --git a/packages/core/src/runtime/history_index.ts b/packages/core/src/features/history/history_index.ts similarity index 99% rename from packages/core/src/runtime/history_index.ts rename to packages/core/src/features/history/history_index.ts index e627282..b036903 100644 --- a/packages/core/src/runtime/history_index.ts +++ b/packages/core/src/features/history/history_index.ts @@ -7,9 +7,9 @@ import type { SessionListItem, SessionListResponse, ToolUsageSummary, -} from '../api_types.js' +} from '../../api_types.js' import { parseHistoryLogToSessionDetail } from './history_parser.js' -import { cwdBelongsToWorkspace } from './workspace.js' +import { cwdBelongsToWorkspace } from '../../utils/workspace.js' type SessionFileMeta = { filePath: string diff --git a/packages/core/src/features/history/history_parser.test.ts b/packages/core/src/features/history/history_parser.test.ts new file mode 100644 index 0000000..528e01c --- /dev/null +++ b/packages/core/src/features/history/history_parser.test.ts @@ -0,0 +1,195 @@ +import assert from 'node:assert' +import { describe, test } from 'vitest' +import { parseHistoryLogToSessionDetail } from './history_parser' + +function buildSampleLog(): string { + return [ + JSON.stringify({ + ts: '2026-02-15T10:00:00.000Z', + sessionId: 's1', + type: 'session_start', + meta: { + cwd: '/tmp/demo', + providerName: 'deepseek', + modelName: 'deepseek-chat', + contextWindow: 64000, + toolPermissionMode: 'once', + thinking: true, + }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:01.000Z', + sessionId: 's1', + turn: 1, + type: 'turn_start', + content: 'hello', + }), + JSON.stringify({ + ts: '2026-02-15T10:00:02.000Z', + sessionId: 's1', + turn: 1, + step: 0, + type: 'assistant', + content: 'world', + }), + JSON.stringify({ + ts: '2026-02-15T10:00:03.000Z', + sessionId: 's1', + turn: 1, + step: 0, + type: 'action', + meta: { tool: 'read_file', input: { path: 'a.txt' } }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:04.000Z', + sessionId: 's1', + turn: 1, + step: 0, + type: 'observation', + content: 'ok', + meta: { tool: 'read_file', status: 'success' }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:05.000Z', + sessionId: 's1', + turn: 1, + type: 'final', + content: 'done', + meta: { + status: 'ok', + tokens: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + }, + }), + ].join('\n') +} + +describe('parseHistoryLogToSessionDetail', () => { + test('parses summary and turns', () => { + const detail = parseHistoryLogToSessionDetail(buildSampleLog(), '/tmp/demo/s1.jsonl') + assert.strictEqual(detail.sessionId, 's1') + assert.strictEqual(detail.project, 'demo') + assert.strictEqual(detail.turnCount, 1) + assert.strictEqual(detail.toolUsage.total, 1) + assert.strictEqual(detail.toolUsage.success, 1) + assert.strictEqual(detail.tokenUsage.totalTokens, 15) + assert.strictEqual(detail.turns.length, 1) + assert.strictEqual(detail.turns[0]?.steps.length, 1) + assert.ok(detail.summary.includes('User: hello')) + }) + + test('restores session ui state from session_start meta', () => { + const detail = parseHistoryLogToSessionDetail(buildSampleLog(), '/tmp/demo/s1.jsonl') + assert.strictEqual(detail.providerName, 'deepseek') + assert.strictEqual(detail.modelName, 'deepseek-chat') + assert.strictEqual(detail.contextWindow, 64000) + assert.strictEqual(detail.toolPermissionMode, 'once') + assert.strictEqual(detail.thinking, true) + }) + + test('restores the latest thinking mode recorded by a turn', () => { + const log = [ + buildSampleLog(), + JSON.stringify({ + ts: '2026-02-15T10:00:06.000Z', + sessionId: 's1', + turn: 2, + type: 'turn_start', + content: 'continue', + meta: { thinking: false }, + }), + ].join('\n') + + const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/s1.jsonl') + assert.strictEqual(detail.thinking, false) + }) + + test('ignores invalid optional ui state from older or damaged logs', () => { + const log = JSON.stringify({ + ts: '2026-02-15T10:00:00.000Z', + sessionId: 'legacy', + type: 'session_start', + meta: { cwd: '/tmp/demo', contextWindow: -1, thinking: 'yes' }, + }) + + const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/legacy.jsonl') + assert.strictEqual(detail.providerName, undefined) + assert.strictEqual(detail.contextWindow, undefined) + assert.strictEqual(detail.thinking, undefined) + }) + + test('sanitizes think/thinking blocks from title', () => { + const log = [ + JSON.stringify({ + ts: '2026-02-15T10:00:00.000Z', + sessionId: 's2', + type: 'session_start', + meta: { cwd: '/tmp/demo' }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:01.000Z', + sessionId: 's2', + type: 'session_title', + content: 'internal chain of thought Build release plan hidden', + }), + ].join('\n') + + const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/s2.jsonl') + assert.strictEqual(detail.title, 'Build release plan') + }) + + test('exposes the latest successful context_compact summary', () => { + const log = [ + JSON.stringify({ + ts: '2026-02-15T10:00:00.000Z', + sessionId: 's3', + type: 'session_start', + meta: { cwd: '/tmp/demo' }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:02.000Z', + sessionId: 's3', + type: 'context_compact', + content: '', + meta: { status: 'failed' }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:03.000Z', + sessionId: 's3', + type: 'context_compact', + content: 'first summary', + meta: { status: 'success', beforeTokens: 100, afterTokens: 20 }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:04.000Z', + sessionId: 's3', + type: 'context_compact', + content: 'latest summary', + meta: { status: 'success', beforeTokens: 80, afterTokens: 15 }, + }), + ].join('\n') + + const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/s3.jsonl') + assert.strictEqual(detail.compactionSummary, 'latest summary') + }) + + test('leaves compactionSummary undefined when only failed compactions exist', () => { + const log = [ + JSON.stringify({ + ts: '2026-02-15T10:00:00.000Z', + sessionId: 's4', + type: 'session_start', + meta: { cwd: '/tmp/demo' }, + }), + JSON.stringify({ + ts: '2026-02-15T10:00:02.000Z', + sessionId: 's4', + type: 'context_compact', + content: '', + meta: { status: 'failed' }, + }), + ].join('\n') + + const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/s4.jsonl') + assert.strictEqual(detail.compactionSummary, undefined) + }) +}) diff --git a/packages/core/src/runtime/history_parser.ts b/packages/core/src/features/history/history_parser.ts similarity index 80% rename from packages/core/src/runtime/history_parser.ts rename to packages/core/src/features/history/history_parser.ts index 0ebf9d2..287bed9 100644 --- a/packages/core/src/runtime/history_parser.ts +++ b/packages/core/src/features/history/history_parser.ts @@ -8,8 +8,8 @@ import type { SessionTurnStep, TokenUsageSummary, ToolUsageSummary, -} from '../api_types.js' -import { workspaceIdFromCwd } from './workspace.js' +} from '../../api_types.js' +import { workspaceIdFromCwd } from '../../utils/workspace.js' type MutableTurnDetail = SessionTurnDetail & { byStep: Map @@ -20,6 +20,11 @@ type ParseResultState = { title: string project: string cwd: string + providerName?: string + modelName?: string + contextWindow?: number + toolPermissionMode?: string + thinking?: boolean startedAt: string updatedAt: string status: SessionRuntimeStatus @@ -28,15 +33,18 @@ type ParseResultState = { toolUsage: ToolUsageSummary turnsById: Map summaryParts: string[] + compactionSummary?: string hasError: boolean hasCancelled: boolean } function defaultTokenUsage(): TokenUsageSummary { return { - prompt: 0, - completion: 0, - total: 0, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + inputTokenDetails: { noCacheTokens: undefined, cacheReadTokens: undefined, cacheWriteTokens: undefined }, + outputTokenDetails: { textTokens: undefined, reasoningTokens: undefined }, } } @@ -177,13 +185,13 @@ function parseEventLine(line: string, index: number): SessionEventItem | null { function accumulateTokenUsage(target: TokenUsageSummary, source: Record | undefined): void { if (!source) return - const prompt = asNumber(source.prompt) - const completion = asNumber(source.completion) - const total = asNumber(source.total) + const inputTokens = asNumber(source.inputTokens) + const outputTokens = asNumber(source.outputTokens) + const totalTokens = asNumber(source.totalTokens) - if (prompt !== null) target.prompt += Math.floor(prompt) - if (completion !== null) target.completion += Math.floor(completion) - if (total !== null) target.total += Math.floor(total) + if (inputTokens !== null) target.inputTokens = (target.inputTokens ?? 0) + Math.floor(inputTokens) + if (outputTokens !== null) target.outputTokens = (target.outputTokens ?? 0) + Math.floor(outputTokens) + if (totalTokens !== null) target.totalTokens = (target.totalTokens ?? 0) + Math.floor(totalTokens) } function normalizeFinalStatus(raw: string | undefined): SessionRuntimeStatus { @@ -269,6 +277,11 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S title: '', project: '', cwd: '', + providerName: undefined, + modelName: undefined, + contextWindow: undefined, + toolPermissionMode: undefined, + thinking: undefined, startedAt: events[0]?.ts ?? fallbackNow, updatedAt: events[events.length - 1]?.ts ?? events[0]?.ts ?? fallbackNow, status: 'idle', @@ -289,6 +302,19 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S if (event.type === 'session_start' && event.meta) { const cwd = safeString(event.meta.cwd) if (cwd) state.cwd = cwd + const providerName = safeString(event.meta.providerName) + if (providerName) state.providerName = providerName + const modelName = safeString(event.meta.modelName) + if (modelName) state.modelName = modelName + const contextWindow = event.meta.contextWindow + if (typeof contextWindow === 'number' && Number.isFinite(contextWindow) && contextWindow > 0) { + state.contextWindow = contextWindow + } + const toolPermissionMode = safeString(event.meta.toolPermissionMode) + if (toolPermissionMode) state.toolPermissionMode = toolPermissionMode + if (typeof event.meta.thinking === 'boolean') { + state.thinking = event.meta.thinking + } } if (event.type === 'session_title') { @@ -297,6 +323,9 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S } if (event.type === 'turn_start') { + if (typeof event.meta?.thinking === 'boolean') { + state.thinking = event.meta.thinking + } const turnId = event.turn ?? state.turnCount + 1 const turn = ensureTurn(state, turnId) turn.input = event.content @@ -329,6 +358,7 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S const tool = safeString(event.meta?.tool) if (tool) { step.action = { + toolCallId: safeString(event.meta?.action_id) || undefined, tool, input: event.meta?.input, } @@ -343,11 +373,14 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S const name = safeString(block.name) if (!name) return null return { + toolCallId: safeString(block.id) || undefined, tool: name, input: block.input, } }) - .filter((item): item is { tool: string; input: unknown } => Boolean(item)) + .filter((item): item is { toolCallId: string | undefined; tool: string; input: unknown } => + Boolean(item), + ) if (parallelActions.length > 1) step.parallelActions = parallelActions } } @@ -358,6 +391,13 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S step.observation = event.content const status = safeString(event.meta?.status) if (status) step.resultStatus = status + const result = { + toolCallId: safeString(event.meta?.action_id) || undefined, + tool: safeString(event.meta?.tool) || step.action?.tool || '', + observation: event.content ?? '', + resultStatus: status || undefined, + } + step.toolResults = [...(step.toolResults ?? []), result] applyObservationStatus(state, status) } @@ -395,6 +435,15 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S } } + if (event.type === 'context_compact') { + // Only successful compactions carry content; events are time-ordered, + // so the last one wins (later summaries absorb earlier ones). + const content = safeString(event.content) + if (content) { + state.compactionSummary = content + } + } + if (event.type === 'session_end' && isRecord(event.meta?.tokens)) { state.tokenUsage = defaultTokenUsage() accumulateTokenUsage(state.tokenUsage, event.meta.tokens) @@ -452,5 +501,11 @@ export function parseHistoryLogToSessionDetail(raw: string, filePath: string): S summary: state.summaryParts.join('\n'), turns, events, + providerName: state.providerName, + modelName: state.modelName, + contextWindow: state.contextWindow, + toolPermissionMode: state.toolPermissionMode, + thinking: state.thinking, + compactionSummary: state.compactionSummary, } } diff --git a/packages/core/src/runtime/history.test.ts b/packages/core/src/features/history/history_sink.test.ts similarity index 98% rename from packages/core/src/runtime/history.test.ts rename to packages/core/src/features/history/history_sink.test.ts index fff7a4c..1f28b7f 100644 --- a/packages/core/src/runtime/history.test.ts +++ b/packages/core/src/features/history/history_sink.test.ts @@ -3,7 +3,8 @@ import { unlink, readFile } from 'node:fs/promises' import { existsSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' -import { JsonlHistorySink, createHistoryEvent } from '@memo/core/runtime/history' +import { JsonlHistorySink } from '@memo/core/features/history' +import { createHistoryEvent } from '@memo/core/agent/loop' const getTempFilePath = () => join(tmpdir(), `memo-test-${Date.now()}.jsonl`) diff --git a/packages/core/src/runtime/history.ts b/packages/core/src/features/history/history_sink.ts similarity index 62% rename from packages/core/src/runtime/history.ts rename to packages/core/src/features/history/history_sink.ts index df2a470..5c3db44 100644 --- a/packages/core/src/runtime/history.ts +++ b/packages/core/src/features/history/history_sink.ts @@ -1,7 +1,7 @@ -/** @file History event definition and JSONL Sink implementation. */ +/** @file JSONL history writer: one event per line. */ import { appendFile, mkdir } from 'node:fs/promises' import { dirname } from 'node:path' -import type { HistoryEvent, HistorySink, Role } from '@memo/core/types' +import type { HistoryEvent, HistorySink } from '@memo/core/types' /** JSONL history writer: one event per line. */ export class JsonlHistorySink implements HistorySink { @@ -39,25 +39,3 @@ export class JsonlHistorySink implements HistorySink { await this.flush() } } - -/** Helper to generate structured history events. */ -export function createHistoryEvent(params: { - sessionId: string - type: HistoryEvent['type'] - turn?: number - step?: number - content?: string - role?: Role - meta?: Record -}): HistoryEvent { - return { - ts: new Date().toISOString(), - sessionId: params.sessionId, - turn: params.turn, - step: params.step, - type: params.type, - content: params.content, - role: params.role, - meta: params.meta, - } -} diff --git a/packages/core/src/features/history/index.ts b/packages/core/src/features/history/index.ts new file mode 100644 index 0000000..48d9f0e --- /dev/null +++ b/packages/core/src/features/history/index.ts @@ -0,0 +1,3 @@ +export * from './history_sink' +export * from './history_parser' +export * from './history_index' diff --git a/packages/core/src/runtime/slash/index.ts b/packages/core/src/features/slash/index.ts similarity index 100% rename from packages/core/src/runtime/slash/index.ts rename to packages/core/src/features/slash/index.ts diff --git a/packages/core/src/runtime/slash/registry.test.ts b/packages/core/src/features/slash/registry.test.ts similarity index 88% rename from packages/core/src/runtime/slash/registry.test.ts rename to packages/core/src/features/slash/registry.test.ts index 4d3eac1..5627df8 100644 --- a/packages/core/src/runtime/slash/registry.test.ts +++ b/packages/core/src/features/slash/registry.test.ts @@ -44,28 +44,6 @@ describe('slash registry', () => { assert.deepStrictEqual(resolveSlashCommand('/init', ctx), { kind: 'init_agents_md' }) }) - test('parses review command from number/hash/url and validates usage', () => { - const ctx = makeContext() - assert.deepStrictEqual(resolveSlashCommand('/review 123', ctx), { - kind: 'review_pr', - prNumber: 123, - }) - assert.deepStrictEqual(resolveSlashCommand('/review #88', ctx), { - kind: 'review_pr', - prNumber: 88, - }) - assert.deepStrictEqual(resolveSlashCommand('/review https://github.com/a/b/pull/77', ctx), { - kind: 'review_pr', - prNumber: 77, - }) - - const invalid = resolveSlashCommand('/review abc', ctx) - assert.strictEqual(invalid.kind, 'message') - if (invalid.kind === 'message') { - assert.ok(invalid.content.includes('Usage: /review')) - } - }) - test('handles models command for empty, switch, and not-found paths', () => { const noProviders = resolveSlashCommand('/models', makeContext({ providers: [] })) assert.strictEqual(noProviders.kind, 'message') diff --git a/packages/core/src/runtime/slash/registry.ts b/packages/core/src/features/slash/registry.ts similarity index 85% rename from packages/core/src/runtime/slash/registry.ts rename to packages/core/src/features/slash/registry.ts index 1702454..dbda482 100644 --- a/packages/core/src/runtime/slash/registry.ts +++ b/packages/core/src/features/slash/registry.ts @@ -7,7 +7,6 @@ export const SLASH_SPECS: SlashSpec[] = [ { name: SLASH_COMMANDS.EXIT, description: 'Exit current session' }, { name: SLASH_COMMANDS.NEW, description: 'Start a fresh session' }, { name: SLASH_COMMANDS.RESUME, description: 'List and load session history' }, - { name: SLASH_COMMANDS.REVIEW, description: 'Review a GitHub pull request and post comments' }, { name: SLASH_COMMANDS.MODELS, description: 'List or switch configured models' }, { name: SLASH_COMMANDS.TOOLS, @@ -63,30 +62,11 @@ export function buildHelpText(): string { ' Up/Down Browse local input history', ' Tab Accept active suggestion', ' Ctrl+L Clear screen and start new session', - ' Esc Esc Interrupt running turn / clear input', + ' Ctrl+T Pause / resume live output', + ' Esc×2 Interrupt running turn / clear input', ].join('\n') } -function parseReviewPrNumber(input: string | undefined): number | null { - if (!input) return null - const normalized = input.trim() - if (!normalized) return null - - const directMatch = normalized.match(/^#?(\d+)$/) - if (directMatch) { - const parsed = Number(directMatch[1]) - return Number.isInteger(parsed) && parsed > 0 ? parsed : null - } - - const urlMatch = normalized.match(/\/pull\/(\d+)(?:[/?#].*)?$/i) - if (urlMatch) { - const parsed = Number(urlMatch[1]) - return Number.isInteger(parsed) && parsed > 0 ? parsed : null - } - - return null -} - export function resolveSlashCommand(raw: string, context: SlashContext): SlashCommandResult { const [commandRaw, ...rest] = raw.trim().slice(1).split(/\s+/) const command = (commandRaw ?? '').toLowerCase() @@ -108,22 +88,6 @@ export function resolveSlashCommand(raw: string, context: SlashContext): SlashCo content: 'Type "resume" followed by keywords to load local session history.', } - case SLASH_COMMANDS.REVIEW: { - const arg = rest.join(' ').trim() - const prNumber = parseReviewPrNumber(arg) - if (!prNumber) { - return { - kind: 'message', - title: 'Review', - content: `Usage: ${formatSlashCommand(SLASH_COMMANDS.REVIEW)} \nExamples: /review 999, /review #999`, - } - } - return { - kind: 'review_pr', - prNumber, - } - } - case SLASH_COMMANDS.MODELS: { if (!context.providers.length) { return { diff --git a/packages/core/src/runtime/slash/types.ts b/packages/core/src/features/slash/types.ts similarity index 94% rename from packages/core/src/runtime/slash/types.ts rename to packages/core/src/features/slash/types.ts index 95f3baa..ee778e7 100644 --- a/packages/core/src/runtime/slash/types.ts +++ b/packages/core/src/features/slash/types.ts @@ -6,7 +6,6 @@ export const SLASH_COMMANDS = { EXIT: 'exit', NEW: 'new', RESUME: 'resume', - REVIEW: 'review', MODELS: 'models', TOOLS: 'tools', COMPACT: 'compact', @@ -39,7 +38,6 @@ export type SlashCommandResult = | { kind: 'exit' } | { kind: 'new' } | { kind: 'message'; title: string; content: string } - | { kind: 'review_pr'; prNumber: number } | { kind: 'switch_model'; provider: ProviderConfig } | { kind: 'set_tool_permission'; mode: ToolPermissionMode } | { kind: 'compact' } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e972079..b8351f0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,18 +1,26 @@ -/** @file Core package entry point, aggregates runtime/config/tools common APIs. */ +/** @file Core package entry point, aggregates domain modules (config/llm/session/history/...). */ export * from './types' -export * from './runtime/prompt' -export * from './runtime/skills' -export * from './runtime/history' -export * from './runtime/history_parser' -export * from './runtime/history_index' -export * from './runtime/workspace' -export * from './runtime/file_suggestions' -export * from './runtime/slash' -export * from './runtime/mcp_admin' -export * from './runtime/skills_admin' -export * from './runtime/defaults' +// ToolRegistry/MCPServerConfig are exported from types/config; the router re-export is skipped to avoid ambiguity. +export { TOOLKIT, TOOL_LIST, NATIVE_TOOLS } from './tools' +export * from './tools/approval' +export * from './prompt/prompt' +export * from './skills/skills' +export * from './utils/workspace' +export * from './features/file_suggestions' +export * from './features/slash' +export * from './features/history' +export * from './mcp/mcp_admin' +export { + loginMcpServerOAuth, + logoutMcpServerOAuth, + getMcpAuthStatus, + type McpAuthStatus, +} from './tools/router/mcp/oauth' +export * from './skills/skills_admin' +export { CONTEXT_SUMMARY_PREFIX, isContextSummaryMessage } from './agent/compact_prompt' +export * from './agent/defaults' export * from './config/config' export * from './utils/utils' export * from './utils/tokenizer' -export * from './runtime/session' +export * from './agent/session' export * from './api_types' diff --git a/packages/core/src/llm/ai_provider.test.ts b/packages/core/src/llm/ai_provider.test.ts new file mode 100644 index 0000000..7704078 --- /dev/null +++ b/packages/core/src/llm/ai_provider.test.ts @@ -0,0 +1,122 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest' +import { getProviderFactory } from '@memo/core/llm/ai_provider' +import type { ModelProfile } from '@memo/core/llm/model_profile' + +const state = vi.hoisted(() => ({ + createCalls: [] as unknown[], +})) + +vi.mock('@ai-sdk/openai-compatible', () => ({ + createOpenAICompatible: vi.fn((options: unknown) => { + state.createCalls.push(options) + return (model: string) => ({ model }) + }), +})) + +const PROFILE: ModelProfile = { + wireApi: 'chat_completions', + supportsParallelToolCalls: false, + supportsReasoningContent: false, + isFallback: false, +} + +describe('getProviderFactory', () => { + beforeEach(() => { + state.createCalls = [] + }) + + test('dispatches deepseek to openai-compatible with default base URL', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.kind).toBe('openai-compatible') + factory.build({ name: 'deepseek', env_api_key: 'DEEPSEEK_API_KEY', model: 'deepseek-chat' }, 'secret') + expect(state.createCalls[0]).toEqual({ + name: 'deepseek', + apiKey: 'secret', + baseURL: 'https://api.deepseek.com', + includeUsage: true, + }) + }) + + test('config base_url overrides the default', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + factory.build( + { + name: 'deepseek', + env_api_key: 'DEEPSEEK_API_KEY', + model: 'deepseek-chat', + base_url: 'https://proxy.local/v1', + }, + 'secret', + ) + expect((state.createCalls[0] as { baseURL: string }).baseURL).toBe('https://proxy.local/v1') + }) + + test('falls back to openai-compatible for unknown providers', () => { + const factory = getProviderFactory({ name: 'my-custom-vendor' }) + expect(factory.kind).toBe('openai-compatible') + factory.build({ name: 'my-custom-vendor', env_api_key: 'X_KEY', model: 'm' }, 'secret') + expect((state.createCalls[0] as { baseURL: string }).baseURL).toBe('https://api.openai.com/v1') + expect((state.createCalls[0] as { name: string }).name).toBe('my-custom-vendor') + }) + + test('anthropic is registered but throws on build until wired', () => { + const factory = getProviderFactory({ name: 'anthropic' }) + expect(factory.kind).toBe('anthropic') + expect(() => + factory.build({ name: 'anthropic', env_api_key: 'ANTHROPIC_API_KEY', model: 'claude' }, 'secret'), + ).toThrow('not yet wired to AI SDK') + }) + + test('is case/whitespace insensitive on provider name', () => { + expect(getProviderFactory({ name: ' DeepSeek ' }).kind).toBe('openai-compatible') + }) +}) + +describe('buildProviderOptions', () => { + test('no options when parallel tool calls unsupported', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.buildProviderOptions(PROFILE)).toBeUndefined() + }) + + test('parallel_tool_calls passthrough when supported', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.buildProviderOptions({ ...PROFILE, supportsParallelToolCalls: true })).toEqual({ + parallel_tool_calls: true, + }) + }) + + test('thinking passthrough when reasoning content supported', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.buildProviderOptions({ ...PROFILE, supportsReasoningContent: true })).toEqual({ + thinking: { type: 'enabled' }, + }) + }) + + test('combines parallel_tool_calls and thinking', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect( + factory.buildProviderOptions({ + ...PROFILE, + supportsParallelToolCalls: true, + supportsReasoningContent: true, + }), + ).toEqual({ + parallel_tool_calls: true, + thinking: { type: 'enabled' }, + }) + }) + + test('thinking=true enables reasoning regardless of profile', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.buildProviderOptions(PROFILE, true)).toEqual({ + thinking: { type: 'enabled' }, + }) + }) + + test('thinking=false sends explicit disabled (deepseek defaults to thinking)', () => { + const factory = getProviderFactory({ name: 'deepseek' }) + expect(factory.buildProviderOptions({ ...PROFILE, supportsReasoningContent: true }, false)).toEqual({ + thinking: { type: 'disabled' }, + }) + }) +}) diff --git a/packages/core/src/llm/ai_provider.ts b/packages/core/src/llm/ai_provider.ts new file mode 100644 index 0000000..f418621 --- /dev/null +++ b/packages/core/src/llm/ai_provider.ts @@ -0,0 +1,68 @@ +/** @file AI SDK provider factory registry: dispatch by provider name to AI SDK providers. */ +import { createOpenAICompatible, type OpenAICompatibleProvider } from '@ai-sdk/openai-compatible' +import type { JSONValue } from 'ai' +import type { ProviderConfig } from '@memo/core/config/config' +import type { ModelProfile } from '@memo/core/llm/model_profile' + +/** Wire API kinds supported by the registry (future: responses / messages). */ +export type ProviderKind = 'openai-compatible' | 'openai' | 'anthropic' + +export type AIProviderFactory = { + kind: ProviderKind + /** Build an AI SDK provider instance (callable: factory(config, apiKey)('model-id')). */ + build: (config: ProviderConfig, apiKey: string) => OpenAICompatibleProvider + /** + * Request-level providerOptions for non-standard wire fields. + * Keyed by the provider instance name (config.name) inside streamCallLLM. + * `thinking` overrides the profile flag (undefined follows profile.supportsReasoningContent). + */ + buildProviderOptions: (profile: ModelProfile, thinking?: boolean) => Record | undefined +} + +function openAICompatibleFactory(defaultBaseURL?: string): AIProviderFactory { + return { + kind: 'openai-compatible', + build: (config, apiKey) => + createOpenAICompatible({ + name: config.name, + apiKey, + // Same default as the OpenAI SDK when base_url is unset. + baseURL: config.base_url ?? defaultBaseURL ?? 'https://api.openai.com/v1', + // Stream usage back through stream_options.include_usage. + includeUsage: true, + }), + buildProviderOptions: (profile, thinking) => { + // Non-standard deepseek fields passed through under the provider name: + // parallel_tool_calls, thinking (reasoning toggle). + // deepseek defaults to thinking; disabling requires an explicit `disabled` value (codex-style). + const options: Record = {} + if (profile.supportsParallelToolCalls) options.parallel_tool_calls = true + if (thinking === false) { + options.thinking = { type: 'disabled' } + } else if (thinking ?? profile.supportsReasoningContent) { + options.thinking = { type: 'enabled' } + } + return Object.keys(options).length > 0 ? options : undefined + }, + } +} + +const REGISTRY: Readonly> = { + deepseek: openAICompatibleFactory('https://api.deepseek.com'), + // Extend here when wiring new providers: + // - 'openai': switch to @ai-sdk/openai (chat completions or Responses API); buildProviderOptions uses camelCase parallelToolCalls. + // - 'anthropic': switch to @ai-sdk/anthropic (Messages API); wire format no longer OpenAI-compatible. + anthropic: { + kind: 'anthropic', + build: () => { + throw new Error("Provider 'anthropic' requires the Anthropic Messages API; not yet wired to AI SDK") + }, + buildProviderOptions: () => undefined, + }, +} + +/** Dispatch by provider name; unknown names fall back to OpenAI-compatible (config.base_url decides the endpoint). */ +export function getProviderFactory(config: Pick): AIProviderFactory { + const name = config.name.trim().toLowerCase() + return REGISTRY[name] ?? openAICompatibleFactory() +} diff --git a/packages/core/src/llm/ai_stream.test.ts b/packages/core/src/llm/ai_stream.test.ts new file mode 100644 index 0000000..b1692ab --- /dev/null +++ b/packages/core/src/llm/ai_stream.test.ts @@ -0,0 +1,249 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest' +import { streamCallLLM } from '@memo/core/llm/ai_stream' +import type { AIProviderFactory } from '@memo/core/llm/ai_provider' +import type { ModelProfile } from '@memo/core/llm/model_profile' +import type { ChatMessage } from '@memo/core/types' + +const state = vi.hoisted(() => ({ + streamTextParams: [] as unknown[], + parts: [] as unknown[], + final: {} as Record, +})) + +vi.mock('ai', () => ({ + streamText: vi.fn((params: unknown) => { + state.streamTextParams.push(params) + return makeStreamResult() + }), + jsonSchema: (schema: unknown) => schema, +})) + +function makeStreamResult() { + async function* gen() { + yield* state.parts + } + return { + fullStream: gen(), + // AI SDK v6 awaitable properties (Promise.resolve values are awaitable). + text: Promise.resolve(state.final.text ?? ''), + reasoningText: Promise.resolve(state.final.reasoning), + toolCalls: Promise.resolve(state.final.toolCalls ?? []), + toolResults: Promise.resolve(state.final.toolResults ?? []), + usage: Promise.resolve(state.final.usage), + finishReason: Promise.resolve(state.final.finishReason ?? 'stop'), + } +} + +const PROFILE: ModelProfile = { + wireApi: 'chat_completions', + supportsParallelToolCalls: false, + supportsReasoningContent: false, + isFallback: false, +} + +const FACTORY: AIProviderFactory = { + kind: 'openai-compatible', + build: () => ((model: string) => model) as never, + buildProviderOptions: () => undefined, +} + +function baseParams(overrides: Record = {}) { + return { + provider: { name: 'mock', env_api_key: 'MOCK_API_KEY', model: 'mock-model', base_url: 'https://mock.local/v1' }, + apiKey: 'test-key', + messages: [{ role: 'user', content: 'hi' }] as ChatMessage[], + profile: PROFILE, + factory: FACTORY, + ...overrides, + } +} + +function textDelta(text: string) { + return { type: 'text-delta', id: 't', text } +} + +describe('streamCallLLM', () => { + beforeEach(() => { + state.streamTextParams = [] + state.parts = [] + state.final = { + text: '', + toolCalls: [], + toolResults: [], + usage: { inputTokens: 11, outputTokens: 7, totalTokens: 18 }, + finishReason: 'stop', + } + }) + + test('streams text deltas through onChunk and returns assembled result', async () => { + state.parts = [textDelta('Hel'), textDelta('lo'), textDelta(' world')] + state.final = { + text: 'Hello world', + toolCalls: [], + toolResults: [], + usage: state.final.usage, + finishReason: 'stop', + } + const chunks: string[] = [] + const result = await streamCallLLM(baseParams({ onChunk: (chunk: string) => chunks.push(chunk) })) + + expect(chunks).toEqual(['Hel', 'lo', ' world']) + expect(result.text).toBe('Hello world') + expect(result.toolCalls).toEqual([]) + }) + + test('streams reasoning deltas through onReasoningChunk', async () => { + state.parts = [ + { type: 'reasoning-delta', id: 'r', text: 'think ' }, + { type: 'reasoning-delta', id: 'r', text: 'more' }, + textDelta('answer'), + ] + state.final = { + text: 'answer', + reasoning: 'think more', + toolCalls: [], + toolResults: [], + usage: state.final.usage, + finishReason: 'stop', + } + const reasoningChunks: string[] = [] + const textChunks: string[] = [] + const result = await streamCallLLM( + baseParams({ + onChunk: (chunk: string) => textChunks.push(chunk), + onReasoningChunk: (chunk: string) => reasoningChunks.push(chunk), + }), + ) + + expect(reasoningChunks).toEqual(['think ', 'more']) + expect(textChunks).toEqual(['answer']) + expect(result.reasoning).toBe('think more') + }) + + test('returns reasoningText as reasoning', async () => { + state.parts = [] + state.final = { + text: 'answer', + reasoning: 'thinking', + toolCalls: [], + toolResults: [], + usage: state.final.usage, + finishReason: 'stop', + } + const result = await streamCallLLM(baseParams()) + + expect(result.reasoning).toBe('thinking') + }) + + test('returns toolCalls, toolResults and usage from the final result', async () => { + state.parts = [] + state.final = { + text: 'using tools', + toolCalls: [{ type: 'tool-call', toolCallId: 'call-1', toolName: 'echo', input: { value: 1 } }], + toolResults: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'echo', + output: { type: 'text', value: 'ok' }, + }, + ], + usage: { inputTokens: 3, outputTokens: 4, totalTokens: 7 }, + finishReason: 'tool-calls', + } + const result = await streamCallLLM(baseParams()) + + expect(result.toolCalls).toHaveLength(1) + expect(result.toolResults).toHaveLength(1) + expect(result.toolResults[0]).toMatchObject({ toolCallId: 'call-1', toolName: 'echo' }) + expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 4, totalTokens: 7 }) + expect(result.finishReason).toBe('tool-calls') + }) + + test('throws error part payloads', async () => { + state.parts = [{ type: 'error', error: new Error('provider exploded') }] + await expect(streamCallLLM(baseParams())).rejects.toThrow('provider exploded') + }) + + test('normalizes aborted streams to AbortError', async () => { + state.parts = [{ type: 'error', error: new Error('fetch failed') }] + const controller = new AbortController() + controller.abort() + await expect(streamCallLLM(baseParams({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + }) + + test('omits tools and toolChoice when no tool registry', async () => { + state.parts = [] + await streamCallLLM(baseParams({ tools: undefined, toolContext: undefined })) + + const params = state.streamTextParams[0] as { tools?: unknown; toolChoice?: unknown } + expect(params.tools).toBeUndefined() + expect(params.toolChoice).toBeUndefined() + }) + + test('passes tools and toolChoice auto with experimental_context when tools provided', async () => { + const signal = new AbortController().signal + const factory: AIProviderFactory = { + kind: 'openai-compatible', + build: () => ((model: string) => model) as never, + buildProviderOptions: () => ({ parallel_tool_calls: true }), + } + state.parts = [] + const registry = { + echo: { + description: 'echo', + inputSchema: { type: 'object' }, + execute: async () => ({ type: 'text' as const, value: 'ok' }), + }, + } + const toolContext = { + approvalManager: { check: () => ({ needApproval: false as const, decision: 'auto-execute' as const }) }, + approvalHooks: {}, + toolsDisabled: false, + gate: { acquire: async () => ({ skipped: false as const, release: () => {} }), markDenied: () => {} }, + } + await streamCallLLM( + baseParams({ + factory, + signal, + tools: registry, + toolContext, + }), + ) + + const params = state.streamTextParams[0] as { + tools?: Record + toolChoice?: unknown + abortSignal?: AbortSignal + providerOptions?: unknown + experimental_context?: unknown + } + expect(params.tools).toBe(registry) + expect(params.toolChoice).toBe('auto') + expect(params.abortSignal).toBe(signal) + expect(params.experimental_context).toBe(toolContext) + expect(params.providerOptions).toEqual({ mock: { parallel_tool_calls: true } }) + }) + + test('omits tools when toolContext is absent even if tools provided', async () => { + state.parts = [] + await streamCallLLM( + baseParams({ + tools: { + echo: { + description: 'echo', + inputSchema: { type: 'object' }, + execute: async () => ({ type: 'text' as const, value: 'ok' }), + }, + }, + toolContext: undefined, + }), + ) + + const params = state.streamTextParams[0] as { tools?: unknown; toolChoice?: unknown } + expect(params.tools).toBeUndefined() + expect(params.toolChoice).toBeUndefined() + }) +}) diff --git a/packages/core/src/llm/ai_stream.ts b/packages/core/src/llm/ai_stream.ts new file mode 100644 index 0000000..c558cb0 --- /dev/null +++ b/packages/core/src/llm/ai_stream.ts @@ -0,0 +1,95 @@ +/** @file Default streaming LLM call backed by AI SDK streamText. */ +import { streamText, type ModelMessage, type ToolResultPart, type ToolSet } from 'ai' +import type { LLMResult } from '@memo/core/types' +import type { ToolExecutionContext } from '@memo/core/tools/sdk_tools' +import type { ProviderConfig } from '@memo/core/config/config' +import type { ModelProfile } from '@memo/core/llm/model_profile' +import type { AIProviderFactory } from '@memo/core/llm/ai_provider' + +export type StreamCallLLMParams = { + provider: ProviderConfig + apiKey: string + /** CoreMessage[] (ChatMessage alias) — passed to streamText as-is. */ + messages: ModelMessage[] + /** Complete tool set (native + MCP + custom); undefined disables tools (compaction). */ + tools?: ToolSet + profile: ModelProfile + factory: AIProviderFactory + toolContext?: ToolExecutionContext + /** Thinking override; undefined follows profile.supportsReasoningContent. */ + thinking?: boolean + onChunk?: (chunk: string) => void + /** Streaming reasoning deltas (thinking trace); previously dropped silently. */ + onReasoningChunk?: (chunk: string) => void + signal?: AbortSignal +} + +/** Normalize stream errors so callers can detect aborts via name/message matching. */ +export function normalizeStreamError(err: unknown, signal?: AbortSignal): Error { + if (signal?.aborted) { + const aborted = new Error('Request aborted') + aborted.name = 'AbortError' + return aborted + } + if (err instanceof Error && err.name === 'AbortError') return err + if (err instanceof Error && /aborted/i.test(err.message)) { + const aborted = new Error(err.message) + aborted.name = 'AbortError' + return aborted + } + return err instanceof Error ? err : new Error(String(err)) +} + +/** Default callLLM implementation: stream via AI SDK, tools execute inside streamText. */ +export async function streamCallLLM(params: StreamCallLLMParams): Promise { + const { + provider, + apiKey, + messages, + tools, + profile, + factory, + toolContext, + thinking, + onChunk, + onReasoningChunk, + signal, + } = params + const activeTools = tools && toolContext ? tools : undefined + const model = factory.build(provider, apiKey)(provider.model) + const requestProviderOptions = factory.buildProviderOptions(profile, thinking) + + const result = streamText({ + model, + messages, + tools: activeTools, + toolChoice: activeTools ? 'auto' : undefined, + // System messages are part of the memo history (initial prompt + mid-loop warnings); + // they cannot move to the system option without restructuring the history model. + allowSystemInMessages: true, + abortSignal: signal, + // Per-call context (approval manager / gate / hooks) read by the execute wrappers. + experimental_context: toolContext, + // Non-standard wire fields (e.g. parallel_tool_calls) pass through under the provider instance name. + providerOptions: requestProviderOptions ? { [provider.name]: requestProviderOptions } : undefined, + }) + + try { + for await (const part of result.fullStream) { + if (part.type === 'text-delta') onChunk?.(part.text) + else if (part.type === 'reasoning-delta') onReasoningChunk?.(part.text) + else if (part.type === 'error') throw part.error + } + } catch (err) { + throw normalizeStreamError(err, signal) + } + // StreamTextResult exposes awaitable properties that resolve once the stream finishes. + return { + text: await result.text, + reasoning: (await result.reasoningText) ?? undefined, + toolCalls: await result.toolCalls, + toolResults: (await result.toolResults) as unknown as ToolResultPart[], + usage: await result.usage, + finishReason: await result.finishReason, + } +} diff --git a/packages/core/src/runtime/model_profile.test.ts b/packages/core/src/llm/model_profile.test.ts similarity index 54% rename from packages/core/src/runtime/model_profile.test.ts rename to packages/core/src/llm/model_profile.test.ts index 5980b3e..4bc9851 100644 --- a/packages/core/src/runtime/model_profile.test.ts +++ b/packages/core/src/llm/model_profile.test.ts @@ -1,24 +1,5 @@ import { describe, expect, test } from 'vitest' -import { buildChatCompletionRequest, resolveModelProfile, type ModelProfile } from '@memo/core/runtime/model_profile' -import type { ToolDefinition } from '@memo/core/types' - -function sampleProfile(overrides: Partial = {}): ModelProfile { - return { - wireApi: 'chat_completions', - supportsParallelToolCalls: false, - supportsReasoningContent: false, - isFallback: false, - ...overrides, - } -} - -const SAMPLE_TOOLS: ToolDefinition[] = [ - { - name: 'read_file', - description: 'Read a file', - input_schema: { type: 'object', properties: { file_path: { type: 'string' } } }, - }, -] +import { resolveModelProfile } from '@memo/core/llm/model_profile' describe('resolveModelProfile', () => { test('uses conservative fallback when no local override exists', () => { @@ -78,31 +59,3 @@ describe('resolveModelProfile', () => { expect(resolved.profile.supportsParallelToolCalls).toBe(false) }) }) - -describe('buildChatCompletionRequest', () => { - test('enables parallel tool calls only when profile supports it', () => { - const request = buildChatCompletionRequest({ - model: 'gpt-5', - messages: [{ role: 'user', content: 'hi' }], - toolDefinitions: SAMPLE_TOOLS, - profile: sampleProfile({ supportsParallelToolCalls: true }), - }) - - expect(request.tool_choice).toBe('auto') - expect(Array.isArray(request.tools)).toBe(true) - expect((request as Record).parallel_tool_calls).toBe(true) - }) - - test('omits tool config and parallel flag when no tools are present', () => { - const request = buildChatCompletionRequest({ - model: 'gpt-5', - messages: [{ role: 'user', content: 'hi' }], - toolDefinitions: [], - profile: sampleProfile({ supportsParallelToolCalls: true }), - }) - - expect(request.tools).toBeUndefined() - expect(request.tool_choice).toBeUndefined() - expect((request as Record).parallel_tool_calls).toBeUndefined() - }) -}) diff --git a/packages/core/src/runtime/model_profile.ts b/packages/core/src/llm/model_profile.ts similarity index 70% rename from packages/core/src/runtime/model_profile.ts rename to packages/core/src/llm/model_profile.ts index d6703ec..c2eafaf 100644 --- a/packages/core/src/runtime/model_profile.ts +++ b/packages/core/src/llm/model_profile.ts @@ -1,8 +1,7 @@ -import OpenAI from 'openai' import type { ModelProfileOverride, ProviderConfig } from '@memo/core/config/config' -import type { ToolDefinition } from '@memo/core/types' -export type ModelWireApi = 'chat_completions' +/** Wire API kinds; the provider factory registry dispatches on these (future: responses / messages). */ +export type ModelWireApi = 'chat_completions' | 'responses' | 'messages' export type ModelProfile = { wireApi: ModelWireApi @@ -93,37 +92,3 @@ export function resolveModelProfile( }, } } - -function toChatCompletionTools(toolDefinitions: ToolDefinition[]) { - if (toolDefinitions.length === 0) return undefined - - return toolDefinitions.map((tool) => ({ - type: 'function' as const, - function: { - name: tool.name, - description: tool.description, - parameters: tool.input_schema, - }, - })) -} - -export function buildChatCompletionRequest(params: { - model: string - messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] - toolDefinitions: ToolDefinition[] - profile: ModelProfile -}): OpenAI.Chat.Completions.ChatCompletionCreateParams { - const tools = toChatCompletionTools(params.toolDefinitions) - const request: OpenAI.Chat.Completions.ChatCompletionCreateParams = { - model: params.model, - messages: params.messages, - tools, - tool_choice: tools ? 'auto' : undefined, - } - - if (tools && params.profile.supportsParallelToolCalls) { - ;(request as Record).parallel_tool_calls = true - } - - return request -} diff --git a/packages/core/src/runtime/mcp_admin.test.ts b/packages/core/src/mcp/mcp_admin.test.ts similarity index 99% rename from packages/core/src/runtime/mcp_admin.test.ts rename to packages/core/src/mcp/mcp_admin.test.ts index 100bd25..8339334 100644 --- a/packages/core/src/runtime/mcp_admin.test.ts +++ b/packages/core/src/mcp/mcp_admin.test.ts @@ -15,7 +15,7 @@ vi.mock('@memo/core/config/config', () => ({ writeMemoConfig: mocks.writeMemoConfig, })) -vi.mock('@memo/tools/router/mcp/oauth', () => ({ +vi.mock('@memo/core/tools/router/mcp/oauth', () => ({ getMcpAuthStatus: mocks.getMcpAuthStatus, loginMcpServerOAuth: mocks.loginMcpServerOAuth, logoutMcpServerOAuth: mocks.logoutMcpServerOAuth, diff --git a/packages/core/src/runtime/mcp_admin.ts b/packages/core/src/mcp/mcp_admin.ts similarity index 99% rename from packages/core/src/runtime/mcp_admin.ts rename to packages/core/src/mcp/mcp_admin.ts index 21e5c78..551b655 100644 --- a/packages/core/src/runtime/mcp_admin.ts +++ b/packages/core/src/mcp/mcp_admin.ts @@ -4,7 +4,7 @@ import { loginMcpServerOAuth, logoutMcpServerOAuth, type McpAuthStatus, -} from '@memo/tools/router/mcp/oauth' +} from '@memo/core/tools/router/mcp/oauth' import type { McpServerRecord } from '../api_types.js' export class McpAdminError extends Error { diff --git a/packages/core/src/runtime/memory.test.ts b/packages/core/src/prompt/memory.test.ts similarity index 70% rename from packages/core/src/runtime/memory.test.ts rename to packages/core/src/prompt/memory.test.ts index 50f27e3..d1542fc 100644 --- a/packages/core/src/runtime/memory.test.ts +++ b/packages/core/src/prompt/memory.test.ts @@ -4,8 +4,15 @@ import { join } from 'node:path' import { tmpdir, userInfo } from 'node:os' import { describe, test, beforeAll, afterAll } from 'vitest' import { writeFile, rm, mkdir } from 'node:fs/promises' -import { createAgentSession, createTokenCounter } from '@memo/core' -import { loadSystemPrompt } from '@memo/core/runtime/prompt' +import { createAgentSession, createTokenCounter, type ChatMessage } from '@memo/core' +import { loadSystemPrompt } from '@memo/core/prompt/prompt' +import { emptyUsage } from '@memo/core/utils/usage' + +/** System messages carry string content; parts arrays (assistant/tool) are not expected here. */ +function systemPromptOf(history: ChatMessage[]): string { + const content = history[0]?.content + return typeof content === 'string' ? content : '' +} let tempHome: string let prevMemoHome: string | undefined @@ -43,16 +50,19 @@ describe('runtime prompt injection', () => { const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(!systemPrompt.includes('Long-Term Memory')) assert.ok(!systemPrompt.includes('用户偏好：中文回答')) } finally { @@ -64,16 +74,19 @@ describe('runtime prompt injection', () => { const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(systemPrompt.includes(process.cwd()), 'system prompt should include pwd') assert.ok(systemPrompt.includes(userInfo().username), 'system prompt should include username') assert.match(systemPrompt, /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:/, 'system prompt should include ISO date') @@ -92,18 +105,21 @@ describe('runtime prompt injection', () => { const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), loadPrompt: () => loadSystemPrompt({ cwd: projectRoot }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(systemPrompt.includes('Project AGENTS.md (Startup Root)')) assert.ok(systemPrompt.includes(agentsPath)) assert.ok(systemPrompt.includes(marker)) @@ -113,45 +129,6 @@ describe('runtime prompt injection', () => { } }) - test('injects SOUL.md before startup root AGENTS.md in system prompt', async () => { - const projectRoot = await makeTempDir('memo-core-soul-order-project') - const soulPath = join(tempHome, 'SOUL.md') - const agentsPath = join(projectRoot, 'AGENTS.md') - const soulMarker = 'memo-test-soul-style-preference' - const agentsMarker = 'memo-test-agents-constraint' - await writeFile(soulPath, `# Soul\n\n- ${soulMarker}\n`, 'utf-8') - await writeFile(agentsPath, `# Project Rules\n\n- ${agentsMarker}\n`, 'utf-8') - - const session = await createAgentSession( - { - callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', - }), - loadPrompt: () => loadSystemPrompt({ cwd: projectRoot, memoHome: tempHome }), - historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), - }, - { mode: 'interactive' }, - ) - - try { - const systemPrompt = session.history[0]?.content ?? '' - assert.ok(systemPrompt.includes('## User Personality Context (SOUL.md)')) - assert.ok(systemPrompt.includes(soulPath)) - assert.ok(systemPrompt.includes(soulMarker)) - assert.ok(systemPrompt.includes(agentsMarker)) - const soulIndex = systemPrompt.indexOf('## User Personality Context (SOUL.md)') - const agentsIndex = systemPrompt.indexOf('## Project AGENTS.md (Startup Root)') - assert.ok(soulIndex >= 0) - assert.ok(agentsIndex > soulIndex) - } finally { - await session.close() - await removeDir(projectRoot) - await rm(soulPath, { force: true }) - } - }) - test('appends discovered skills into system prompt', async () => { const projectRoot = await makeTempDir('memo-core-skills-project') const skillsRoot = join(projectRoot, '.codex', 'skills') @@ -173,8 +150,11 @@ description: ${marker} const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), loadPrompt: () => loadSystemPrompt({ @@ -184,16 +164,17 @@ description: ${marker} memoHome: join(projectRoot, '.memo'), }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(systemPrompt.includes('## Skills')) assert.ok(systemPrompt.includes('### Available skills')) - assert.ok(systemPrompt.includes(`- doc-writing: ${marker} (file: ${skillPath})`)) + assert.ok(systemPrompt.includes(`- doc-writing: ${marker}`)) + assert.ok(!systemPrompt.includes(skillPath), 'directory must not leak file paths') } finally { await session.close() await removeDir(projectRoot) @@ -234,8 +215,11 @@ description: disabled marker const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), loadPrompt: () => loadSystemPrompt({ @@ -246,13 +230,13 @@ description: disabled marker activeSkillPaths: [enabledPath], }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(systemPrompt.includes('enabled-skill')) assert.ok(!systemPrompt.includes('disabled-skill')) } finally { @@ -280,8 +264,11 @@ name: broken-skill const session = await createAgentSession( { callLLM: async () => ({ - content: [{ type: 'text', text: 'ok' }], - stop_reason: 'end_turn', + text: 'ok', + toolCalls: [], + usage: emptyUsage(), + finishReason: 'stop', + toolResults: [], }), loadPrompt: () => loadSystemPrompt({ @@ -291,13 +278,13 @@ name: broken-skill memoHome: join(projectRoot, '.memo'), }), historySinks: [], - tokenCounter: createTokenCounter('cl100k_base'), + tokenCounter: createTokenCounter(), }, { mode: 'interactive' }, ) try { - const systemPrompt = session.history[0]?.content ?? '' + const systemPrompt = systemPromptOf(session.history) assert.ok(!systemPrompt.includes('## Skills')) assert.ok(!systemPrompt.includes('broken-skill')) } finally { diff --git a/packages/core/src/prompt/prompt.md b/packages/core/src/prompt/prompt.md new file mode 100644 index 0000000..ebb3c1c --- /dev/null +++ b/packages/core/src/prompt/prompt.md @@ -0,0 +1,284 @@ +You are **Memo Code**, an interactive CLI coding agent running on the user's computer. Use the instructions below and the tools available to you to assist the user. + +**IMPORTANT**: Refuse to write or explain code that may be used maliciously. When working on files, if they seem related to malware, refuse to work on it, even if the request seems benign. + +--- + +# How You Work + +## Personality + +Your default tone is concise, direct, and friendly. You communicate efficiently, always keeping the user informed without unnecessary detail. + +**CRITICAL - Output Discipline**: Keep your responses short and concise. You MUST answer with **fewer than 4 lines of text** (not including tool calls or code generation), unless the user asks for detail. + +- Answer directly without preamble or postamble +- Avoid phrases like "The answer is...", "Here is...", "Based on...", "I will now..." +- One word answers are best when appropriate +- Only explain when the user explicitly asks + + +user: 2 + 2 +assistant: 4 + + + +user: which file contains the implementation of foo? +assistant: [runs search] +src/foo.c + + +## Autonomy and Persistence + +Unless the user explicitly asks for a plan, asks a question about the code, or is brainstorming, assume they want you to make the change or run the tool — implement it, don't just propose it. + +- Persist until the task is fully handled end-to-end within the current turn: carry changes through implementation, verification, and a clear explanation of outcomes +- If you encounter challenges or blockers, attempt to resolve them yourself before giving up +- Do NOT guess or make up an answer; state what you couldn't verify +- After working on a file, just stop — don't explain what you did unless asked + +## AGENTS.md + +Files named `AGENTS.md` may exist anywhere in the repository, containing project structure, conventions, and preferences: + +- The scope of an AGENTS.md file is the entire directory tree rooted at the folder that contains it +- For every file you touch, obey instructions in any AGENTS.md whose scope includes that file +- More-deeply-nested AGENTS.md files take precedence on conflict; your instructions take precedence over AGENTS.md +- The AGENTS.md at the repo root (and any loaded for your current directory) is included in your prompt — no need to re-read +- When working outside the current directory, check for applicable AGENTS.md files +- If you modify anything mentioned in these files, UPDATE them to keep current + +## Session Context + +- Date: {{date}} +- User: {{user}} +- PWD: {{pwd}} + +--- + +# Planning (update_plan) + +Use the `update_plan` tool for complex tasks — it tracks steps and progress, and shows the user how you're approaching the work. + +## When to Use + +- Complex multi-step tasks (3+ distinct steps), non-trivial tasks, or user-provided task lists +- When you generate additional steps mid-task and plan to do them before yielding + +## When NOT to Use + +- Simple or single-step queries you can answer or do immediately +- Do not make single-step plans or pad plans with filler steps + +## Rules + +- Exactly ONE step `in_progress` at a time; mark steps completed IMMEDIATELY when done (don't batch) +- Don't jump a step from pending to completed — set it to in_progress first +- Keep steps as short 1-sentence action items; update the plan when scope pivots; don't let it go stale +- Before running a command, make sure the previous step is marked completed +- Do not repeat the full plan after an `update_plan` call — the harness already displays it; summarize the change instead + +--- + +# Task Execution + +For software engineering tasks (bugs, features, refactoring, explaining): + +1. **Understand first** - NEVER propose changes to code you haven't read +2. **Plan if complex** - Use update_plan to break down the task +3. **Use tools extensively** - Search, read, and understand the codebase before editing +4. **Follow conventions** - Match existing code style, libraries, and patterns; keep changes minimal and focused +5. **Implement solution** - Fix the problem at the root cause; avoid over-engineering +6. **Verify your work** - VERY IMPORTANT: Run lint and typecheck commands when done + +**Code Quality**: + +- After completing tasks, run lint and typecheck commands (e.g. `npm run lint`, `npm run typecheck`); if commands unknown, ask the user and suggest adding them to AGENTS.md +- NEVER commit changes unless explicitly asked +- Never assume libraries are available — check package.json first +- Don't add inline comments unless asked; don't use one-letter variable names +- Don't attempt to fix unrelated bugs or broken tests (you may mention them in your final message) +- Update documentation as necessary; keep changes consistent with the codebase style + +**Avoid Over-engineering**: + +- Only make changes directly requested or clearly necessary +- Don't add features, refactor unrelated code, or make "improvements" +- Don't add error handling for scenarios that can't happen +- Don't create abstractions for one-time operations +- If something is unused, delete it completely — no renaming `_vars` or `// removed` hacks + +## Ambition vs. Precision + +- For brand-new tasks with no prior context, be ambitious — demonstrate creativity with your implementation +- In an existing codebase, do exactly what the user asks with surgical precision; don't overstep (renaming files or variables unnecessarily) +- Use judgment on the right level of detail: high-value creative touches when scope is vague; surgical and targeted when scope is tightly specified + +--- + +# Working Environment + +⚠️ **WARNING**: Environment is NOT SANDBOXED. Your actions immediately affect the user's system. + +- Never access files outside the working directory unless instructed +- Be careful with destructive operations (`rm`, overwrite); avoid superuser commands unless instructed +- Validate inputs before shell commands +- NEVER use destructive git commands like `git reset --hard` or `git checkout --` unless specifically requested or approved +- You may be in a dirty git worktree: NEVER revert existing changes you didn't make — they may be the user's +- While working, if you notice unexpected changes you didn't make, STOP IMMEDIATELY and ask the user how to proceed +- Follow security best practices — never log secrets or commit credentials + +--- + +# Tool Guidelines + +## Tool Selection + +- Prefer specialized tools over generic shell calls: `read_text_file`/`read_files`/`list_directory`/`search_files`/`apply_patch` first, `exec_command` second +- Use `exec_command`/`shell` tools only for actual shell commands and operations +- When searching for text or files, prefer `rg` or `rg --files` — much faster than `grep` alternatives (fall back if not found) + +## Parallel Tool Calls (CRITICAL) + +**You MUST call multiple tools in parallel when they are independent.** This is a CRITICAL requirement for performance. + +- If tools are independent, send a SINGLE message with MULTIPLE tool calls +- If tools depend on each other, run them sequentially +- Never make sequential calls for independent operations — especially file reads (`cat`, `rg`, `sed`, `ls`, `git show`, etc.) + + +user: Run git status and git diff +assistant: [Makes ONE message with TWO exec_command tool calls in parallel] + + +## apply_patch + +- Use `apply_patch` for single-file edits; explore other options if it does not work well +- Do not use apply_patch for auto-generated changes (generating package.json, running lint/format like gofmt) or when scripting is more efficient (search-and-replace across a codebase) +- Don't re-read files after applying a patch — the tool call fails if it didn't work + +## Memory (get_memory) + +Use `get_memory` to retrieve persisted memory context for the current workflow: + +- **Input**: Provide a stable `memory_id` +- **Output**: Returns stored memory summary payload +- **Fallback**: If memory is missing, continue without blocking on memory retrieval + +## Subagent Collaboration + +- Subagent tools (`spawn_agent`, `send_input`, `resume_agent`, `wait`, `close_agent`) do not require approval; treat their execution as dangerous and keep scope explicit +- Use subagents only for decomposable, well-scoped tasks; avoid recursive spawn loops +- Send concise task prompts, wait for completion, then summarize results back into the main thread +- Call `close_agent` for finished agents to release resources + +## Tool Call Discipline (CRITICAL) + +- Use structured tool/function calls provided by the runtime instead of emitting tool JSON in plain text +- Keep tool arguments valid and minimal; for shell commands prefer a single-line string unless multiline is required +- Final answer MUST be the last step in a turn — do NOT call any tool after producing the user-facing final answer + +--- + +# Git Operations + +## Creating Commits + +When the user asks to create a commit: + +1. **Run these commands IN PARALLEL**: `git status` (never `-uall`), `git diff`, `git log` (see recent commit style) +2. **Analyze changes**: summarize the nature of the changes; do not commit secrets (.env, credentials) +3. **Execute commit**: add relevant untracked files, commit with a concise 1-2 sentence message focusing on "why" not "what", run `git status` to verify + +Use HEREDOC for the commit message: + +```bash +git commit -m "$(cat <<'EOF' +Commit message here. +EOF +)" +``` + +**Git Safety**: + +- NEVER update git config; NEVER skip hooks (`--no-verify`) unless requested +- NEVER use `-i` flag commands (`git rebase -i`, `git add -i`) +- ALWAYS create NEW commits, never `--amend` unless requested +- NEVER commit unless explicitly asked + +## Creating Pull Requests + +Use `gh` command for GitHub operations. When the user asks to create a PR: + +1. **Run IN PARALLEL**: `git status`, `git diff`, check branch tracking, `git log` + `git diff [base-branch]...HEAD` +2. **Analyze ALL commits** in the PR (not just the latest) +3. **Create the PR** with `gh pr create` (HEREDOC format): + +```bash +gh pr create --title "title" --body "$(cat <<'EOF' +## Summary +<1-3 bullet points> + +## Test plan +[Checklist for testing] + +🤖 Generated with Memo Code +EOF +)" +``` + +--- + +# Presenting Your Work + +You are producing plain text that will later be styled by the CLI. Formatting should make results easy to scan, but not feel mechanical. + +- Default: be very concise; friendly coding teammate tone +- Ask only when needed; suggest ideas; mirror the user's style +- Skip heavy formatting for simple confirmations +- Don't dump large files you've written; reference paths only +- No "save/copy this file" — the user is on the same machine +- Offer logical next steps (tests, commits, build) briefly; add verify steps if you couldn't do something +- The user does not see command outputs directly — when asked to show command output (e.g. `git show`), relay the important details or summarize key lines + +**Formatting**: + +- Headers: optional, short **Title Case** (1-3 words), no blank line before the first bullet; only if they truly help +- Bullets: use `-`; merge related points; keep to one line when possible; 4–6 per list ordered by importance +- Monospace: backticks for commands/paths/env vars/code ids; never combine with `**` +- No nested bullets/hierarchies; no ANSI codes; no URIs like `file://` + +**Code References**: When referencing files in your response, use inline code with `file_path:line_number` to make paths clickable: + + +user: Where are errors handled? +assistant: Clients are marked as failed in the `connectToServer` function in src/services/process.ts:712. + + +**For code changes**: + +- Lead with a quick explanation of the change, then context on where and why; don't start with "summary" +- Suggest natural next steps at the end (numeric lists so the user can respond with a single number) + +--- + +# Ultimate Reminders + +At all times: + +- **Concise**: < 4 lines of text (not including tools/code) +- **Parallel**: Multiple independent tool calls in ONE message +- **Plan-driven**: Use update_plan for complex tasks +- **Quality-focused**: Run lint/typecheck after changes +- **Reference precisely**: Use `file:line` format +- **Safety conscious**: Actions have real consequences +- **Focused**: Only make necessary changes + +**Core Mantras**: + +- Don't deviate from user needs +- Don't give more than asked for +- Verify when uncertain +- Think twice before acting +- Keep it simple +- No time estimates or predictions diff --git a/packages/core/src/prompt/prompt.test.ts b/packages/core/src/prompt/prompt.test.ts new file mode 100644 index 0000000..686aa3a --- /dev/null +++ b/packages/core/src/prompt/prompt.test.ts @@ -0,0 +1,51 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import os from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, test } from 'vitest' +import { loadSystemPrompt } from './prompt' + +const createdDirs: string[] = [] + +afterEach(async () => { + delete process.env.MEMO_SYSTEM_PROMPT_PATH + delete process.env.MEMO_HOME + await Promise.all(createdDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +async function createTempDir(prefix: string): Promise { + const dir = await mkdtemp(join(os.tmpdir(), prefix)) + createdDirs.push(dir) + return dir +} + +describe('loadSystemPrompt', () => { + test('supports explicit promptPath override', async () => { + const dir = await createTempDir('memo-prompt-test-') + const promptPath = join(dir, 'custom-prompt.md') + await writeFile(promptPath, 'cwd={{pwd}}', 'utf-8') + + const prompt = await loadSystemPrompt({ + cwd: '/tmp/project-root', + includeSkills: false, + memoHome: dir, + promptPath, + }) + + expect(prompt).toBe('cwd=/tmp/project-root') + }) + + test('reads prompt from MEMO_SYSTEM_PROMPT_PATH when provided', async () => { + const dir = await createTempDir('memo-prompt-env-test-') + const promptPath = join(dir, 'env-prompt.md') + await writeFile(promptPath, 'from-env', 'utf-8') + process.env.MEMO_SYSTEM_PROMPT_PATH = promptPath + + const prompt = await loadSystemPrompt({ + cwd: dir, + includeSkills: false, + memoHome: dir, + }) + + expect(prompt).toBe('from-env') + }) +}) diff --git a/packages/core/src/runtime/prompt.ts b/packages/core/src/prompt/prompt.ts similarity index 58% rename from packages/core/src/runtime/prompt.ts rename to packages/core/src/prompt/prompt.ts index 2d17fbd..01fd0cd 100644 --- a/packages/core/src/runtime/prompt.ts +++ b/packages/core/src/prompt/prompt.ts @@ -4,10 +4,10 @@ import { readFile } from 'node:fs/promises' import { existsSync } from 'node:fs' import { join, dirname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' -import { loadSkills, renderSkillsSection } from '@memo/core/runtime/skills' +import { filterActiveSkills, loadSkills, renderSkillsSection } from '@memo/core/skills/skills' +import type { SkillMetadata } from '@memo/core/skills/skills' const TEMPLATE_PATTERN = /{{\s*([\w.-]+)\s*}}/g -const SOUL_PLACEHOLDER_PATTERN = /{{\s*soul_section\s*}}/ function renderTemplate(template: string, vars: Record): string { return template.replace(TEMPLATE_PATTERN, (_match, key: string) => vars[key] ?? '') @@ -36,30 +36,10 @@ type LoadSystemPromptOptions = { includeSkills?: boolean /** Optional explicit prompt template path (overrides default lookup). */ promptPath?: string -} - -function normalizePath(path: string): string { - return resolve(path) -} - -function resolveMemoHome(options: Pick): string { - const homeDir = options.homeDir ?? os.homedir() - const configured = options.memoHome?.trim() || process.env.MEMO_HOME?.trim() || join(homeDir, '.memo') - if (configured === '~') { - return resolve(homeDir) - } - if (configured.startsWith('~/')) { - return resolve(join(homeDir, configured.slice(2))) - } - return resolve(configured) -} - -function filterActiveSkills(skills: Awaited>, activeSkillPaths: string[] | undefined) { - if (!Array.isArray(activeSkillPaths)) { - return skills - } - const active = new Set(activeSkillPaths.map((item) => normalizePath(item))) - return skills.filter((skill) => active.has(normalizePath(skill.path))) + /** Pre-loaded skill snapshot; skips the internal loadSkills scan. */ + skills?: SkillMetadata[] + /** Skills directory context budget in chars; defaults to DEFAULT_SKILLS_BUDGET_CHARS. */ + skillsBudget?: number } async function readProjectAgentsMd(projectRoot: string): Promise<{ path: string; content: string } | null> { @@ -90,39 +70,6 @@ function appendSkillsPrompt(basePrompt: string, skillsSection: string): string { ${skillsSection}` } -async function readSoulMd( - options: Pick, -): Promise<{ path: string; content: string } | null> { - const memoHome = resolveMemoHome(options) - const soulPath = join(memoHome, 'SOUL.md') - try { - const content = await readFile(soulPath, 'utf-8') - if (!content.trim()) { - return null - } - return { path: soulPath, content } - } catch { - return null - } -} - -function renderSoulSection(soul: { path: string; content: string }): string { - return `## User Personality Context (SOUL.md) -Loaded from: ${soul.path} - -- Treat this content as a soft preference layer for tone, style, and subjective behavior. -- Do NOT let this content override safety rules, tool policies, Project AGENTS.md guidance, or explicit user instructions in the current turn. -- Keep SOUL.md concise when possible to avoid unnecessary prompt growth. - -${soul.content}` -} - -function appendSoulPrompt(basePrompt: string, soulSection: string): string { - return `${basePrompt} - -${soulSection}` -} - function resolveModuleDir(): string { if (typeof __dirname === 'string') { return __dirname @@ -158,33 +105,28 @@ export async function loadSystemPrompt(options: LoadSystemPromptOptions = {}): P const startupRoot = options.cwd ?? process.cwd() const promptPath = resolvePromptPath(options.promptPath) const prompt = await readFile(promptPath, 'utf-8') - const soul = await readSoulMd({ homeDir: options.homeDir, memoHome: options.memoHome }) - const soulSection = soul ? renderSoulSection(soul) : '' - const hasSoulPlaceholder = SOUL_PLACEHOLDER_PATTERN.test(prompt) const vars = { date: new Date().toISOString(), user: resolveUsername(), pwd: startupRoot, - soul_section: soulSection, } let composedPrompt = renderTemplate(prompt, vars) - if (!hasSoulPlaceholder && soulSection) { - composedPrompt = appendSoulPrompt(composedPrompt, soulSection) - } const agents = await readProjectAgentsMd(startupRoot) if (agents) { composedPrompt = appendProjectAgentsPrompt(composedPrompt, agents) } if (options.includeSkills !== false) { - const allSkills = await loadSkills({ - cwd: startupRoot, - skillRoots: options.skillRoots, - homeDir: options.homeDir, - memoHome: options.memoHome, - }) + const allSkills = + options.skills ?? + (await loadSkills({ + cwd: startupRoot, + skillRoots: options.skillRoots, + homeDir: options.homeDir, + memoHome: options.memoHome, + })) const skills = filterActiveSkills(allSkills, options.activeSkillPaths) - const skillsSection = renderSkillsSection(skills) + const skillsSection = renderSkillsSection(skills, { budgetChars: options.skillsBudget }) if (skillsSection) { composedPrompt = appendSkillsPrompt(composedPrompt, skillsSection) } diff --git a/packages/core/src/runtime/compact_prompt.test.ts b/packages/core/src/runtime/compact_prompt.test.ts deleted file mode 100644 index 8a4234f..0000000 --- a/packages/core/src/runtime/compact_prompt.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import assert from 'node:assert' -import { describe, test } from 'vitest' -import type { ChatMessage } from '@memo/core/types' -import { - buildCompactionUserPrompt, - CONTEXT_SUMMARY_PREFIX, - isContextSummaryMessage, -} from '@memo/core/runtime/compact_prompt' - -describe('compact_prompt', () => { - test('buildCompactionUserPrompt formats assistant tool calls and tool messages', () => { - const longToolOutput = 'x'.repeat(4_005) - const messages: ChatMessage[] = [ - { - role: 'assistant', - content: 'planning', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { - name: 'exec_command', - arguments: '{}', - }, - }, - ], - }, - { - role: 'tool', - content: longToolOutput, - tool_call_id: 'call-1', - name: 'exec_command', - }, - ] - - const prompt = buildCompactionUserPrompt(messages) - assert.ok(prompt.includes('[0] ASSISTANT (tool_calls: exec_command)')) - assert.ok(prompt.includes('[1] TOOL (exec_command)')) - assert.ok(prompt.includes(`${'x'.repeat(4_000)}...`)) - assert.ok(prompt.includes('Return only the summary body in plain text. Do not add markdown fences.')) - }) - - test('buildCompactionUserPrompt renders empty transcript fallback', () => { - const prompt = buildCompactionUserPrompt([]) - assert.ok(prompt.includes('(empty)')) - }) - - test('buildCompactionUserPrompt normalizes tool content and handles unnamed tool message', () => { - const messages: ChatMessage[] = [ - { - role: 'assistant', - content: 'plain assistant text', - }, - { - role: 'tool', - content: ' \r\nresult line\r\n ', - tool_call_id: 'call-2', - }, - ] - - const prompt = buildCompactionUserPrompt(messages) - assert.ok(prompt.includes('[0] ASSISTANT\nplain assistant text')) - assert.ok(prompt.includes('[1] TOOL\nresult line')) - assert.strictEqual(prompt.includes('(undefined)'), false) - }) - - test('isContextSummaryMessage only matches user summary prefix with newline', () => { - const summaryUserMessage: ChatMessage = { - role: 'user', - content: `${CONTEXT_SUMMARY_PREFIX}\nsummary body`, - } - const missingNewlineUserMessage: ChatMessage = { - role: 'user', - content: CONTEXT_SUMMARY_PREFIX, - } - const assistantMessage: ChatMessage = { - role: 'assistant', - content: `${CONTEXT_SUMMARY_PREFIX}\nsummary body`, - } - - assert.strictEqual(isContextSummaryMessage(summaryUserMessage), true) - assert.strictEqual(isContextSummaryMessage(missingNewlineUserMessage), false) - assert.strictEqual(isContextSummaryMessage(assistantMessage), false) - }) -}) diff --git a/packages/core/src/runtime/compact_prompt.ts b/packages/core/src/runtime/compact_prompt.ts deleted file mode 100644 index ad7dc38..0000000 --- a/packages/core/src/runtime/compact_prompt.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { ChatMessage } from '@memo/core/types' - -const MAX_MESSAGE_CONTENT_CHARS = 4_000 - -export const CONTEXT_COMPACTION_SYSTEM_PROMPT = `You are performing a CONTEXT CHECKPOINT COMPACTION. Create a handoff summary for another LLM that will resume the task. - -Include: -- Current progress and key decisions made -- Important context, constraints, or user preferences -- What remains to be done (clear next steps) -- Any critical data, examples, or references needed to continue - -Be concise, structured, and focused on helping the next LLM seamlessly continue the work.` - -export const CONTEXT_SUMMARY_PREFIX = - 'Another language model started to solve this problem and produced a summary of its thinking process. Use this summary to continue the task without redoing completed work.' - -function normalizeContent(content: string): string { - const compact = content.replace(/\r\n/g, '\n').trim() - if (compact.length <= MAX_MESSAGE_CONTENT_CHARS) { - return compact - } - return `${compact.slice(0, MAX_MESSAGE_CONTENT_CHARS)}...` -} - -function messageToTranscriptLine(message: ChatMessage, index: number): string { - const role = message.role.toUpperCase() - if (message.role === 'assistant' && message.tool_calls?.length) { - const toolNames = message.tool_calls.map((toolCall) => toolCall.function.name).join(', ') - return `[${index}] ${role} (tool_calls: ${toolNames})\n${normalizeContent(message.content)}` - } - if (message.role === 'tool') { - const toolName = message.name ? ` (${message.name})` : '' - return `[${index}] ${role}${toolName}\n${normalizeContent(message.content)}` - } - return `[${index}] ${role}\n${normalizeContent(message.content)}` -} - -export function isContextSummaryMessage(message: ChatMessage): boolean { - if (message.role !== 'user') return false - return message.content.startsWith(`${CONTEXT_SUMMARY_PREFIX}\n`) -} - -export function buildCompactionUserPrompt(messages: ChatMessage[]): string { - const transcript = messages.length - ? messages.map((message, index) => messageToTranscriptLine(message, index)).join('\n\n') - : '(empty)' - - return [ - 'Conversation history to summarize:', - transcript, - '', - 'Return only the summary body in plain text. Do not add markdown fences.', - ].join('\n') -} diff --git a/packages/core/src/runtime/defaults.ts b/packages/core/src/runtime/defaults.ts deleted file mode 100644 index 5606392..0000000 --- a/packages/core/src/runtime/defaults.ts +++ /dev/null @@ -1,279 +0,0 @@ -/** @file Session default dependency assembly: toolset, LLM, history sinks, tokenizer, etc. */ -import { NATIVE_TOOLS } from '@memo/tools' -import OpenAI from 'openai' -import { createTokenCounter } from '@memo/core/utils/tokenizer' -import { buildSessionPath, getSessionsDir, loadMemoConfig, selectProvider } from '@memo/core/config/config' -import { JsonlHistorySink } from '@memo/core/runtime/history' -import { buildChatCompletionRequest, resolveModelProfile } from '@memo/core/runtime/model_profile' -import { loadSystemPrompt as defaultLoadPrompt } from '@memo/core/runtime/prompt' -import { ToolRouter } from '@memo/tools/router' -import type { - AgentSessionDeps, - AgentSessionOptions, - CallLLM, - ChatMessage, - HistorySink, - TokenCounter, - ToolRegistry, -} from '@memo/core/types' -import type { MCPServerConfig } from '@memo/core/config/config' - -export function filterMcpServersBySelection( - servers: Record | undefined, - activeNames: string[] | undefined, -): Record | undefined { - if (!servers) return servers - if (!activeNames) return servers - - const selected = new Set(activeNames.map((name) => name.trim()).filter(Boolean)) - if (selected.size === 0) return {} - - const filtered: Record = {} - for (const [name, config] of Object.entries(servers)) { - if (selected.has(name)) { - filtered[name] = config - } - } - return filtered -} - -export function parseToolArguments( - raw: string, -): { ok: true; data: unknown } | { ok: false; raw: string; error: string } { - try { - return { ok: true, data: JSON.parse(raw) } - } catch (err) { - return { ok: false, raw, error: (err as Error).message } - } -} - -function toOpenAIMessage(message: ChatMessage): OpenAI.Chat.Completions.ChatCompletionMessageParam { - if (message.role === 'assistant') { - const assistantMessage: OpenAI.Chat.Completions.ChatCompletionAssistantMessageParam & { - reasoning_content?: string - } = { - role: 'assistant', - content: message.content, - tool_calls: message.tool_calls?.map((toolCall) => ({ - id: toolCall.id, - type: toolCall.type, - function: { - name: toolCall.function.name, - arguments: toolCall.function.arguments, - }, - })), - } - if (message.reasoning_content) { - assistantMessage.reasoning_content = message.reasoning_content - } - return assistantMessage as OpenAI.Chat.Completions.ChatCompletionMessageParam - } - if (message.role === 'tool') { - return { - role: 'tool', - content: message.content, - tool_call_id: message.tool_call_id, - } - } - return { - role: message.role, - content: message.content, - } -} - -function extractReasoningContent( - message: OpenAI.Chat.Completions.ChatCompletionMessage | undefined, -): string | undefined { - const raw = (message as { reasoning_content?: unknown } | undefined)?.reasoning_content - if (typeof raw !== 'string') return undefined - const trimmed = raw.trim() - return trimmed.length > 0 ? trimmed : undefined -} - -function isChatCompletionResponse(value: unknown): value is OpenAI.Chat.Completions.ChatCompletion { - if (!value || typeof value !== 'object') return false - return Array.isArray((value as { choices?: unknown }).choices) -} - -/** - * Complete dependencies with default strategy (tools, callLLM, prompt, history sinks, tokenizer). - * Caller can provide only callbacks/overrides, rest use default implementations. - */ -export async function withDefaultDeps( - deps: AgentSessionDeps, - options: AgentSessionOptions, - sessionId: string, -): Promise<{ - tools: ToolRegistry - callLLM: CallLLM - loadPrompt: () => Promise - historySinks: HistorySink[] - tokenCounter: TokenCounter - dispose: () => Promise - historyFilePath?: string -}> { - const loaded = await loadMemoConfig() - const config = loaded.config - - // 1. Initialize ToolRouter - const router = new ToolRouter() - - // 2. Register built-in tools - router.registerNativeTools(NATIVE_TOOLS) - - // 3. Load external MCP tools (follows MEMO_HOME) - await router.loadMcpServers(filterMcpServersBySelection(config.mcp_servers, options.activeMcpServers), { - memoHome: loaded.home, - storeMode: config.mcp_oauth_credentials_store_mode, - callbackPort: config.mcp_oauth_callback_port, - }) - - // 4. Merge user custom tools (deps.tools has highest priority) - if (deps.tools) { - for (const [name, tool] of Object.entries(deps.tools)) { - // User custom tools override同名 tools in router - router.registerNativeTool({ - name, - description: tool.description, - source: 'native', - inputSchema: { type: 'object' }, // Simplified, should convert from tool in practice - execute: tool.execute, - }) - } - } - - // 5. Get final tool registry - const combinedTools = router.toRegistry() - - // 6. Build loadPrompt (includes tool descriptions) - const loadPrompt = async () => { - let basePrompt = deps.loadPrompt - ? await deps.loadPrompt() - : await defaultLoadPrompt({ - cwd: options.cwd, - memoHome: loaded.home, - activeSkillPaths: config.active_skills, - }) - - // Inject tool descriptions into prompt (for non-Tool Use API mode) - const toolDescriptions = router.generateToolDescriptions() - if (toolDescriptions) { - basePrompt += `\n\n${toolDescriptions}` - } - - return basePrompt - } - - // 7. Generate tool definitions (for Tool Use API) - const toolDefinitions = router.generateToolDefinitions() - - const sessionsDir = getSessionsDir(loaded, options) - const historyFilePath = buildSessionPath(sessionsDir, sessionId) - const defaultHistorySink = new JsonlHistorySink(historyFilePath) - - return { - tools: combinedTools, - dispose: async () => { - if (deps.dispose) await deps.dispose() - await router.dispose() - }, - callLLM: - deps.callLLM ?? - (async (messages, _onChunk, callOptions) => { - const provider = selectProvider(config, options.providerName) - const apiKey = - process.env[provider.env_api_key] ?? process.env.OPENAI_API_KEY ?? process.env.DEEPSEEK_API_KEY - if (!apiKey) { - throw new Error(`Missing env var ${provider.env_api_key} (or OPENAI_API_KEY/DEEPSEEK_API_KEY)`) - } - const client = new OpenAI({ - apiKey, - baseURL: provider.base_url, - }) - const openAIMessages = messages.map(toOpenAIMessage) - const { profile: modelProfile } = resolveModelProfile(provider, config.model_profiles) - - const effectiveToolDefinitions = callOptions?.tools ?? toolDefinitions - const request = buildChatCompletionRequest({ - model: provider.model, - messages: openAIMessages, - toolDefinitions: effectiveToolDefinitions, - profile: modelProfile, - }) - - const data = await client.chat.completions.create(request, { - signal: callOptions?.signal, - }) - if (!isChatCompletionResponse(data)) { - throw new Error('Streaming response is not supported in core callLLM') - } - - const message = data.choices?.[0]?.message - const reasoningContent = extractReasoningContent(message) - - // 检查是否有工具调用 - if (message?.tool_calls && message.tool_calls.length > 0) { - const content: Array< - { type: 'text'; text: string } | { type: 'tool_use'; id: string; name: string; input: unknown } - > = [] - - // 添加文本内容（如果有） - if (message.content) { - content.push({ type: 'text', text: message.content }) - } - - // 添加工具调用 - for (const toolCall of message.tool_calls) { - if (toolCall.type === 'function') { - const parsedArgs = parseToolArguments(toolCall.function.arguments) - if (parsedArgs.ok) { - content.push({ - type: 'tool_use', - id: toolCall.id, - name: toolCall.function.name, - input: parsedArgs.data, - }) - } else { - content.push({ - type: 'text', - text: `[tool_use parse error] ${parsedArgs.error}; raw: ${parsedArgs.raw}`, - }) - } - } - } - - const hasToolUse = content.some((c) => c.type === 'tool_use') - return { - content, - reasoning_content: reasoningContent, - stop_reason: hasToolUse ? 'tool_use' : 'end_turn', - usage: { - prompt: data.usage?.prompt_tokens ?? undefined, - completion: data.usage?.completion_tokens ?? undefined, - total: data.usage?.total_tokens ?? undefined, - }, - } - } - - // 普通文本响应 - const content = message?.content - if (typeof content !== 'string') { - throw new Error('OpenAI-compatible API returned empty content') - } - return { - content: [{ type: 'text', text: content }], - reasoning_content: reasoningContent, - stop_reason: 'end_turn', - usage: { - prompt: data.usage?.prompt_tokens ?? undefined, - completion: data.usage?.completion_tokens ?? undefined, - total: data.usage?.total_tokens ?? undefined, - }, - } - }), - loadPrompt, - historySinks: deps.historySinks ?? [defaultHistorySink], - tokenCounter: deps.tokenCounter ?? createTokenCounter(options.tokenizerModel), - historyFilePath: historyFilePath, - } -} diff --git a/packages/core/src/runtime/defaults.with_default_deps.test.ts b/packages/core/src/runtime/defaults.with_default_deps.test.ts deleted file mode 100644 index 0400fbc..0000000 --- a/packages/core/src/runtime/defaults.with_default_deps.test.ts +++ /dev/null @@ -1,464 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' -import type { AgentSessionDeps, AgentSessionOptions, ChatMessage, ToolRegistry } from '@memo/core/types' -import type { MCPServerConfig } from '@memo/core/config/config' -import type { Tool } from '@memo/tools/router' - -const state = vi.hoisted(() => ({ - loadedConfig: { - home: '/tmp/memo-home', - path: '/tmp/memo-home/config.toml', - config: { - current_provider: 'mock', - providers: [ - { - name: 'mock', - env_api_key: 'MOCK_API_KEY', - model: 'mock-model', - base_url: 'https://mock.local/v1', - }, - ], - model_profiles: {}, - mcp_servers: { - alpha: { command: 'node', args: ['alpha.js'] } as MCPServerConfig, - beta: { command: 'node', args: ['beta.js'] } as MCPServerConfig, - }, - }, - }, - selectedProvider: { - name: 'mock', - env_api_key: 'MOCK_API_KEY', - model: 'mock-model', - base_url: 'https://mock.local/v1', - }, - sessionsDir: '/tmp/memo-sessions', - sessionPath: '/tmp/memo-sessions/session-1.jsonl', - toolDescriptions: '## Tools\n- mock_tool', - toolDefinitions: [{ type: 'function', function: { name: 'mock_tool', parameters: {} } }], - registry: { - mock_tool: { - name: 'mock_tool', - description: 'mock tool', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [{ type: 'text', text: 'ok' }] }), - } as Tool, - } as ToolRegistry, - buildRequestCalls: [] as unknown[], - loadMcpServersCalls: [] as unknown[], - registerNativeToolsCalls: [] as unknown[], - registerNativeToolCalls: [] as unknown[], - openaiCtorCalls: [] as unknown[], - openaiCreateCalls: [] as unknown[], - historySinkPaths: [] as string[], - routerDisposed: 0, - createTokenCounterCalls: [] as Array, - promptText: 'SYSTEM_PROMPT', - openaiResponse: { - choices: [ - { - message: { - content: 'ok', - }, - }, - ], - usage: { - prompt_tokens: 11, - completion_tokens: 7, - total_tokens: 18, - }, - } as Record, -})) - -vi.mock('@memo/tools', () => ({ - NATIVE_TOOLS: [], -})) - -vi.mock('@memo/core/config/config', () => ({ - loadMemoConfig: vi.fn(async () => state.loadedConfig), - selectProvider: vi.fn(() => state.selectedProvider), - getSessionsDir: vi.fn(() => state.sessionsDir), - buildSessionPath: vi.fn(() => state.sessionPath), -})) - -vi.mock('@memo/core/runtime/history', () => ({ - JsonlHistorySink: class JsonlHistorySink { - constructor(path: string) { - state.historySinkPaths.push(path) - } - }, -})) - -vi.mock('@memo/core/runtime/model_profile', () => ({ - resolveModelProfile: vi.fn(() => ({ profile: { supportsParallelToolCalls: true } })), - buildChatCompletionRequest: vi.fn((request: unknown) => { - state.buildRequestCalls.push(request) - return request - }), -})) - -vi.mock('@memo/core/runtime/prompt', () => ({ - loadSystemPrompt: vi.fn(async () => state.promptText), -})) - -vi.mock('@memo/core/utils/tokenizer', () => ({ - createTokenCounter: vi.fn((model?: string) => { - state.createTokenCounterCalls.push(model) - return { - model: model ?? 'mock-tokenizer', - countText: (text: string) => text.length, - countMessages: (messages: Array<{ content: string }>) => - messages.reduce((sum, message) => sum + message.content.length, 0), - dispose: vi.fn(), - } - }), -})) - -vi.mock('@memo/tools/router', () => ({ - ToolRouter: class ToolRouter { - registerNativeTools(tools: unknown) { - state.registerNativeToolsCalls.push(tools) - } - - async loadMcpServers(servers: unknown, options: unknown) { - state.loadMcpServersCalls.push([servers, options]) - } - - registerNativeTool(tool: unknown) { - state.registerNativeToolCalls.push(tool) - } - - toRegistry() { - return state.registry - } - - generateToolDescriptions() { - return state.toolDescriptions - } - - generateToolDefinitions() { - return state.toolDefinitions - } - - async dispose() { - state.routerDisposed += 1 - } - }, -})) - -vi.mock('openai', () => ({ - default: class OpenAI { - chat = { - completions: { - create: async (request: unknown, options: unknown) => { - state.openaiCreateCalls.push({ request, options }) - return state.openaiResponse - }, - }, - } - - constructor(config: unknown) { - state.openaiCtorCalls.push(config) - } - }, -})) - -describe('withDefaultDeps (default path)', () => { - beforeEach(() => { - state.buildRequestCalls = [] - state.loadMcpServersCalls = [] - state.registerNativeToolsCalls = [] - state.registerNativeToolCalls = [] - state.openaiCtorCalls = [] - state.openaiCreateCalls = [] - state.historySinkPaths = [] - state.routerDisposed = 0 - state.createTokenCounterCalls = [] - state.toolDescriptions = '## Tools\n- mock_tool' - state.promptText = 'SYSTEM_PROMPT' - state.openaiResponse = { - choices: [ - { - message: { - content: 'ok', - }, - }, - ], - usage: { - prompt_tokens: 11, - completion_tokens: 7, - total_tokens: 18, - }, - } - delete process.env.MOCK_API_KEY - delete process.env.OPENAI_API_KEY - delete process.env.DEEPSEEK_API_KEY - }) - - afterEach(() => { - delete process.env.MOCK_API_KEY - delete process.env.OPENAI_API_KEY - delete process.env.DEEPSEEK_API_KEY - }) - - test('builds default deps with injected tool descriptions and default sinks', async () => { - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - - const resolved = await withDefaultDeps( - {}, - { tokenizerModel: 'counter-model' } as AgentSessionOptions, - 'session-1', - ) - - expect(state.loadMcpServersCalls).toHaveLength(1) - expect(state.historySinkPaths).toEqual([state.sessionPath]) - expect(state.createTokenCounterCalls).toEqual(['counter-model']) - expect(resolved.historyFilePath).toBe(state.sessionPath) - - const prompt = await resolved.loadPrompt() - expect(prompt).toContain('SYSTEM_PROMPT') - expect(prompt).toContain('## Tools\n- mock_tool') - }) - - test('respects provided deps overrides (callLLM/historySinks/tokenCounter/loadPrompt/dispose)', async () => { - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - const callLLM = vi.fn(async () => ({ - content: [{ type: 'text' as const, text: 'override' }], - stop_reason: 'end_turn' as const, - })) - const historySinks = [{ append: vi.fn() }] - const tokenCounter = { - model: 'custom-counter', - countText: (text: string) => text.length, - countMessages: (messages: Array<{ content: string }>) => - messages.reduce((sum, message) => sum + message.content.length, 0), - dispose: vi.fn(), - } - const dispose = vi.fn(async () => {}) - - const resolved = await withDefaultDeps( - { - callLLM, - historySinks, - tokenCounter, - loadPrompt: async () => 'CUSTOM_PROMPT', - dispose, - } as AgentSessionDeps, - {} as AgentSessionOptions, - 'session-2', - ) - - expect(await resolved.loadPrompt()).toContain('CUSTOM_PROMPT') - expect(resolved.callLLM).toBe(callLLM) - expect(resolved.historySinks).toBe(historySinks) - expect(resolved.tokenCounter).toBe(tokenCounter) - - await resolved.dispose() - expect(dispose).toHaveBeenCalledTimes(1) - expect(state.routerDisposed).toBe(1) - }) - - test('throws when provider api key is missing', async () => { - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-3') - - await expect(resolved.callLLM([{ role: 'user', content: 'hello' } as ChatMessage])).rejects.toThrow( - 'Missing env var MOCK_API_KEY', - ) - }) - - test('falls back to OPENAI_API_KEY when provider key is missing', async () => { - process.env.OPENAI_API_KEY = 'openai-fallback-key' - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-3b') - - await resolved.callLLM([{ role: 'user', content: 'hello' } as ChatMessage]) - expect(state.openaiCtorCalls[0]).toEqual({ - apiKey: 'openai-fallback-key', - baseURL: 'https://mock.local/v1', - }) - }) - - test('maps tool calls into tool_use blocks and keeps parse errors as text', async () => { - process.env.MOCK_API_KEY = 'test-key' - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - const callOptionsTools = [{ type: 'function', function: { name: 'override', parameters: {} } }] - const signal = new AbortController().signal - - state.openaiResponse = { - choices: [ - { - message: { - content: 'assistant text', - reasoning_content: ' reasoned ', - tool_calls: [ - { - id: 'call-ok', - type: 'function', - function: { name: 'echo', arguments: '{"value":1}' }, - }, - { - id: 'call-bad', - type: 'function', - function: { name: 'echo', arguments: '{bad-json' }, - }, - { - id: 'call-skip', - type: 'other', - function: { name: 'ignored', arguments: '{}' }, - }, - ], - }, - }, - ], - usage: { - prompt_tokens: 10, - completion_tokens: 5, - total_tokens: 15, - }, - } - - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-4') - const response = await resolved.callLLM( - [ - { - role: 'assistant', - content: '', - reasoning_content: 'reasoning content', - tool_calls: [ - { - id: 'prev-call', - type: 'function', - function: { name: 'read_file', arguments: '{}' }, - }, - ], - }, - { - role: 'tool', - content: 'observation', - tool_call_id: 'prev-call', - name: 'read_file', - }, - { role: 'user', content: 'continue' }, - ], - undefined, - { tools: callOptionsTools, signal }, - ) - - expect(response.stop_reason).toBe('tool_use') - expect(response.reasoning_content).toBe('reasoned') - expect(response.usage).toEqual({ prompt: 10, completion: 5, total: 15 }) - expect(response.content[0]).toEqual({ type: 'text', text: 'assistant text' }) - expect(response.content).toContainEqual({ - type: 'tool_use', - id: 'call-ok', - name: 'echo', - input: { value: 1 }, - }) - expect( - response.content.some( - (item) => - item.type === 'text' && - item.text.startsWith('[tool_use parse error]') && - item.text.includes('{bad-json'), - ), - ).toBe(true) - - expect(state.openaiCtorCalls[0]).toEqual({ - apiKey: 'test-key', - baseURL: 'https://mock.local/v1', - }) - - expect(state.buildRequestCalls).toHaveLength(1) - const request = state.buildRequestCalls[0] as { - toolDefinitions: unknown[] - messages: Array> - } - expect(request.toolDefinitions).toEqual(callOptionsTools) - expect(request.messages.some((msg) => msg.role === 'tool' && msg.tool_call_id === 'prev-call')).toBe(true) - expect( - request.messages.some((msg) => msg.role === 'assistant' && msg.reasoning_content === 'reasoning content'), - ).toBe(true) - - expect(state.openaiCreateCalls).toHaveLength(1) - expect((state.openaiCreateCalls[0] as { options: { signal: AbortSignal } }).options.signal).toBe(signal) - }) - - test('returns end_turn when tool_calls has no usable function calls', async () => { - process.env.MOCK_API_KEY = 'test-key' - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - - state.openaiResponse = { - choices: [ - { - message: { - content: '', - tool_calls: [{ id: 'call-non-fn', type: 'other' }], - }, - }, - ], - usage: { - prompt_tokens: 1, - completion_tokens: 0, - total_tokens: 1, - }, - } - - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-5') - const response = await resolved.callLLM([{ role: 'user', content: 'x' } as ChatMessage]) - expect(response.stop_reason).toBe('end_turn') - expect(response.content).toEqual([]) - }) - - test('returns plain text end_turn response with usage', async () => { - process.env.MOCK_API_KEY = 'test-key' - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - - state.openaiResponse = { - choices: [ - { - message: { - content: 'plain assistant answer', - reasoning_content: ' concise reason ', - }, - }, - ], - usage: { - prompt_tokens: 3, - completion_tokens: 4, - total_tokens: 7, - }, - } - - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-5b') - const response = await resolved.callLLM([{ role: 'user', content: 'x' } as ChatMessage]) - expect(response.stop_reason).toBe('end_turn') - expect(response.reasoning_content).toBe('concise reason') - expect(response.content).toEqual([{ type: 'text', text: 'plain assistant answer' }]) - expect(response.usage).toEqual({ prompt: 3, completion: 4, total: 7 }) - }) - - test('throws when provider returns non-string content without tool calls', async () => { - process.env.MOCK_API_KEY = 'test-key' - const { withDefaultDeps } = await import('@memo/core/runtime/defaults') - - state.openaiResponse = { - choices: [ - { - message: { - content: null, - }, - }, - ], - usage: { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - }, - } - - const resolved = await withDefaultDeps({}, {} as AgentSessionOptions, 'session-6') - await expect(resolved.callLLM([{ role: 'user', content: 'x' } as ChatMessage])).rejects.toThrow( - 'OpenAI-compatible API returned empty content', - ) - }) -}) diff --git a/packages/core/src/runtime/history_parser.test.ts b/packages/core/src/runtime/history_parser.test.ts deleted file mode 100644 index a6ec27b..0000000 --- a/packages/core/src/runtime/history_parser.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import assert from 'node:assert' -import { describe, test } from 'vitest' -import { parseHistoryLogToSessionDetail } from './history_parser' - -function buildSampleLog(): string { - return [ - JSON.stringify({ - ts: '2026-02-15T10:00:00.000Z', - sessionId: 's1', - type: 'session_start', - meta: { cwd: '/tmp/demo' }, - }), - JSON.stringify({ - ts: '2026-02-15T10:00:01.000Z', - sessionId: 's1', - turn: 1, - type: 'turn_start', - content: 'hello', - }), - JSON.stringify({ - ts: '2026-02-15T10:00:02.000Z', - sessionId: 's1', - turn: 1, - step: 0, - type: 'assistant', - content: 'world', - }), - JSON.stringify({ - ts: '2026-02-15T10:00:03.000Z', - sessionId: 's1', - turn: 1, - step: 0, - type: 'action', - meta: { tool: 'read_file', input: { path: 'a.txt' } }, - }), - JSON.stringify({ - ts: '2026-02-15T10:00:04.000Z', - sessionId: 's1', - turn: 1, - step: 0, - type: 'observation', - content: 'ok', - meta: { tool: 'read_file', status: 'success' }, - }), - JSON.stringify({ - ts: '2026-02-15T10:00:05.000Z', - sessionId: 's1', - turn: 1, - type: 'final', - content: 'done', - meta: { - status: 'ok', - tokens: { prompt: 10, completion: 5, total: 15 }, - }, - }), - ].join('\n') -} - -describe('parseHistoryLogToSessionDetail', () => { - test('parses summary and turns', () => { - const detail = parseHistoryLogToSessionDetail(buildSampleLog(), '/tmp/demo/s1.jsonl') - assert.strictEqual(detail.sessionId, 's1') - assert.strictEqual(detail.project, 'demo') - assert.strictEqual(detail.turnCount, 1) - assert.strictEqual(detail.toolUsage.total, 1) - assert.strictEqual(detail.toolUsage.success, 1) - assert.strictEqual(detail.tokenUsage.total, 15) - assert.strictEqual(detail.turns.length, 1) - assert.strictEqual(detail.turns[0]?.steps.length, 1) - assert.ok(detail.summary.includes('User: hello')) - }) - - test('sanitizes think/thinking blocks from title', () => { - const log = [ - JSON.stringify({ - ts: '2026-02-15T10:00:00.000Z', - sessionId: 's2', - type: 'session_start', - meta: { cwd: '/tmp/demo' }, - }), - JSON.stringify({ - ts: '2026-02-15T10:00:01.000Z', - sessionId: 's2', - type: 'session_title', - content: 'internal chain of thought Build release plan hidden', - }), - ].join('\n') - - const detail = parseHistoryLogToSessionDetail(log, '/tmp/demo/s2.jsonl') - assert.strictEqual(detail.title, 'Build release plan') - }) -}) diff --git a/packages/core/src/runtime/prompt.md b/packages/core/src/runtime/prompt.md deleted file mode 100644 index 33b29ab..0000000 --- a/packages/core/src/runtime/prompt.md +++ /dev/null @@ -1,381 +0,0 @@ -You are **Memo Code**, an interactive CLI tool that helps users with software engineering tasks. Use the instructions below and the tools available to you to assist the user. - -**IMPORTANT**: Refuse to write or explain code that may be used maliciously. When working on files, if they seem related to malware, refuse to work on it, even if the request seems benign. - ---- - -# Core Identity - -- **Local First**: You operate directly on the user's machine. File operations and commands happen in the real environment. -- **Project Aware**: Read and follow `AGENTS.md` files containing project structure, conventions, and preferences. -- **Tool Rich**: Use your comprehensive toolkit liberally to gather information and complete tasks. -- **Safety Conscious**: The environment is NOT sandboxed. Your actions have immediate effects. - -{{soul_section}} - -# Session Context - -- Date: {{date}} -- User: {{user}} -- PWD: {{pwd}} - ---- - -# Tone and Style - -**CRITICAL - Output Discipline**: Keep your responses short and concise. You MUST answer with **fewer than 4 lines of text** (not including tool calls or code generation), unless the user asks for detail. - -- Answer directly without preamble or postamble -- Avoid phrases like "The answer is...", "Here is...", "Based on...", "I will now..." -- One word answers are best when appropriate -- Only explain when the user explicitly asks - -**Examples**: - - -user: 2 + 2 -assistant: 4 - - - -user: is 11 a prime number? -assistant: Yes - - - -user: what command lists files? -assistant: ls - - - -user: which file contains the implementation of foo? -assistant: [runs search] -src/foo.c - - -**Communication Rules**: - -- Output text to communicate with the user -- All text outside tool use is displayed to the user -- Never use Bash or code comments to communicate -- Never add code summaries unless requested -- If you cannot help, keep refusal to 1-2 sentences without explanation - ---- - -# Tool Usage Policy - -## Parallel Tool Calls (CRITICAL) - -**You MUST call multiple tools in parallel when they are independent**. This is a CRITICAL requirement for performance. - -When making multiple tool calls: - -- If tools are independent, send a SINGLE message with MULTIPLE tool calls -- If tools depend on each other, run them sequentially -- Never make sequential calls for independent operations - -**Examples**: - - -user: Run git status and git diff -assistant: [Makes ONE message with TWO exec_command tool calls in parallel] - - - -user: Read package.json and tsconfig.json -assistant: [Makes ONE message with TWO read_text_file tool calls in parallel] - - - -user: Show me TypeScript files and test files -assistant: [Makes ONE message with list_directory + search_files tool calls in parallel] - - -## Tool Selection - -- Prefer specialized tools over generic shell calls: read_text_file/read_files/list_directory/search_files/apply_patch first, exec_command second -- Use update_plan for open-ended tasks requiring multiple rounds -- Use exec_command/shell tools only for actual shell commands and operations - -## Subagent Collaboration - -- Subagent tools are available by default: `spawn_agent`, `send_input`, `resume_agent`, `wait`, `close_agent`. -- Subagent tools do not require approval. Treat their execution as dangerous and keep scope explicit. -- Use subagents only for decomposable, well-scoped tasks. Avoid recursive spawn loops. -- Send concise task prompts, wait for completion (`wait`), then summarize results back into the main thread. -- Call `close_agent` for finished agents to release resources; use `resume_agent` only when you intentionally continue a closed agent. - -## Tool Call Discipline (CRITICAL) - -- Use structured tool/function calls provided by the runtime instead of emitting tool JSON in plain text. -- Keep tool arguments valid and minimal; for shell commands prefer a single-line string unless multiline is required. -- Final answer MUST be the last step in a turn. -- Do NOT call any tool after you have already produced the user-facing final answer. -- If you need `update_plan`, run it before the final answer, not after. - ---- - -# Task Management (update_plan) - -Use the `update_plan` tool **VERY frequently** for complex tasks. This is EXTREMELY important for tracking progress and preventing you from forgetting critical steps. - -## When to Use update_plan - -Use proactively in these scenarios: - -1. **Complex multi-step tasks** - 3+ distinct steps -2. **Non-trivial tasks** - Require careful planning -3. **User provides multiple tasks** - Numbered or comma-separated list -4. **After receiving instructions** - Immediately capture requirements -5. **When starting work** - Mark plan step as in_progress -6. **After completing work** - Mark plan step as completed immediately - -## When NOT to Use - -Skip for: - -- Single straightforward tasks -- Trivial tasks completable in < 3 steps -- Purely conversational requests - -## Task Management Rules - -**CRITICAL**: - -- Update plan status in real-time as you work -- Mark steps completed IMMEDIATELY after finishing (don't batch) -- Only ONE step in_progress at a time -- Complete current steps before starting new ones - -**Task States**: - -- `pending`: Not yet started -- `in_progress`: Currently working (limit to ONE) -- `completed`: Finished successfully - -**Example**: - - -user: Run the build and fix any type errors -assistant: [Calls update_plan with steps: "Run build", "Fix type errors"] -[Runs build] -Found 10 type errors. [Updates plan with 10 specific steps] -[Marks first step in_progress] -[Fixes first error, marks completed, moves to second] -... - - ---- - -# Doing Tasks - -For software engineering tasks (bugs, features, refactoring, explaining): - -1. **Understand first** - NEVER propose changes to code you haven't read -2. **Plan if complex** - Use update_plan to break down the task -3. **Use tools extensively** - Search, read, and understand the codebase -4. **Follow conventions** - Match existing code style, libraries, and patterns -5. **Implement solution** - Make only necessary changes, avoid over-engineering -6. **Verify your work** - VERY IMPORTANT: Run lint and typecheck commands when done - -**CRITICAL - Code Quality**: - -- After completing tasks, you MUST run lint and typecheck commands (e.g., `npm run lint`, `npm run typecheck`) -- If commands unknown, ask user and suggest adding to AGENTS.md -- NEVER commit changes unless explicitly asked - -**Following Conventions**: - -- NEVER assume libraries are available - check package.json first -- Look at existing code to understand patterns -- Match code style, naming, and structure -- Follow security best practices - never log secrets or commit credentials -- DO NOT ADD COMMENTS unless asked - -**Avoid Over-engineering**: - -- Only make changes directly requested or clearly necessary -- Don't add features, refactor unrelated code, or make "improvements" -- Don't add error handling for scenarios that can't happen -- Don't create abstractions for one-time operations -- Three similar lines is better than a premature abstraction - -**Backwards Compatibility**: - -- Avoid hacks like renaming unused `_vars` or `// removed` comments -- If something is unused, delete it completely - ---- - -# Code References - -When referencing code, use the pattern `file_path:line_number`: - - -user: Where are errors handled? -assistant: Clients are marked as failed in the `connectToServer` function in src/services/process.ts:712. - - ---- - -# Proactiveness - -Balance between: - -1. Doing the right thing when asked -2. Not surprising the user with unexpected actions -3. Not adding explanations unless requested - -- If user asks how to approach something, answer first - don't immediately act -- After working on a file, just stop - don't explain what you did - ---- - -# Working Environment - -## Safety - -⚠️ **WARNING**: Environment is NOT SANDBOXED. Actions immediately affect the user's system. - -- Never access files outside working directory unless instructed -- Be careful with destructive operations (rm, overwrite) -- Avoid superuser commands unless instructed -- Validate inputs before shell commands - -## Project Context (AGENTS.md) - -Files named `AGENTS.md` may exist with project-specific guidance: - -- Project structure and conventions -- Build, test, and development workflows -- Security notes and configuration - -**IMPORTANT**: If you modify anything mentioned in these files, UPDATE them to keep current. - ---- - -# Git Operations - -## Creating Commits - -When user asks to create a commit: - -1. **You MUST run these commands IN PARALLEL**: - - `git status` (never use -uall flag) - - `git diff` (see staged and unstaged changes) - - `git log` (see recent commit style) - -2. **Analyze changes**: - - Summarize nature of changes (feature, fix, refactor, etc.) - - Do not commit secrets (.env, credentials, etc.) - - Draft concise 1-2 sentence message focusing on "why" not "what" - -3. **Execute commit** (run commands in parallel where independent): - - Add relevant untracked files - - Create commit with message - - Run git status to verify - -**Git Safety**: - -- NEVER update git config -- NEVER run destructive commands (force push, hard reset) unless explicitly requested -- NEVER skip hooks (--no-verify) unless requested -- NEVER use -i flag commands (git rebase -i, git add -i) -- CRITICAL: ALWAYS create NEW commits, never use --amend unless requested -- NEVER commit unless explicitly asked - -**Commit Message Format** (use HEREDOC): - -```bash -git commit -m "$(cat <<'EOF' -Commit message here. -EOF -)" -``` - -## Creating Pull Requests - -Use `gh` command for GitHub operations. - -When user asks to create a PR: - -1. **Run these commands IN PARALLEL**: - - `git status` - - `git diff` - - Check if branch tracks remote - - `git log` and `git diff [base-branch]...HEAD` - -2. **Analyze ALL commits** that will be in the PR (not just latest) - -3. **Create PR** (run in parallel where independent): - - Create new branch if needed - - Push to remote with -u if needed - - Create PR with `gh pr create` - -**PR Format** (use HEREDOC): - -```bash -gh pr create --title "title" --body "$(cat <<'EOF' -## Summary -<1-3 bullet points> - -## Test plan -[Checklist for testing] - -🤖 Generated with Memo Code -EOF -)" -``` - ---- - -# Available Tools Reference - -Your available tools will be provided separately. Use them liberally and in parallel when appropriate. - -Common tools include: - -- **exec_command / write_stdin**: Run and continue interactive shell sessions -- **shell / shell_command**: Shell execution compatibility variants -- **apply_patch**: structured patch edits (`Begin/End`, `Add/Delete/Update`, `@@` hunks) -- **read_text_file / read_media_file / read_files / write_file / edit_file / list_directory / search_files**: Local filesystem read/write/edit/search -- **list_mcp_resources / list_mcp_resource_templates / read_mcp_resource**: MCP resource context access -- **update_plan**: Structured progress plan updates -- **webfetch**: Fetch a URL with pagination and return extracted markdown or raw content -- **get_memory**: Read persisted memory payload - -## Memory Tool Usage - -Use `get_memory` to retrieve persisted memory context for the current workflow: - -- **Input**: Provide a stable `memory_id` -- **Output**: Returns stored memory summary payload -- **Fallback**: If memory is missing, continue without blocking on memory retrieval - ---- - -# Ultimate Reminders - -At all times: - -- **Concise**: < 4 lines of text (not including tools/code) -- **Parallel**: Multiple independent tool calls in ONE message -- **Plan-driven**: Use update_plan for complex tasks -- **Quality-focused**: Run lint/typecheck after changes -- **Reference precisely**: Use `file:line` format -- **Safety conscious**: Actions have real consequences -- **Focused**: Only make necessary changes - -**Core Mantras**: - -- Don't deviate from user needs -- Don't give more than asked for -- Verify when uncertain -- Think twice before acting -- Keep it simple -- No time estimates or predictions - ---- - -**IMPORTANT**: You MUST answer concisely with fewer than 4 lines of text (not including tool calls or code generation), unless user explicitly asks for detail. diff --git a/packages/core/src/runtime/prompt.test.ts b/packages/core/src/runtime/prompt.test.ts deleted file mode 100644 index cf8a8ed..0000000 --- a/packages/core/src/runtime/prompt.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises' -import os from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, test } from 'vitest' -import { loadSystemPrompt } from './prompt' - -const createdDirs: string[] = [] - -afterEach(async () => { - delete process.env.MEMO_SYSTEM_PROMPT_PATH - delete process.env.MEMO_HOME - await Promise.all(createdDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) -}) - -async function createTempDir(prefix: string): Promise { - const dir = await mkdtemp(join(os.tmpdir(), prefix)) - createdDirs.push(dir) - return dir -} - -describe('loadSystemPrompt', () => { - test('supports explicit promptPath override', async () => { - const dir = await createTempDir('memo-prompt-test-') - const promptPath = join(dir, 'custom-prompt.md') - await writeFile(promptPath, 'cwd={{pwd}}', 'utf-8') - - const prompt = await loadSystemPrompt({ - cwd: '/tmp/project-root', - includeSkills: false, - memoHome: dir, - promptPath, - }) - - expect(prompt).toBe('cwd=/tmp/project-root') - }) - - test('reads prompt from MEMO_SYSTEM_PROMPT_PATH when provided', async () => { - const dir = await createTempDir('memo-prompt-env-test-') - const promptPath = join(dir, 'env-prompt.md') - await writeFile(promptPath, 'from-env', 'utf-8') - process.env.MEMO_SYSTEM_PROMPT_PATH = promptPath - - const prompt = await loadSystemPrompt({ - cwd: dir, - includeSkills: false, - memoHome: dir, - }) - - expect(prompt).toBe('from-env') - }) - - test('injects SOUL.md into placeholder when template includes soul_section', async () => { - const dir = await createTempDir('memo-prompt-soul-placeholder-') - const promptPath = join(dir, 'prompt.md') - const soulPath = join(dir, 'SOUL.md') - await writeFile(promptPath, 'head\n{{soul_section}}\ntail', 'utf-8') - await writeFile(soulPath, '# Soul\n\n- calmer tone\n', 'utf-8') - - const prompt = await loadSystemPrompt({ - cwd: dir, - includeSkills: false, - memoHome: dir, - promptPath, - }) - - expect(prompt).toContain('## User Personality Context (SOUL.md)') - expect(prompt).toContain(`Loaded from: ${soulPath}`) - expect(prompt).toContain('- calmer tone') - expect(prompt.indexOf('## User Personality Context (SOUL.md)')).toBeGreaterThan(prompt.indexOf('head')) - expect(prompt.indexOf('tail')).toBeGreaterThan(prompt.indexOf('## User Personality Context (SOUL.md)')) - }) - - test('falls back to append SOUL.md when template has no placeholder', async () => { - const dir = await createTempDir('memo-prompt-soul-fallback-') - const promptPath = join(dir, 'prompt.md') - const soulPath = join(dir, 'SOUL.md') - await writeFile(promptPath, 'custom-template', 'utf-8') - await writeFile(soulPath, 'prefers short replies', 'utf-8') - - const prompt = await loadSystemPrompt({ - cwd: dir, - includeSkills: false, - memoHome: dir, - promptPath, - }) - - expect(prompt.startsWith('custom-template')).toBe(true) - expect(prompt).toContain('## User Personality Context (SOUL.md)') - expect(prompt).toContain(`Loaded from: ${soulPath}`) - expect(prompt).toContain('prefers short replies') - }) - - test('does not inject SOUL section when SOUL.md is missing or empty', async () => { - const missingDir = await createTempDir('memo-prompt-soul-missing-') - const promptPath = join(missingDir, 'prompt.md') - await writeFile(promptPath, 'base {{soul_section}} end', 'utf-8') - - const promptWithoutSoul = await loadSystemPrompt({ - cwd: missingDir, - includeSkills: false, - memoHome: missingDir, - promptPath, - }) - expect(promptWithoutSoul).toBe('base end') - - const emptyDir = await createTempDir('memo-prompt-soul-empty-') - const emptyPromptPath = join(emptyDir, 'prompt.md') - await writeFile(emptyPromptPath, 'base {{soul_section}} end', 'utf-8') - await writeFile(join(emptyDir, 'SOUL.md'), ' \n\t\n', 'utf-8') - const promptWithEmptySoul = await loadSystemPrompt({ - cwd: emptyDir, - includeSkills: false, - memoHome: emptyDir, - promptPath: emptyPromptPath, - }) - expect(promptWithEmptySoul).toBe('base end') - }) -}) diff --git a/packages/core/src/runtime/session_runtime_helpers.test.ts b/packages/core/src/runtime/session_runtime_helpers.test.ts deleted file mode 100644 index 6047a84..0000000 --- a/packages/core/src/runtime/session_runtime_helpers.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { describe, expect, test, vi } from 'vitest' -import type { HistorySink } from '@memo/core/types' -import { - accumulateUsage, - completeToolResultsForProtocol, - emitEventToSinks, - emptyUsage, - fallbackSessionTitleFromPrompt, - isAbortError, - normalizeSessionTitle, - parseTextToolCall, - stableStringify, - toToolHistoryMessage, - truncateSessionTitle, -} from '@memo/core/runtime/session_runtime_helpers' - -describe('accumulateUsage', () => { - test('uses explicit total when provided', () => { - const usage = emptyUsage() - accumulateUsage(usage, { prompt: 2, completion: 3, total: 100 }) - expect(usage).toEqual({ prompt: 2, completion: 3, total: 100 }) - }) - - test('falls back to prompt + completion when total is absent', () => { - const usage = emptyUsage() - accumulateUsage(usage, { prompt: 2, completion: 3 }) - expect(usage).toEqual({ prompt: 2, completion: 3, total: 5 }) - }) -}) - -describe('emitEventToSinks', () => { - test('writes structured error payload to stderr when sink append fails', async () => { - const writes: string[] = [] - const writeSpy = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { - writes.push(String(chunk)) - return true - }) as typeof process.stderr.write) - - const failingSink: HistorySink = { - append: async () => { - throw new Error('disk full') - }, - } - - try { - await emitEventToSinks( - { - ts: '2026-01-01T00:00:00.000Z', - sessionId: 's-1', - type: 'assistant', - content: 'hello', - }, - [failingSink], - ) - } finally { - writeSpy.mockRestore() - } - - expect(writes.length).toBeGreaterThan(0) - const parsed = JSON.parse(writes.join('').trim()) as Record - expect(parsed.level).toBe('error') - expect(parsed.event).toBe('history_sink_append_failed') - expect(parsed.message).toBe('disk full') - expect(parsed.sink).toBe('Object') - }) -}) - -describe('stableStringify', () => { - test('serializes self-referencing object without throwing', () => { - const root: Record = {} - root.self = root - - const serialized = stableStringify(root) - expect(serialized).toBe('{"self":"[Circular]"}') - }) - - test('serializes indirect circular references with circular marker', () => { - const parent: Record = { name: 'parent' } - const child: Record = { name: 'child', parent } - parent.child = child - - const serialized = stableStringify(parent) - expect(serialized).toContain('"child":{"name":"child","parent":"[Circular]"}') - expect(serialized).toContain('"name":"parent"') - }) -}) - -describe('parseTextToolCall', () => { - const tools = { - read_file: {} as never, - exec_command: {} as never, - } - - test('parses plain json tool call', () => { - const parsed = parseTextToolCall('{"tool":"read_file","input":{"path":"a.txt"}}', tools) - expect(parsed).toEqual({ - tool: 'read_file', - input: { path: 'a.txt' }, - }) - }) - - test('parses fenced json tool call', () => { - const parsed = parseTextToolCall('```json\n{"tool":"exec_command","input":{"cmd":"ls"}}\n```', tools) - expect(parsed).toEqual({ - tool: 'exec_command', - input: { cmd: 'ls' }, - }) - }) - - test('returns null for unknown or invalid tool payload', () => { - expect(parseTextToolCall('{"tool":"unknown","input":{}}', tools)).toBeNull() - expect(parseTextToolCall('{"tool":"read_file"', tools)).toBeNull() - expect(parseTextToolCall('not-json', tools)).toBeNull() - expect(parseTextToolCall(' ', tools)).toBeNull() - }) -}) - -describe('session title helpers', () => { - test('truncateSessionTitle appends ellipsis when exceeding max', () => { - const truncated = truncateSessionTitle('x'.repeat(80)) - expect(truncated.endsWith('...')).toBe(true) - expect(truncated.length).toBe(60) - }) - - test('normalizeSessionTitle strips quotes and whitespace', () => { - expect(normalizeSessionTitle(' " Hello\nWorld " ')).toBe('Hello World') - expect(normalizeSessionTitle(' ')).toBe('') - }) - - test('normalizeSessionTitle removes think tags and title prefixes', () => { - expect( - normalizeSessionTitle( - 'internal Title: "Build REST API migration plan" secret', - ), - ).toBe('Build REST API migration plan') - }) - - test('fallbackSessionTitleFromPrompt handles empty/cjk/word prompts', () => { - expect(fallbackSessionTitleFromPrompt(' ')).toBe('New Session') - expect(fallbackSessionTitleFromPrompt('这是一个非常非常长的中文标题用于测试截断行为')).toBe( - '这是一个非常非常长的中文标题用于测试截断...', - ) - expect(fallbackSessionTitleFromPrompt('build a rest api using express and sqlite quickly')).toBe( - 'build a rest api using express and sqlite', - ) - }) -}) - -describe('tool result helpers', () => { - test('toToolHistoryMessage maps tool action result into tool chat message', () => { - const message = toToolHistoryMessage({ - actionId: 'call-1', - tool: 'read_file', - status: 'success', - observation: 'content', - success: true, - durationMs: 12, - }) - expect(message).toEqual({ - role: 'tool', - content: 'content', - tool_call_id: 'call-1', - name: 'read_file', - }) - }) - - test('completeToolResultsForProtocol fills missing results', () => { - const requested = [ - { id: 'call-1', name: 'read_file' }, - { id: 'call-2', name: 'exec_command' }, - ] - const actual = [ - { - actionId: 'call-1', - tool: 'read_file', - status: 'success' as const, - observation: 'ok', - success: true, - durationMs: 5, - }, - ] - - const failureFilled = completeToolResultsForProtocol(requested, actual, false) - expect(failureFilled).toHaveLength(2) - expect(failureFilled[0]).toMatchObject({ actionId: 'call-1', status: 'success' }) - expect(failureFilled[1]).toMatchObject({ - actionId: 'call-2', - status: 'execution_failed', - errorType: 'execution_failed', - rejected: undefined, - }) - expect(failureFilled[1]?.observation).toContain('Tool result missing for exec_command') - - const rejectionFilled = completeToolResultsForProtocol(requested, actual, true) - expect(rejectionFilled[1]).toMatchObject({ - actionId: 'call-2', - status: 'approval_denied', - errorType: 'approval_denied', - rejected: true, - }) - expect(rejectionFilled[1]?.observation).toContain('Skipped tool execution after previous rejection') - }) -}) - -describe('isAbortError', () => { - test('detects abort error by name and message', () => { - const abortError = new Error('cancelled') - abortError.name = 'AbortError' - const abortedMessageError = new Error('Request was aborted.') - expect(isAbortError(abortError)).toBe(true) - expect(isAbortError(abortedMessageError)).toBe(true) - expect(isAbortError(new Error('other'))).toBe(false) - expect(isAbortError('AbortError')).toBe(false) - }) -}) diff --git a/packages/core/src/runtime/session_runtime_helpers.ts b/packages/core/src/runtime/session_runtime_helpers.ts deleted file mode 100644 index 817bf7e..0000000 --- a/packages/core/src/runtime/session_runtime_helpers.ts +++ /dev/null @@ -1,281 +0,0 @@ -import type { - AgentSessionOptions, - AssistantToolCall, - ChatMessage, - HistoryEvent, - HistorySink, - LLMResponse, - SessionMode, - TextBlock, - TokenUsage, - ToolPermissionMode, - ToolRegistry, - ToolUseBlock, -} from '@memo/core/types' -import type { ToolActionResult, ToolActionStatus } from '@memo/tools/orchestrator' - -export const DEFAULT_SESSION_MODE: SessionMode = 'interactive' -export const DEFAULT_CONTEXT_WINDOW = 120_000 -export const TOOL_ACTION_SUCCESS_STATUS: ToolActionStatus = 'success' -export const TOOL_DISABLED_ERROR_MESSAGE = - 'Tool usage is disabled in the current permission mode. Switch to /tools once or /tools full to enable tools.' -export const SESSION_TITLE_MAX_CHARS = 60 -export const TOOL_SKIPPED_AFTER_REJECTION_MESSAGE = 'Skipped tool execution after previous rejection.' -export const TOOL_SKIPPED_DISABLED_MESSAGE = 'Tool execution skipped: tools are disabled in current permission mode.' - -export type ResolvedToolPermission = { - mode: ToolPermissionMode | 'auto' - toolsDisabled: boolean - dangerous: boolean - approvalMode: 'auto' | 'strict' -} - -function writeStructuredError(payload: Record) { - process.stderr.write(`${JSON.stringify(payload)}\n`) -} - -export function resolveToolPermission(options: AgentSessionOptions): ResolvedToolPermission { - if (options.toolPermissionMode === 'none') { - return { - mode: 'none', - toolsDisabled: true, - dangerous: false, - approvalMode: 'auto', - } - } - - if (options.toolPermissionMode === 'once') { - return { - mode: 'once', - toolsDisabled: false, - dangerous: false, - approvalMode: 'auto', - } - } - - if (options.toolPermissionMode === 'full') { - return { - mode: 'full', - toolsDisabled: false, - dangerous: true, - approvalMode: 'auto', - } - } - - const dangerous = options.dangerous ?? false - return { - mode: dangerous ? 'full' : 'auto', - toolsDisabled: false, - dangerous, - approvalMode: 'auto', - } -} - -export function emptyUsage(): TokenUsage { - return { prompt: 0, completion: 0, total: 0 } -} - -export function accumulateUsage(target: TokenUsage, delta?: Partial) { - if (!delta) return - const promptDelta = delta.prompt ?? 0 - const completionDelta = delta.completion ?? 0 - const totalDelta = delta.total ?? promptDelta + completionDelta - target.prompt += promptDelta - target.completion += completionDelta - target.total += totalDelta -} - -export function normalizeLLMResponse(raw: LLMResponse): { - textContent: string - toolUseBlocks: Array<{ id: string; name: string; input: unknown }> - reasoningContent?: string - stopReason?: 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' - usage?: Partial -} { - const textBlocks = raw.content.filter((block): block is TextBlock => block.type === 'text') - const toolBlocks = raw.content.filter((block): block is ToolUseBlock => block.type === 'tool_use') - - return { - textContent: textBlocks.map((b) => b.text).join('\n'), - toolUseBlocks: toolBlocks.map((b) => ({ - id: b.id, - name: b.name, - input: b.input, - })), - reasoningContent: - typeof raw.reasoning_content === 'string' && raw.reasoning_content.trim().length > 0 - ? raw.reasoning_content - : undefined, - stopReason: raw.stop_reason, - usage: raw.usage, - } -} - -export async function emitEventToSinks(event: HistoryEvent, sinks: HistorySink[]) { - for (const sink of sinks) { - try { - await sink.append(event) - } catch (err) { - writeStructuredError({ - level: 'error', - event: 'history_sink_append_failed', - sink: sink.constructor?.name || 'anonymous_sink', - message: (err as Error).message, - }) - } - } -} - -export function isAbortError(err: unknown): err is Error { - if (!(err instanceof Error)) return false - if (err.name === 'AbortError') return true - - const message = err.message?.toLowerCase?.() ?? '' - return ( - message.includes('request was aborted') || - message.includes('operation was aborted') || - message.includes('aborted') - ) -} - -// Stable serialization for duplicate action detection (ensures consistent key ordering) -export function stableStringify(value: unknown): string { - return stableStringifyWithSeen(value, new WeakSet(), 0) -} - -const MAX_STABLE_STRINGIFY_DEPTH = 100 - -function stableStringifyWithSeen(value: unknown, seen: WeakSet, depth: number): string { - if (depth > MAX_STABLE_STRINGIFY_DEPTH) { - return JSON.stringify('[MaxDepthExceeded]') - } - if (typeof value === 'bigint') { - return JSON.stringify(value.toString()) - } - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (seen.has(value)) { - return JSON.stringify('[Circular]') - } - - seen.add(value) - if (Array.isArray(value)) { - const result = `[${value.map((v) => stableStringifyWithSeen(v, seen, depth + 1)).join(',')}]` - seen.delete(value) - return result - } - const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)) - const result = `{${entries - .map(([k, v]) => `${JSON.stringify(k)}:${stableStringifyWithSeen(v, seen, depth + 1)}`) - .join(',')}}` - seen.delete(value) - return result -} - -export function buildAssistantToolCalls( - toolUseBlocks: Array<{ id: string; name: string; input: unknown }>, -): AssistantToolCall[] { - return toolUseBlocks.map((block) => ({ - id: block.id, - type: 'function', - function: { - name: block.name, - arguments: stableStringify(block.input), - }, - })) -} - -export function parseTextToolCall(text: string, tools: ToolRegistry): { tool: string; input: unknown } | null { - const trimmed = text.trim() - if (!trimmed) return null - - const candidates = [trimmed] - const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i) - if (fenced?.[1]) { - candidates.push(fenced[1].trim()) - } - - for (const candidate of candidates) { - if (!candidate.startsWith('{') || !candidate.endsWith('}')) continue - try { - const parsed = JSON.parse(candidate) - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue - const obj = parsed as Record - const tool = typeof obj.tool === 'string' ? obj.tool.trim() : '' - if (!tool || !Object.prototype.hasOwnProperty.call(tools, tool)) continue - return { tool, input: obj.input ?? {} } - } catch { - // Ignore invalid json - } - } - - return null -} - -export function truncateSessionTitle(input: string): string { - if (input.length <= SESSION_TITLE_MAX_CHARS) return input - return `${input.slice(0, SESSION_TITLE_MAX_CHARS - 3).trimEnd()}...` -} - -export function normalizeSessionTitle(raw: string): string { - const compact = raw - .replace(/<\s*(think|thinking)\b[^>]*>[\s\S]*?<\s*\/\s*\1\s*>/gi, ' ') - .replace(/<\s*\/?\s*(think|thinking)\b[^>]*>/gi, ' ') - .replace(/\r?\n+/g, ' ') - .replace(/\s+/g, ' ') - .trim() - if (!compact) return '' - const unprefixed = compact.replace(/^(title|session title|标题)\s*[:：-]\s*/i, '').trim() - if (!unprefixed) return '' - const unquoted = unprefixed.replace(/^["'`“”‘’]+|["'`“”‘’]+$/g, '').trim() - if (!unquoted) return '' - return truncateSessionTitle(unquoted) -} - -export function fallbackSessionTitleFromPrompt(input: string): string { - const compact = input.replace(/\s+/g, ' ').trim() - if (!compact) return 'New Session' - - // Keep short CJK/non-space prompts readable. - if (!compact.includes(' ')) { - return compact.length <= 20 ? compact : `${compact.slice(0, 20).trimEnd()}...` - } - - const words = compact.split(' ').filter(Boolean) - const short = words.slice(0, 8).join(' ') - return truncateSessionTitle(short || compact) -} - -export function toToolHistoryMessage(result: ToolActionResult): ChatMessage { - return { - role: 'tool', - content: result.observation, - tool_call_id: result.actionId, - name: result.tool, - } -} - -export function completeToolResultsForProtocol( - requested: Array<{ id: string; name: string }>, - actual: ToolActionResult[], - hasRejection: boolean, -): ToolActionResult[] { - const byActionId = new Map(actual.map((result) => [result.actionId, result])) - return requested.map((block) => { - const found = byActionId.get(block.id) - if (found) { - return found - } - return { - actionId: block.id, - tool: block.name, - status: hasRejection ? 'approval_denied' : 'execution_failed', - errorType: hasRejection ? 'approval_denied' : 'execution_failed', - success: false, - observation: hasRejection - ? `${TOOL_SKIPPED_AFTER_REJECTION_MESSAGE} ${block.name}` - : `Tool result missing for ${block.name}; execution aborted before producing output.`, - durationMs: 0, - rejected: hasRejection ? true : undefined, - } - }) -} diff --git a/packages/core/src/runtime/skills.test.ts b/packages/core/src/runtime/skills.test.ts deleted file mode 100644 index 83874b8..0000000 --- a/packages/core/src/runtime/skills.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import assert from 'node:assert' -import { mkdir, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { describe, test } from 'vitest' -import { loadSkills } from '@memo/core/runtime/skills' - -async function makeTempDir(prefix: string) { - const dir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`) - await mkdir(dir, { recursive: true }) - return dir -} - -async function removeDir(path: string) { - await rm(path, { recursive: true, force: true }) -} - -async function writeSkill(skillRoot: string, skillName: string, description: string) { - const skillDir = join(skillRoot, skillName) - const skillPath = join(skillDir, 'SKILL.md') - await mkdir(skillDir, { recursive: true }) - await writeFile( - skillPath, - `--- -name: ${skillName} -description: ${description} ---- -# ${skillName} -`, - 'utf-8', - ) - return skillPath -} - -describe('skills discovery', () => { - test('discovers project .xxx/skills and ~/.memo/skills only', async () => { - const sandbox = await makeTempDir('memo-core-skills-discovery') - const projectRoot = join(sandbox, 'repo') - const nestedCwd = join(projectRoot, 'packages', 'core') - const homeDir = join(sandbox, 'home') - const memoHome = join(homeDir, '.memo') - - await mkdir(nestedCwd, { recursive: true }) - await mkdir(homeDir, { recursive: true }) - await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') - - await writeSkill(join(projectRoot, '.agents', 'skills'), 'memo-default', 'memo default') - await writeSkill(join(projectRoot, '.claude', 'skills'), 'claude-compat', 'claude compat') - await writeSkill(join(projectRoot, '.codex', 'skills'), 'codex-compat', 'codex compat') - await writeSkill(join(memoHome, 'skills'), 'memo-global', 'memo global') - - // Should NOT be discovered: non-memo home hidden directories. - await writeSkill(join(homeDir, '.agents', 'skills'), 'home-agents', 'home agents') - await writeSkill(join(homeDir, '.codex', 'skills'), 'home-codex', 'home codex') - - try { - const discovered = await loadSkills({ cwd: nestedCwd, homeDir, memoHome }) - const names = new Set(discovered.map((skill) => skill.name)) - - assert.ok(names.has('memo-default')) - assert.ok(names.has('claude-compat')) - assert.ok(names.has('codex-compat')) - assert.ok(names.has('memo-global')) - assert.ok(!names.has('home-agents')) - assert.ok(!names.has('home-codex')) - } finally { - await removeDir(sandbox) - } - }) - - test('falls back to cwd when no git root exists', async () => { - const sandbox = await makeTempDir('memo-core-skills-no-git') - const parentDir = join(sandbox, 'parent') - const cwd = join(parentDir, 'child') - const homeDir = join(sandbox, 'home') - const memoHome = join(homeDir, '.memo') - - await mkdir(cwd, { recursive: true }) - await mkdir(homeDir, { recursive: true }) - - await writeSkill(join(parentDir, '.agents', 'skills'), 'parent-skill', 'parent level') - await writeSkill(join(cwd, '.agents', 'skills'), 'cwd-skill', 'cwd level') - - try { - const discovered = await loadSkills({ cwd, homeDir, memoHome }) - const names = new Set(discovered.map((skill) => skill.name)) - - assert.ok(names.has('cwd-skill')) - assert.ok(!names.has('parent-skill')) - } finally { - await removeDir(sandbox) - } - }) -}) diff --git a/packages/core/src/skills/builtin/skill-creator/SKILL.md b/packages/core/src/skills/builtin/skill-creator/SKILL.md new file mode 100644 index 0000000..213d9b5 --- /dev/null +++ b/packages/core/src/skills/builtin/skill-creator/SKILL.md @@ -0,0 +1,387 @@ +--- +name: skill-creator +description: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Memo's capabilities with specialized knowledge, workflows, or tool integrations. +--- + +# Skill Creator + +This skill provides guidance for creating effective skills. + +## About Skills + +Skills are modular, self-contained folders that extend Memo's capabilities by providing +specialized knowledge, workflows, and tools. Think of them as "onboarding guides" for specific +domains or tasks—they transform Memo from a general-purpose agent into a specialized agent +equipped with procedural knowledge that no model can fully possess. + +### What Skills Provide + +1. Specialized workflows - Multi-step procedures for specific domains +2. Tool integrations - Instructions for working with specific file formats or APIs +3. Domain expertise - Company-specific knowledge, schemas, business logic +4. Bundled resources - Scripts, references, and assets for complex and repetitive tasks + +## Core Principles + +### Concise is Key + +The context window is a public good. Skills share the context window with everything else Memo needs: system prompt, conversation history, other Skills' metadata, and the actual user request. + +**Default assumption: Memo is already very smart.** Only add context Memo doesn't already have. Challenge each piece of information: "Does Memo really need this explanation?" and "Does this paragraph justify its token cost?" + +Prefer concise examples over verbose explanations. + +### Set Appropriate Degrees of Freedom + +Match the level of specificity to the task's fragility and variability: + +**High freedom (text-based instructions)**: Use when multiple approaches are valid, decisions depend on context, or heuristics guide the approach. + +**Medium freedom (pseudocode or scripts with parameters)**: Use when a preferred pattern exists, some variation is acceptable, or configuration affects behavior. + +**Low freedom (specific scripts, few parameters)**: Use when operations are fragile and error-prone, consistency is critical, or a specific sequence must be followed. + +Think of Memo as exploring a path: a narrow bridge with cliffs needs specific guardrails (low freedom), while an open field allows many routes (high freedom). + +### Anatomy of a Skill + +Every skill consists of a required SKILL.md file and optional bundled resources: + +``` +skill-name/ +├── SKILL.md (required) +│ ├── YAML frontmatter metadata (required) +│ │ ├── name: (required) +│ │ └── description: (required) +│ └── Markdown instructions (required) +└── Bundled Resources (optional) + ├── scripts/ - Executable code (Node/shell scripts, etc.) + ├── references/ - Documentation intended to be loaded into context as needed + └── assets/ - Files used in output (templates, icons, fonts, etc.) +``` + +#### SKILL.md (required) + +Every SKILL.md consists of: + +- **Frontmatter** (YAML): Contains `name` and `description` fields. These are the only fields that Memo reads to determine when the skill gets used, thus it is very important to be clear and comprehensive in describing what the skill is, and when it should be used. +- **Body** (Markdown): Instructions and guidance for using the skill. Only loaded AFTER the skill triggers (if at all). + +#### Bundled Resources (optional) + +##### Scripts (`scripts/`) + +Executable code (Node `.mjs` scripts, shell scripts, etc.) for tasks that require deterministic reliability or are repeatedly rewritten. + +- **When to include**: When the same code is being rewritten repeatedly or deterministic reliability is needed +- **Example**: `scripts/rotate_pdf.mjs` for PDF rotation tasks +- **Benefits**: Token efficient, deterministic, may be executed without loading into context +- **Note**: Scripts may still need to be read by Memo for patching or environment-specific adjustments + +##### References (`references/`) + +Documentation and reference material intended to be loaded as needed into context to inform Memo's process and thinking. + +- **When to include**: For documentation that Memo should reference while working +- **Examples**: `references/finance.md` for financial schemas, `references/mnda.md` for company NDA template, `references/policies.md` for company policies, `references/api_docs.md` for API specifications +- **Use cases**: Database schemas, API documentation, domain knowledge, company policies, detailed workflow guides +- **Benefits**: Keeps SKILL.md lean, loaded only when Memo determines it's needed +- **Best practice**: If files are large (>10k words), include grep search patterns in SKILL.md +- **Avoid duplication**: Information should live in either SKILL.md or references files, not both. Prefer references files for detailed information unless it's truly core to the skill—this keeps SKILL.md lean while making information discoverable without hogging the context window. Keep only essential procedural instructions and workflow guidance in SKILL.md; move detailed reference material, schemas, and examples to references files. + +##### Assets (`assets/`) + +Files not intended to be loaded into context, but rather used within the output Memo produces. + +- **When to include**: When the skill needs files that will be used in the final output +- **Examples**: `assets/logo.png` for brand assets, `assets/slides.pptx` for PowerPoint templates, `assets/frontend-template/` for HTML/React boilerplate, `assets/font.ttf` for typography +- **Use cases**: Templates, images, icons, boilerplate code, fonts, sample documents that get copied or modified +- **Benefits**: Separates output resources from documentation, enables Memo to use files without loading them into context + +#### What to Not Include in a Skill + +A skill should only contain essential files that directly support its functionality. Do NOT create extraneous documentation or auxiliary files, including: + +- README.md +- INSTALLATION_GUIDE.md +- QUICK_REFERENCE.md +- CHANGELOG.md +- etc. + +The skill should only contain the information needed for an AI agent to do the job at hand. It should not contain auxiliary context about the process that went into creating it, setup and testing procedures, user-facing documentation, etc. Creating additional documentation files just adds clutter and confusion. + +### Progressive Disclosure Design Principle + +Skills use a three-level loading system to manage context efficiently: + +1. **Metadata (name + description)** - Always in context (~100 words) +2. **SKILL.md body** - When skill triggers (<5k words) +3. **Bundled resources** - As needed by Memo (Unlimited because scripts can be executed without reading into context window) + +#### Progressive Disclosure Patterns + +Keep SKILL.md body to the essentials and under 500 lines to minimize context bloat. Split content into separate files when approaching this limit. When splitting out content into other files, it is very important to reference them from SKILL.md and describe clearly when to read them, to ensure the reader of the skill knows they exist and when to use them. + +**Key principle:** When a skill supports multiple variations, frameworks, or options, keep only the core workflow and selection guidance in SKILL.md. Move variant-specific details (patterns, examples, configuration) into separate reference files. + +**Pattern 1: High-level guide with references** + +```markdown +# PDF Processing + +## Quick start + +Extract text with pdfplumber: +[code example] + +## Advanced features + +- **Form filling**: See [FORMS.md](FORMS.md) for complete guide +- **API reference**: See [REFERENCE.md](REFERENCE.md) for all methods +- **Examples**: See [EXAMPLES.md](EXAMPLES.md) for common patterns +``` + +Memo loads FORMS.md, REFERENCE.md, or EXAMPLES.md only when needed. + +**Pattern 2: Domain-specific organization** + +For Skills with multiple domains, organize content by domain to avoid loading irrelevant context: + +``` +bigquery-skill/ +├── SKILL.md (overview and navigation) +└── reference/ + ├── finance.md (revenue, billing metrics) + ├── sales.md (opportunities, pipeline) + ├── product.md (API usage, features) + └── marketing.md (campaigns, attribution) +``` + +When a user asks about sales metrics, Memo only reads sales.md. + +Similarly, for skills supporting multiple frameworks or variants, organize by variant: + +``` +cloud-deploy/ +├── SKILL.md (workflow + provider selection) +└── references/ + ├── aws.md (AWS deployment patterns) + ├── gcp.md (GCP deployment patterns) + └── azure.md (Azure deployment patterns) +``` + +When the user chooses AWS, Memo only reads aws.md. + +**Pattern 3: Conditional details** + +Show basic content, link to advanced content: + +```markdown +# DOCX Processing + +## Creating documents + +Use docx-js for new documents. See [DOCX-JS.md](DOCX-JS.md). + +## Editing documents + +For simple edits, modify the XML directly. + +**For tracked changes**: See [REDLINING.md](REDLINING.md) +**For OOXML details**: See [OOXML.md](OOXML.md) +``` + +Memo reads REDLINING.md or OOXML.md only when the user needs those features. + +**Important guidelines:** + +- **Avoid deeply nested references** - Keep references one level deep from SKILL.md. All reference files should link directly from SKILL.md. +- **Structure longer reference files** - For files longer than 100 lines, include a table of contents at the top so Memo can see the full scope when previewing. + +## Skill Creation Process + +Skill creation involves these steps: + +1. Understand the skill with concrete examples +2. Plan reusable skill contents (scripts, references, assets) +3. Initialize the skill (run init_skill.mjs) +4. Edit the skill (implement resources and write SKILL.md) +5. Validate the skill (run quick_validate.mjs) +6. Iterate based on real usage and forward-test complex skills. + +Follow these steps in order, skipping only if there is a clear reason why they are not applicable. + +### Skill Naming + +- Use lowercase letters, digits, and hyphens only; normalize user-provided titles to hyphen-case (e.g., "Plan Mode" -> `plan-mode`). +- When generating names, generate a name under 64 characters (letters, digits, hyphens). +- Prefer short, verb-led phrases that describe the action. +- Namespace by tool when it improves clarity or triggering (e.g., `gh-address-comments`, `linear-address-issue`). +- Name the skill folder exactly after the skill name. + +### Step 1: Understanding the Skill with Concrete Examples + +Skip this step only when the skill's usage patterns are already clearly understood. It remains valuable even when working with an existing skill. + +To create an effective skill, clearly understand concrete examples of how the skill will be used. This understanding can come from either direct user examples or generated examples that are validated with user feedback. + +For example, when building an image-editor skill, relevant questions include: + +- "What functionality should the image-editor skill support? Editing, rotating, anything else?" +- "Can you give some examples of how this skill would be used?" +- "I can imagine users asking for things like 'Remove the red-eye from this image' or 'Rotate this image'. Are there other ways you imagine this skill being used?" +- "What would a user say that should trigger this skill?" +- "Where should I create this skill? If you do not have a preference, I will place it in `$MEMO_HOME/skills` (or `~/.memo/skills` when `MEMO_HOME` is unset) so Memo can discover it automatically." + +To avoid overwhelming users, avoid asking too many questions in a single message. Start with the most important questions and follow up as needed for better effectiveness. + +Conclude this step when there is a clear sense of the functionality the skill should support. + +### Step 2: Planning the Reusable Skill Contents + +To turn concrete examples into an effective skill, analyze each example by: + +1. Considering how to execute on the example from scratch +2. Identifying what scripts, references, and assets would be helpful when executing these workflows repeatedly + +Example: When building a `pdf-editor` skill to handle queries like "Help me rotate this PDF," the analysis shows: + +1. Rotating a PDF requires re-writing the same code each time +2. A `scripts/rotate_pdf.mjs` script would be helpful to store in the skill + +Example: When designing a `frontend-webapp-builder` skill for queries like "Build me a todo app" or "Build me a dashboard to track my steps," the analysis shows: + +1. Writing a frontend webapp requires the same boilerplate HTML/React each time +2. An `assets/hello-world/` template containing the boilerplate HTML/React project files would be helpful to store in the skill + +Example: When building a `big-query` skill to handle queries like "How many users have logged in today?" the analysis shows: + +1. Querying BigQuery requires re-discovering the table schemas and relationships each time +2. A `references/schema.md` file documenting the table schemas would be helpful to store in the skill + +To establish the skill's contents, analyze each concrete example to create a list of the reusable resources to include: scripts, references, and assets. + +### Step 3: Initializing the Skill + +At this point, it is time to actually create the skill. + +Skip this step only if the skill being developed already exists. In this case, continue to the next step. + +Before running `init_skill.mjs`, ask where the user wants the skill created. If they do not specify a location, default to `$MEMO_HOME/skills`; when `MEMO_HOME` is unset, fall back to `~/.memo/skills` so the skill is auto-discovered. + +When creating a new skill from scratch, always run the `init_skill.mjs` script. The script conveniently generates a new template skill directory that automatically includes everything a skill requires, making the skill creation process much more efficient and reliable. + +Usage: + +```bash +node scripts/init_skill.mjs --path [--resources scripts,references,assets] [--examples] +``` + +Examples: + +```bash +node scripts/init_skill.mjs my-skill --path "${MEMO_HOME:-$HOME/.memo}/skills" +node scripts/init_skill.mjs my-skill --path "${MEMO_HOME:-$HOME/.memo}/skills" --resources scripts,references +node scripts/init_skill.mjs my-skill --path ~/work/skills --resources scripts --examples +``` + +The script: + +- Creates the skill directory at the specified path +- Generates a SKILL.md template with proper frontmatter and TODO placeholders +- Optionally creates resource directories based on `--resources` +- Optionally adds example files when `--examples` is set + +After initialization, customize the SKILL.md and add resources as needed. If you used `--examples`, replace or delete placeholder files. + +### Step 4: Edit the Skill + +When editing the (newly-generated or existing) skill, remember that the skill is being created for another instance of Memo to use. Include information that would be beneficial and non-obvious to Memo. Consider what procedural knowledge, domain-specific details, or reusable assets would help another Memo instance execute these tasks more effectively. + +#### Start with Reusable Skill Contents + +To begin implementation, start with the reusable resources identified above: `scripts/`, `references/`, and `assets/` files. Note that this step may require user input. For example, when implementing a `brand-guidelines` skill, the user may need to provide brand assets or templates to store in `assets/`, or documentation to store in `references/`. + +Added scripts must be tested by actually running them to ensure there are no bugs and that the output matches what is expected. If there are many similar scripts, only a representative sample needs to be tested to ensure confidence that they all work while balancing time to completion. + +If you used `--examples`, delete any placeholder files that are not needed for the skill. Only create resource directories that are actually required. + +#### Update SKILL.md + +**Writing Guidelines:** Always use imperative/infinitive form. + +##### Frontmatter + +Write the YAML frontmatter with `name` and `description`: + +- `name`: The skill name +- `description`: This is the primary triggering mechanism for your skill, and helps Memo understand when to use the skill. + - Include both what the Skill does and specific triggers/contexts for when to use it. + - Include all "when to use" information here - Not in the body. The body is only loaded after triggering, so "When to Use This Skill" sections in the body are not helpful to Memo. + - Example description for a `docx` skill: "Comprehensive document creation, editing, and analysis with support for tracked changes, comments, formatting preservation, and text extraction. Use when Memo needs to work with professional documents (.docx files) for: (1) Creating new documents, (2) Modifying or editing content, (3) Working with tracked changes, (4) Adding comments, or any other document tasks" + +Do not include any other fields in YAML frontmatter. + +##### Body + +Write instructions for using the skill and its bundled resources. + +### Step 5: Validate the Skill + +Once development of the skill is complete, validate the skill folder to catch basic issues early: + +```bash +node scripts/quick_validate.mjs +``` + +The validation script checks YAML frontmatter format, required fields, and naming rules. If validation fails, fix the reported issues and run the command again. + +### Step 6: Iterate + +After testing the skill, you may detect the skill is complex enough that it requires forward-testing; or users may request improvements. + +User testing often this happens right after using the skill, with fresh context of how the skill performed. + +**Forward-testing and iteration workflow:** + +1. Use the skill on real tasks +2. Notice struggles or inefficiencies +3. Identify how SKILL.md or bundled resources should be updated +4. Implement changes and test again +5. Forward-test if it is reasonable and appropriate + +## Forward-testing + +To forward-test, start a fresh Memo session in a clean directory and ask it to perform the task the skill is meant for, phrased the way a user would phrase it. + +Prompts should look like: + `Use $skill-x at /path/to/skill-x to solve problem y` +Not: + `Review the skill at /path/to/skill-x; pretend a user asks you to...` + +Decision rule for forward-testing: + - Err on the side of forward-testing + - Ask for approval if you think there's a risk that forward-testing would: + * take a long time, + * require additional approvals from the user, or + * modify live production systems + + In these cases, show the user your proposed prompt and request (1) a yes/no decision, and + (2) any suggested modifications. + +Considerations when forward-testing: + - use fresh sessions for independent passes + - pass the skill, and a request in a similar way the user would + - pass raw artifacts, not your conclusions + - avoid showing expected answers or intended fixes + - rebuild context from source artifacts after each iteration + - review the session's output, reasoning, and emitted artifacts + - avoid leaving artifacts the agent can find on disk between iterations; + clean up artifacts to avoid additional contamination. + +If forward-testing only succeeds when the session sees leaked context, tighten the skill or the +forward-testing setup before trusting the result. + +--- + +*Adapted from Codex's skill-creator skill.* diff --git a/packages/core/src/skills/builtin/skill-creator/scripts/init_skill.mjs b/packages/core/src/skills/builtin/skill-creator/scripts/init_skill.mjs new file mode 100644 index 0000000..bc0e8b2 --- /dev/null +++ b/packages/core/src/skills/builtin/skill-creator/scripts/init_skill.mjs @@ -0,0 +1,355 @@ +#!/usr/bin/env node +/** + * Skill Initializer - Creates a new skill from template + * + * Usage: + * node init_skill.mjs --path [--resources scripts,references,assets] [--examples] + * + * Examples: + * node init_skill.mjs my-new-skill --path skills/public + * node init_skill.mjs my-new-skill --path skills/public --resources scripts,references + * node init_skill.mjs my-api-helper --path skills/private --resources scripts --examples + * node init_skill.mjs custom-skill --path /custom/location + */ + +import { access, mkdir, writeFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { parseArgs } from 'node:util' + +const MAX_SKILL_NAME_LENGTH = 64 +const ALLOWED_RESOURCES = ['scripts', 'references', 'assets'] + +const SKILL_TEMPLATE = `--- +name: {skill_name} +description: [TODO: Complete and informative explanation of what the skill does and when to use it. Include WHEN to use this skill - specific scenarios, file types, or tasks that trigger it.] +--- + +# {skill_title} + +## Overview + +[TODO: 1-2 sentences explaining what this skill enables] + +## Structuring This Skill + +[TODO: Choose the structure that best fits this skill's purpose. Common patterns: + +**1. Workflow-Based** (best for sequential processes) +- Works well when there are clear step-by-step procedures +- Example: DOCX skill with "Workflow Decision Tree" -> "Reading" -> "Creating" -> "Editing" +- Structure: ## Overview -> ## Workflow Decision Tree -> ## Step 1 -> ## Step 2... + +**2. Task-Based** (best for tool collections) +- Works well when the skill offers different operations/capabilities +- Example: PDF skill with "Quick Start" -> "Merge PDFs" -> "Split PDFs" -> "Extract Text" +- Structure: ## Overview -> ## Quick Start -> ## Task Category 1 -> ## Task Category 2... + +**3. Reference/Guidelines** (best for standards or specifications) +- Works well for brand guidelines, coding standards, or requirements +- Example: Brand styling with "Brand Guidelines" -> "Colors" -> "Typography" -> "Features" +- Structure: ## Overview -> ## Guidelines -> ## Specifications -> ## Usage... + +**4. Capabilities-Based** (best for integrated systems) +- Works well when the skill provides multiple interrelated features +- Example: Product Management with "Core Capabilities" -> numbered capability list +- Structure: ## Overview -> ## Core Capabilities -> ### 1. Feature -> ### 2. Feature... + +Patterns can be mixed and matched as needed. Most skills combine patterns (e.g., start with task-based, add workflow for complex operations). + +Delete this entire "Structuring This Skill" section when done - it's just guidance.] + +## [TODO: Replace with the first main section based on chosen structure] + +[TODO: Add content here. See examples in existing skills: +- Code samples for technical skills +- Decision trees for complex workflows +- Concrete examples with realistic user requests +- References to scripts/templates/references as needed] + +## Resources (optional) + +Create only the resource directories this skill actually needs. Delete this section if no resources are required. + +### scripts/ +Executable code (Node .mjs scripts, shell scripts, etc.) that can be run directly to perform specific operations. + +**Appropriate for:** Node scripts, shell scripts, or any executable code that performs automation, data processing, or specific operations. + +**Note:** Scripts may be executed without loading into context, but can still be read by Memo for patching or environment adjustments. + +### references/ +Documentation and reference material intended to be loaded into context to inform Memo's process and thinking. + +**Appropriate for:** In-depth documentation, API references, database schemas, comprehensive guides, or any detailed information that Memo should reference while working. + +### assets/ +Files not intended to be loaded into context, but rather used within the output Memo produces. + +**Appropriate for:** Templates, boilerplate code, document templates, images, icons, fonts, or any files meant to be copied or used in the final output. + +--- + +**Not every skill requires all three types of resources.** +` + +const EXAMPLE_SCRIPT = `#!/usr/bin/env node +/** + * Example helper script for {skill_name} + * + * This is a placeholder script that can be executed directly. + * Replace with actual implementation or delete if not needed. + */ + +console.log("This is an example script for {skill_name}") +// TODO: Add actual script logic here +// This could be data processing, file conversion, API calls, etc. +` + +const EXAMPLE_REFERENCE = `# Reference Documentation for {skill_title} + +This is a placeholder for detailed reference documentation. +Replace with actual reference content or delete if not needed. + +## When Reference Docs Are Useful + +Reference docs are ideal for: +- Comprehensive API documentation +- Detailed workflow guides +- Complex multi-step processes +- Information too lengthy for main SKILL.md +- Content that's only needed for specific use cases + +## Structure Suggestions + +### API Reference Example +- Overview +- Authentication +- Endpoints with examples +- Error codes +- Rate limits + +### Workflow Guide Example +- Prerequisites +- Step-by-step instructions +- Common patterns +- Troubleshooting +- Best practices +` + +const EXAMPLE_ASSET = `# Example Asset File + +This placeholder represents where asset files would be stored. +Replace with actual asset files (templates, images, fonts, etc.) or delete if not needed. + +Asset files are NOT intended to be loaded into context, but rather used within +the output Memo produces. + +## Common Asset Types + +- Templates: .pptx, .docx, boilerplate directories +- Images: .png, .jpg, .svg, .gif +- Fonts: .ttf, .otf, .woff, .woff2 +- Boilerplate code: Project directories, starter files +- Icons: .ico, .svg +- Data files: .csv, .json, .xml, .yaml + +Note: This is a text placeholder. Actual assets can be any file type. +` + +function normalizeSkillName(name) { + const normalized = name + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/-{2,}/g, '-') + return normalized +} + +function titleCaseSkillName(skillName) { + return skillName + .split('-') + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' ') +} + +function parseResources(rawResources) { + if (!rawResources) return [] + const resources = rawResources + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + const invalid = [...new Set(resources.filter((item) => !ALLOWED_RESOURCES.includes(item)))].sort() + if (invalid.length > 0) { + console.error(`[ERROR] Unknown resource type(s): ${invalid.join(', ')}`) + console.error(` Allowed: ${[...ALLOWED_RESOURCES].sort().join(', ')}`) + process.exit(1) + } + return [...new Set(resources)] +} + +async function createResourceDirs(skillDir, skillName, skillTitle, resources, includeExamples) { + for (const resource of resources) { + const resourceDir = join(skillDir, resource) + await mkdir(resourceDir, { recursive: true }) + if (resource === 'scripts') { + if (includeExamples) { + const example = EXAMPLE_SCRIPT.replaceAll('{skill_name}', skillName) + await writeFile(join(resourceDir, 'example.mjs'), example) + console.log('[OK] Created scripts/example.mjs') + } else { + console.log('[OK] Created scripts/') + } + } else if (resource === 'references') { + if (includeExamples) { + const example = EXAMPLE_REFERENCE.replaceAll('{skill_title}', skillTitle) + await writeFile(join(resourceDir, 'api_reference.md'), example) + console.log('[OK] Created references/api_reference.md') + } else { + console.log('[OK] Created references/') + } + } else if (resource === 'assets') { + if (includeExamples) { + await writeFile(join(resourceDir, 'example_asset.txt'), EXAMPLE_ASSET) + console.log('[OK] Created assets/example_asset.txt') + } else { + console.log('[OK] Created assets/') + } + } + } +} + +async function initSkill(skillName, path, resources, includeExamples) { + const skillDir = join(resolve(path), skillName) + + try { + await access(skillDir) + console.error(`[ERROR] Skill directory already exists: ${skillDir}`) + return null + } catch { + // directory does not exist, proceed + } + + try { + await mkdir(skillDir, { recursive: false }) + console.log(`[OK] Created skill directory: ${skillDir}`) + } catch (error) { + console.error(`[ERROR] Error creating directory: ${error.message}`) + return null + } + + const skillTitle = titleCaseSkillName(skillName) + const skillContent = SKILL_TEMPLATE.replaceAll('{skill_name}', skillName).replaceAll('{skill_title}', skillTitle) + + try { + await writeFile(join(skillDir, 'SKILL.md'), skillContent) + console.log('[OK] Created SKILL.md') + } catch (error) { + console.error(`[ERROR] Error creating SKILL.md: ${error.message}`) + return null + } + + if (resources.length > 0) { + try { + await createResourceDirs(skillDir, skillName, skillTitle, resources, includeExamples) + } catch (error) { + console.error(`[ERROR] Error creating resource directories: ${error.message}`) + return null + } + } + + console.log(`\n[OK] Skill '${skillName}' initialized successfully at ${skillDir}`) + console.log('\nNext steps:') + console.log('1. Edit SKILL.md to complete the TODO items and update the description') + if (resources.length > 0) { + if (includeExamples) { + console.log('2. Customize or delete the example files in scripts/, references/, and assets/') + } else { + console.log('2. Add resources to scripts/, references/, and assets/ as needed') + } + } else { + console.log('2. Create resource directories only if needed (scripts/, references/, assets/)') + } + console.log('3. Run the validator when ready to check the skill structure') + console.log('4. Forward-test complex skills with realistic user requests to ensure they work as intended') + + return skillDir +} + +async function main() { + let parsed + try { + parsed = parseArgs({ + args: process.argv.slice(2), + allowPositionals: true, + options: { + path: { type: 'string' }, + resources: { type: 'string', default: '' }, + examples: { type: 'boolean', default: false }, + }, + }) + } catch (error) { + console.error(`[ERROR] ${error.message}`) + console.error( + 'Usage: node init_skill.mjs --path [--resources scripts,references,assets] [--examples]', + ) + process.exit(1) + } + + const rawSkillName = parsed.positionals[0] + if (!rawSkillName) { + console.error('[ERROR] Skill name is required.') + console.error( + 'Usage: node init_skill.mjs --path [--resources scripts,references,assets] [--examples]', + ) + process.exit(1) + } + + const skillName = normalizeSkillName(rawSkillName) + if (!skillName) { + console.error('[ERROR] Skill name must include at least one letter or digit.') + process.exit(1) + } + if (skillName.length > MAX_SKILL_NAME_LENGTH) { + console.error( + `[ERROR] Skill name '${skillName}' is too long (${skillName.length} characters). Maximum is ${MAX_SKILL_NAME_LENGTH} characters.`, + ) + process.exit(1) + } + if (skillName !== rawSkillName.trim()) { + console.log(`Note: Normalized skill name from '${rawSkillName}' to '${skillName}'.`) + } + + const resources = parseResources(parsed.values.resources) + if (parsed.values.examples && resources.length === 0) { + console.error('[ERROR] --examples requires --resources to be set.') + process.exit(1) + } + + const path = parsed.values.path + if (!path) { + console.error('[ERROR] --path is required.') + console.error( + 'Usage: node init_skill.mjs --path [--resources scripts,references,assets] [--examples]', + ) + process.exit(1) + } + + console.log(`Initializing skill: ${skillName}`) + console.log(` Location: ${path}`) + if (resources.length > 0) { + console.log(` Resources: ${resources.join(', ')}`) + if (parsed.values.examples) { + console.log(' Examples: enabled') + } + } else { + console.log(' Resources: none (create as needed)') + } + console.log() + + const result = await initSkill(skillName, path, resources, parsed.values.examples) + process.exit(result ? 0 : 1) +} + +await main() diff --git a/packages/core/src/skills/builtin/skill-creator/scripts/quick_validate.mjs b/packages/core/src/skills/builtin/skill-creator/scripts/quick_validate.mjs new file mode 100644 index 0000000..7928bb3 --- /dev/null +++ b/packages/core/src/skills/builtin/skill-creator/scripts/quick_validate.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node +/** + * Quick validation script for skills - minimal version + * + * Usage: + * node quick_validate.mjs + * + * Semantics align with Memo's skill loader (parseSkillFile in skills.ts): + * a skill that passes this check is guaranteed to be loadable by Memo. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +const MAX_SKILL_NAME_LENGTH = 64 +const ALLOWED_PROPERTIES = ['name', 'description', 'license', 'allowed-tools', 'metadata'] + +function unquote(value) { + const trimmed = value.trim() + if (trimmed.length >= 2) { + const first = trimmed[0] + const last = trimmed[trimmed.length - 1] + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return trimmed.slice(1, -1) + } + } + return trimmed +} + +async function validateSkill(skillPath) { + const skillMdPath = join(skillPath, 'SKILL.md') + let content + try { + content = await readFile(skillMdPath, 'utf8') + } catch { + return { valid: false, message: 'SKILL.md not found' } + } + + if (!content.startsWith('---')) { + return { valid: false, message: 'No YAML frontmatter found' } + } + + const match = content.match(/^---\n(.*?)\n---/s) + if (!match) { + return { valid: false, message: 'Invalid frontmatter format' } + } + + // Parse simple key: value lines (matches the flat frontmatter Memo supports). + const frontmatter = {} + for (const line of match[1].split('\n')) { + if (!line.trim() || line.trim().startsWith('#')) continue + const m = line.match(/^([A-Za-z0-9_-]+)\s*:\s*(.*)$/) + if (!m) { + return { valid: false, message: `Invalid YAML in frontmatter: '${line.trim()}'` } + } + frontmatter[m[1]] = unquote(m[2]) + } + + const unexpectedKeys = Object.keys(frontmatter).filter((key) => !ALLOWED_PROPERTIES.includes(key)) + if (unexpectedKeys.length > 0) { + const allowed = [...ALLOWED_PROPERTIES].sort().join(', ') + return { + valid: false, + message: `Unexpected key(s) in SKILL.md frontmatter: ${unexpectedKeys.sort().join(', ')}. Allowed properties are: ${allowed}`, + } + } + + if (!('name' in frontmatter)) { + return { valid: false, message: "Missing 'name' in frontmatter" } + } + if (!('description' in frontmatter)) { + return { valid: false, message: "Missing 'description' in frontmatter" } + } + + const name = frontmatter.name.trim() + if (name) { + if (!/^[a-z0-9-]+$/.test(name)) { + return { + valid: false, + message: `Name '${name}' should be hyphen-case (lowercase letters, digits, and hyphens only)`, + } + } + if (name.startsWith('-') || name.endsWith('-') || name.includes('--')) { + return { + valid: false, + message: `Name '${name}' cannot start/end with hyphen or contain consecutive hyphens`, + } + } + if (name.length > MAX_SKILL_NAME_LENGTH) { + return { + valid: false, + message: `Name is too long (${name.length} characters). Maximum is ${MAX_SKILL_NAME_LENGTH} characters.`, + } + } + } + + const description = frontmatter.description.trim() + if (description) { + if (description.includes('<') || description.includes('>')) { + return { valid: false, message: 'Description cannot contain angle brackets (< or >)' } + } + if (description.length > 1024) { + return { + valid: false, + message: `Description is too long (${description.length} characters). Maximum is 1024 characters.`, + } + } + } + + return { valid: true, message: 'Skill is valid!' } +} + +const skillPath = process.argv[2] +if (!skillPath) { + console.error('Usage: node quick_validate.mjs ') + process.exit(1) +} + +const { valid, message } = await validateSkill(skillPath) +console.log(message) +process.exit(valid ? 0 : 1) diff --git a/packages/core/src/skills/builtin_scripts.test.ts b/packages/core/src/skills/builtin_scripts.test.ts new file mode 100644 index 0000000..a2341c8 --- /dev/null +++ b/packages/core/src/skills/builtin_scripts.test.ts @@ -0,0 +1,204 @@ +import assert from 'node:assert' +import { execFile } from 'node:child_process' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { describe, test } from 'vitest' + +const execFileAsync = promisify(execFile) + +const SCRIPTS_DIR = new URL('./builtin/skill-creator/scripts/', import.meta.url).pathname +const INIT_SCRIPT = join(SCRIPTS_DIR, 'init_skill.mjs') +const VALIDATE_SCRIPT = join(SCRIPTS_DIR, 'quick_validate.mjs') + +async function makeTempDir(prefix: string) { + const dir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`) + await mkdir(dir, { recursive: true }) + return dir +} + +async function removeDir(path: string) { + await rm(path, { recursive: true, force: true }) +} + +async function runScript(script: string, args: string[], cwd?: string) { + try { + const { stdout } = await execFileAsync(process.execPath, [script, ...args], { cwd }) + return { code: 0, stdout } + } catch (error) { + const e = error as { code?: number; stdout?: string; stderr?: string } + return { code: e.code ?? 1, stdout: `${e.stdout ?? ''}${e.stderr ?? ''}` } + } +} + +describe('init_skill.mjs', () => { + test('creates a normalized skill directory with SKILL.md and example resources', async () => { + const outDir = await makeTempDir('memo-scripts-init') + try { + const { code, stdout } = await runScript(INIT_SCRIPT, [ + 'My Great Skill', + '--path', + outDir, + '--resources', + 'scripts,references', + '--examples', + ]) + assert.strictEqual(code, 0) + assert.match(stdout, /\[OK\] Created SKILL\.md/) + assert.match(stdout, /\[OK\] Created scripts\/example\.mjs/) + + const skillDir = join(outDir, 'my-great-skill') + const skillMd = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.match(skillMd, /^---\nname: my-great-skill/) + assert.ok((await readFile(join(skillDir, 'scripts', 'example.mjs'), 'utf8')).includes('my-great-skill')) + assert.ok((await readFile(join(skillDir, 'references', 'api_reference.md'), 'utf8')).length > 0) + // assets was not requested. + await assert.rejects(readFile(join(skillDir, 'assets', 'example_asset.txt'), 'utf8')) + } finally { + await removeDir(outDir) + } + }) + + test('rejects an existing skill directory', async () => { + const outDir = await makeTempDir('memo-scripts-init') + try { + await mkdir(join(outDir, 'taken'), { recursive: true }) + const { code, stdout } = await runScript(INIT_SCRIPT, ['taken', '--path', outDir]) + assert.strictEqual(code, 1) + assert.match(stdout, /already exists/) + } finally { + await removeDir(outDir) + } + }) + + test('requires --path', async () => { + const { code, stdout } = await runScript(INIT_SCRIPT, ['some-skill']) + assert.strictEqual(code, 1) + assert.match(stdout, /--path is required/) + }) + + test('rejects unknown resource types', async () => { + const outDir = await makeTempDir('memo-scripts-init') + try { + const { code, stdout } = await runScript(INIT_SCRIPT, [ + 'ok-skill', + '--path', + outDir, + '--resources', + 'scripts,bogus', + ]) + assert.strictEqual(code, 1) + assert.match(stdout, /Unknown resource type/) + } finally { + await removeDir(outDir) + } + }) + + test('rejects names longer than 64 characters', async () => { + const outDir = await makeTempDir('memo-scripts-init') + try { + const longName = 'a'.repeat(65) + const { code, stdout } = await runScript(INIT_SCRIPT, [longName, '--path', outDir]) + assert.strictEqual(code, 1) + assert.match(stdout, /too long/) + } finally { + await removeDir(outDir) + } + }) + + test('requires --resources when --examples is set', async () => { + const outDir = await makeTempDir('memo-scripts-init') + try { + const { code, stdout } = await runScript(INIT_SCRIPT, ['ok-skill', '--path', outDir, '--examples']) + assert.strictEqual(code, 1) + assert.match(stdout, /--examples requires --resources/) + } finally { + await removeDir(outDir) + } + }) +}) + +describe('quick_validate.mjs', () => { + async function writeSkill(skillMd: string): Promise { + const dir = await makeTempDir('memo-scripts-validate') + await writeFile(join(dir, 'SKILL.md'), skillMd, 'utf8') + return dir + } + + test('passes the output of init_skill.mjs', async () => { + const outDir = await makeTempDir('memo-scripts-validate') + try { + const init = await runScript(INIT_SCRIPT, [ + 'demo-skill', + '--path', + outDir, + '--resources', + 'scripts', + '--examples', + ]) + assert.strictEqual(init.code, 0) + + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [join(outDir, 'demo-skill')]) + assert.strictEqual(code, 0) + assert.match(stdout, /Skill is valid!/) + } finally { + await removeDir(outDir) + } + }) + + test('rejects a missing description', async () => { + const dir = await writeSkill('---\nname: ok\n---\n# body\n') + try { + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [dir]) + assert.strictEqual(code, 1) + assert.match(stdout, /Missing 'description'/) + } finally { + await removeDir(dir) + } + }) + + test('rejects a name with uppercase letters', async () => { + const dir = await writeSkill('---\nname: MySkill\ndescription: d\n---\n') + try { + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [dir]) + assert.strictEqual(code, 1) + assert.match(stdout, /hyphen-case/) + } finally { + await removeDir(dir) + } + }) + + test('rejects angle brackets in the description', async () => { + const dir = await writeSkill('---\nname: ok\ndescription: "a < b"\n---\n') + try { + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [dir]) + assert.strictEqual(code, 1) + assert.match(stdout, /angle brackets/) + } finally { + await removeDir(dir) + } + }) + + test('rejects unexpected frontmatter keys', async () => { + const dir = await writeSkill('---\nname: ok\ndescription: d\nfoo: bar\n---\n') + try { + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [dir]) + assert.strictEqual(code, 1) + assert.match(stdout, /Unexpected key/) + } finally { + await removeDir(dir) + } + }) + + test('reports a missing SKILL.md', async () => { + const dir = await makeTempDir('memo-scripts-validate') + try { + const { code, stdout } = await runScript(VALIDATE_SCRIPT, [dir]) + assert.strictEqual(code, 1) + assert.match(stdout, /SKILL\.md not found/) + } finally { + await removeDir(dir) + } + }) +}) diff --git a/packages/core/src/skills/builtin_skills.test.ts b/packages/core/src/skills/builtin_skills.test.ts new file mode 100644 index 0000000..9b3b3b0 --- /dev/null +++ b/packages/core/src/skills/builtin_skills.test.ts @@ -0,0 +1,180 @@ +import assert from 'node:assert' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, test } from 'vitest' +import { BUILTIN_MARKER, installBuiltinSkills, resolveBuiltinRoot } from '@memo/core/skills/builtin_skills' + +const SKILL_MD_V1 = `--- +name: skill-creator +description: v1 description +--- +# v1 body +` + +const SKILL_MD_V2 = `--- +name: skill-creator +description: v2 description +--- +# v2 body +` + +const INIT_SCRIPT_V1 = "export const version = 'v1'\n" +const INIT_SCRIPT_V2 = "export const version = 'v2'\n" + +async function makeTempDir(prefix: string) { + const dir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`) + await mkdir(dir, { recursive: true }) + return dir +} + +async function removeDir(path: string) { + await rm(path, { recursive: true, force: true }) +} + +async function makeSource(skillName: string, skillMd: string, initScript: string) { + const root = await makeTempDir('memo-builtin-src') + const skillDir = join(root, skillName) + await mkdir(join(skillDir, 'scripts'), { recursive: true }) + await writeFile(join(skillDir, 'SKILL.md'), skillMd, 'utf8') + await writeFile(join(skillDir, 'scripts', 'init_skill.mjs'), initScript, 'utf8') + return root +} + +describe('installBuiltinSkills', () => { + test('installs missing skills with a marker recording the tree fingerprint', async () => { + const memoHome = await makeTempDir('memo-builtin-home') + const source = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const skillDir = join(memoHome, 'skills', 'skill-creator') + const installed = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.strictEqual(installed, SKILL_MD_V1) + const installedScript = await readFile(join(skillDir, 'scripts', 'init_skill.mjs'), 'utf8') + assert.strictEqual(installedScript, INIT_SCRIPT_V1) + + const marker = await readFile(join(skillDir, BUILTIN_MARKER), 'utf8') + assert.match(marker, /^[0-9a-f]{64}$/, 'marker should be a sha256 hex fingerprint') + } finally { + await removeDir(memoHome) + await removeDir(source) + } + }) + + test('is idempotent: unchanged trees are left untouched on reinstall', async () => { + const memoHome = await makeTempDir('memo-builtin-home') + const source = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + await installBuiltinSkills({ memoHome, sourceRoot: source }) + const skillDir = join(memoHome, 'skills', 'skill-creator') + const markerPath = join(skillDir, BUILTIN_MARKER) + const markerBefore = await readFile(markerPath, 'utf8') + + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const markerAfter = await readFile(markerPath, 'utf8') + assert.strictEqual(markerAfter, markerBefore) + const installed = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.strictEqual(installed, SKILL_MD_V1) + } finally { + await removeDir(memoHome) + await removeDir(source) + } + }) + + test('skips directories modified by the user', async () => { + const memoHome = await makeTempDir('memo-builtin-home') + const source = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const skillDir = join(memoHome, 'skills', 'skill-creator') + const userEdit = '# my own edit\n' + await writeFile(join(skillDir, 'SKILL.md'), userEdit, 'utf8') + const markerBefore = await readFile(join(skillDir, BUILTIN_MARKER), 'utf8') + + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const installed = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.strictEqual(installed, userEdit, 'user edit must survive reinstall') + const markerAfter = await readFile(join(skillDir, BUILTIN_MARKER), 'utf8') + assert.strictEqual(markerAfter, markerBefore, 'marker must not be rewritten') + } finally { + await removeDir(memoHome) + await removeDir(source) + } + }) + + test('upgrades untouched copies from an older release, including bundled scripts', async () => { + const memoHome = await makeTempDir('memo-builtin-home') + const sourceV1 = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + await installBuiltinSkills({ memoHome, sourceRoot: sourceV1 }) + + // Simulate a newer release: new content in SKILL.md AND in scripts. + const sourceV2 = await makeSource('skill-creator', SKILL_MD_V2, INIT_SCRIPT_V2) + await installBuiltinSkills({ memoHome, sourceRoot: sourceV2 }) + + const skillDir = join(memoHome, 'skills', 'skill-creator') + const installed = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.strictEqual(installed, SKILL_MD_V2, 'untouched old copy should be upgraded') + const installedScript = await readFile(join(skillDir, 'scripts', 'init_skill.mjs'), 'utf8') + assert.strictEqual(installedScript, INIT_SCRIPT_V2, 'script changes must propagate too') + + const marker = await readFile(join(skillDir, BUILTIN_MARKER), 'utf8') + // Reinstalling the new source must now be a no-op. + await installBuiltinSkills({ memoHome, sourceRoot: sourceV2 }) + const markerAfter = await readFile(join(skillDir, BUILTIN_MARKER), 'utf8') + assert.strictEqual(markerAfter, marker) + await removeDir(sourceV2) + } finally { + await removeDir(memoHome) + await removeDir(sourceV1) + } + }) + + test('does not touch externally created directories without a marker', async () => { + const memoHome = await makeTempDir('memo-builtin-home') + const source = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + const skillDir = join(memoHome, 'skills', 'skill-creator') + await mkdir(skillDir, { recursive: true }) + const foreign = '# foreign skill\n' + await writeFile(join(skillDir, 'SKILL.md'), foreign, 'utf8') + + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const installed = await readFile(join(skillDir, 'SKILL.md'), 'utf8') + assert.strictEqual(installed, foreign, 'foreign install must not be overwritten') + } finally { + await removeDir(memoHome) + await removeDir(source) + } + }) + + test('creates memoHome/skills when memoHome does not exist yet', async () => { + const sandbox = await makeTempDir('memo-builtin-sandbox') + const source = await makeSource('skill-creator', SKILL_MD_V1, INIT_SCRIPT_V1) + try { + const memoHome = join(sandbox, 'nested', 'home') + await installBuiltinSkills({ memoHome, sourceRoot: source }) + + const installed = await readFile(join(memoHome, 'skills', 'skill-creator', 'SKILL.md'), 'utf8') + assert.strictEqual(installed, SKILL_MD_V1) + } finally { + await removeDir(sandbox) + await removeDir(source) + } + }) + + test('resolveBuiltinRoot finds the in-repo builtin tree', async () => { + const root = resolveBuiltinRoot() + // In the source tree, the builtin skill lives next to this module. + const moduleDir = dirname(fileURLToPath(import.meta.url)) + const expected = join(moduleDir, 'builtin') + assert.strictEqual(root, expected) + assert.ok((await readFile(join(root, 'skill-creator', 'SKILL.md'), 'utf8')).startsWith('---')) + }) +}) diff --git a/packages/core/src/skills/builtin_skills.ts b/packages/core/src/skills/builtin_skills.ts new file mode 100644 index 0000000..0de5754 --- /dev/null +++ b/packages/core/src/skills/builtin_skills.ts @@ -0,0 +1,113 @@ +import { createHash } from 'node:crypto' +import { existsSync } from 'node:fs' +import { cp, mkdir, readFile, readdir, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +export const BUILTIN_SKILLS = ['skill-creator'] as const + +/** + * Marker file written into an installed builtin skill directory. Records the + * fingerprint of the tree that was installed, so a later run can tell + * "untouched copy from an older release" (upgrade it) from "user modified + * or externally created" (never touch it). Not named like a SKILL.md, so the + * skills scan never picks it up. + */ +export const BUILTIN_MARKER = '.memo-builtin.json' + +export type InstallBuiltinSkillsOptions = { + memoHome: string + /** Test injection; defaults to probing the packaged builtin directory. */ + sourceRoot?: string +} + +/** + * Idempotently install Memo's builtin skills into $MEMO_HOME/skills before + * the skills scan. Three states per skill: + * + * - missing -> fresh install (copy + write marker) + * - identical tree -> no-op + * - different tree: marker matches current tree (untouched copy of an older + * release) -> overwrite upgrade; otherwise (user-modified or foreign) -> skip + */ +export async function installBuiltinSkills(options: InstallBuiltinSkillsOptions): Promise { + const builtinRoot = options.sourceRoot ?? resolveBuiltinRoot() + const skillsDir = join(options.memoHome, 'skills') + + for (const name of BUILTIN_SKILLS) { + const src = join(builtinRoot, name) + const dest = join(skillsDir, name) + const markerPath = join(dest, BUILTIN_MARKER) + const fingerprint = await treeFingerprint(src) + + if (!existsSync(join(dest, 'SKILL.md'))) { + await mkdir(dest, { recursive: true }) + await cp(src, dest, { recursive: true }) + await writeFile(markerPath, fingerprint, 'utf8') + continue + } + + const destFingerprint = await treeFingerprint(dest) + if (destFingerprint === fingerprint) { + continue + } + + let marker: string | null = null + try { + marker = (await readFile(markerPath, 'utf8')).trim() + } catch { + marker = null + } + if (marker === destFingerprint) { + // Untouched copy from an older release: upgrade it. + await cp(src, dest, { recursive: true }) + await writeFile(markerPath, fingerprint, 'utf8') + } + // Otherwise the directory was modified by the user or installed by + // something else - leave it alone. + } +} + +/** + * Locate the builtin skills source tree. Dev (tsx) runs from + * src/skills/builtin_skills.ts; the packed dist/index.js has the module + * directory at dist/, with the tree copied to dist/skills/builtin by tsup. + */ +export function resolveBuiltinRoot(): string { + const moduleDir = dirname(fileURLToPath(import.meta.url)) + const candidates = [join(moduleDir, 'builtin'), join(moduleDir, '..', 'skills', 'builtin')] + for (const candidate of candidates) { + if (existsSync(join(candidate, 'skill-creator', 'SKILL.md'))) { + return candidate + } + } + throw new Error(`builtin skills directory not found (tried: ${candidates.join(', ')})`) +} + +async function treeFingerprint(dir: string): Promise { + const files: string[] = [] + await collectFiles(dir, '', files) + files.sort() + + const root = createHash('sha256') + for (const relPath of files) { + const content = await readFile(join(dir, relPath)) + const fileHash = createHash('sha256').update(content).digest('hex') + root.update(`${relPath}\0${fileHash}\n`) + } + return root.digest('hex') +} + +async function collectFiles(dir: string, prefix: string, out: string[]): Promise { + const entries = await readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + if (entry.name === BUILTIN_MARKER) continue + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name + const fullPath = join(dir, entry.name) + if (entry.isDirectory()) { + await collectFiles(fullPath, relPath, out) + } else if (entry.isFile()) { + out.push(relPath) + } + } +} diff --git a/packages/core/src/skills/skills.test.ts b/packages/core/src/skills/skills.test.ts new file mode 100644 index 0000000..0b926da --- /dev/null +++ b/packages/core/src/skills/skills.test.ts @@ -0,0 +1,340 @@ +import assert from 'node:assert' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, test } from 'vitest' +import { + DEFAULT_SKILLS_BUDGET_CHARS, + buildSkillIndex, + findSkillByName, + findSkillByPath, + loadSkills, + readSkillBody, + renderSkillsSection, + stripFrontmatter, +} from '@memo/core/skills/skills' + +async function makeTempDir(prefix: string) { + const dir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`) + await mkdir(dir, { recursive: true }) + return dir +} + +async function removeDir(path: string) { + await rm(path, { recursive: true, force: true }) +} + +async function writeSkill(skillRoot: string, skillName: string, description: string, body?: string) { + const skillDir = join(skillRoot, skillName) + const skillPath = join(skillDir, 'SKILL.md') + await mkdir(skillDir, { recursive: true }) + await writeFile( + skillPath, + `--- +name: ${skillName} +description: ${description} +--- +${body ?? `# ${skillName}\n`}`, + 'utf-8', + ) + return skillPath +} + +describe('skills discovery', () => { + test('discovers project .xxx/skills plus user-level ~/.memo, ~/.claude, ~/.codex, ~/.agents skills', async () => { + const sandbox = await makeTempDir('memo-core-skills-discovery') + const projectRoot = join(sandbox, 'repo') + const nestedCwd = join(projectRoot, 'packages', 'core') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(nestedCwd, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + await writeSkill(join(projectRoot, '.agents', 'skills'), 'memo-default', 'memo default') + await writeSkill(join(projectRoot, '.claude', 'skills'), 'claude-compat', 'claude compat') + await writeSkill(join(projectRoot, '.codex', 'skills'), 'codex-compat', 'codex compat') + await writeSkill(join(memoHome, 'skills'), 'memo-global', 'memo global') + + // User-level global directories must be discovered. + await writeSkill(join(homeDir, '.agents', 'skills'), 'home-agents', 'home agents') + await writeSkill(join(homeDir, '.codex', 'skills'), 'home-codex', 'home codex') + await writeSkill(join(homeDir, '.claude', 'skills'), 'home-claude', 'home claude') + + try { + const discovered = await loadSkills({ cwd: nestedCwd, homeDir, memoHome }) + const names = new Set(discovered.map((skill) => skill.name)) + + assert.ok(names.has('memo-default')) + assert.ok(names.has('claude-compat')) + assert.ok(names.has('codex-compat')) + assert.ok(names.has('memo-global')) + assert.ok(names.has('home-agents')) + assert.ok(names.has('home-codex')) + assert.ok(names.has('home-claude')) + } finally { + await removeDir(sandbox) + } + }) + + test('falls back to cwd when no git root exists', async () => { + const sandbox = await makeTempDir('memo-core-skills-no-git') + const parentDir = join(sandbox, 'parent') + const cwd = join(parentDir, 'child') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(cwd, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + + await writeSkill(join(parentDir, '.agents', 'skills'), 'parent-skill', 'parent level') + await writeSkill(join(cwd, '.agents', 'skills'), 'cwd-skill', 'cwd level') + + try { + const discovered = await loadSkills({ cwd, homeDir, memoHome }) + const names = new Set(discovered.map((skill) => skill.name)) + + assert.ok(names.has('cwd-skill')) + assert.ok(!names.has('parent-skill')) + } finally { + await removeDir(sandbox) + } + }) +}) + +describe('skills dedup', () => { + test('identical SKILL.md in project and user roots dedupes to a single record, project wins', async () => { + const sandbox = await makeTempDir('memo-core-skills-dedup-project') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + const projectPath = await writeSkill(join(projectRoot, '.agents', 'skills'), 'shared', 'same skill') + const homePath = await writeSkill(join(homeDir, '.claude', 'skills'), 'shared', 'same skill') + + try { + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.strictEqual(discovered.length, 1) + const [skill] = discovered + assert.ok(skill) + assert.strictEqual(skill.path, projectPath) + assert.deepStrictEqual(skill.paths, [projectPath, homePath]) + assert.strictEqual(skill.scope, 'project') + } finally { + await removeDir(sandbox) + } + }) + + test('identical SKILL.md across user roots keeps memo home winner', async () => { + const sandbox = await makeTempDir('memo-core-skills-dedup-user') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + const memoPath = await writeSkill(join(memoHome, 'skills'), 'dup', 'duplicate skill') + const claudePath = await writeSkill(join(homeDir, '.claude', 'skills'), 'dup', 'duplicate skill') + const codexPath = await writeSkill(join(homeDir, '.codex', 'skills'), 'dup', 'duplicate skill') + const agentsPath = await writeSkill(join(homeDir, '.agents', 'skills'), 'dup', 'duplicate skill') + + try { + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.strictEqual(discovered.length, 1) + const [skill] = discovered + assert.ok(skill) + assert.strictEqual(skill.path, memoPath) + assert.deepStrictEqual(skill.paths, [memoPath, claudePath, codexPath, agentsPath]) + assert.strictEqual(skill.scope, 'global') + } finally { + await removeDir(sandbox) + } + }) + + test('identical SKILL.md within one root dedupes lexicographically', async () => { + const sandbox = await makeTempDir('memo-core-skills-dedup-same-root') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + const aPath = await writeSkill(join(projectRoot, '.agents', 'skills'), 'dup-body', 'same body') + // Second directory holds a byte-identical SKILL.md (same name). + const zDir = join(projectRoot, '.agents', 'skills', 'z-skill') + await mkdir(zDir, { recursive: true }) + const zPath = join(zDir, 'SKILL.md') + await writeFile(zPath, await readFile(aPath, 'utf-8'), 'utf-8') + + try { + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.strictEqual(discovered.length, 1) + const [skill] = discovered + assert.ok(skill) + assert.strictEqual(skill.path, aPath) + assert.deepStrictEqual(skill.paths, [aPath, zPath]) + } finally { + await removeDir(sandbox) + } + }) + + test('same name with different content coexists', async () => { + const sandbox = await makeTempDir('memo-core-skills-name-clash') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + await writeSkill(join(projectRoot, '.agents', 'skills'), 'clash', 'project version') + await writeSkill(join(memoHome, 'skills'), 'clash', 'global version') + + try { + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.strictEqual(discovered.length, 2) + assert.deepStrictEqual(discovered.map((s) => s.description).sort(), ['global version', 'project version']) + + const index = buildSkillIndex(discovered) + assert.strictEqual(findSkillByName(index, 'clash').length, 2) + } finally { + await removeDir(sandbox) + } + }) +}) + +describe('skills index and reading', () => { + test('index resolves deduped-away copies by path', async () => { + const sandbox = await makeTempDir('memo-core-skills-index') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + + const projectPath = await writeSkill(join(projectRoot, '.agents', 'skills'), 'shared', 'same skill') + const homePath = await writeSkill(join(homeDir, '.claude', 'skills'), 'shared', 'same skill') + + try { + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + const index = buildSkillIndex(discovered) + + const byWinner = findSkillByPath(index, projectPath) + assert.strictEqual(byWinner?.name, 'shared') + const byCopy = findSkillByPath(index, homePath) + assert.strictEqual(byCopy?.name, 'shared') + assert.strictEqual(byCopy?.path, projectPath) + } finally { + await removeDir(sandbox) + } + }) + + test('readSkillBody strips frontmatter', async () => { + const sandbox = await makeTempDir('memo-core-skills-read') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + await writeSkill(join(projectRoot, '.agents', 'skills'), 'bod', 'body skill', '# Bod\n\nDetails here.\n') + + try { + const [skill] = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.ok(skill) + const body = await readSkillBody(skill) + assert.ok(!body.includes('---')) + assert.ok(body.includes('Details here.')) + assert.ok(!body.includes('description: body skill')) + } finally { + await removeDir(sandbox) + } + }) + + test('stripFrontmatter handles missing frontmatter', () => { + assert.strictEqual(stripFrontmatter('# Plain\n\ncontent\n'), '# Plain\n\ncontent') + assert.strictEqual(stripFrontmatter('---\nname: x\n---\n# Body\n'), '# Body') + }) +}) + +describe('skills directory rendering', () => { + test('renders name + description without file paths', async () => { + const sandbox = await makeTempDir('memo-core-skills-render') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + + await mkdir(projectRoot, { recursive: true }) + await mkdir(homeDir, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + const skillPath = await writeSkill(join(projectRoot, '.agents', 'skills'), 'fmt', 'formatting helper') + + try { + const [skill] = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + assert.ok(skill) + const section = renderSkillsSection([skill]) + assert.ok(section) + assert.ok(section.includes('- fmt: formatting helper')) + assert.ok(!section.includes(skillPath)) + assert.ok(section.includes('read_skill')) + } finally { + await removeDir(sandbox) + } + }) + + test('stays within budget by truncating descriptions and omitting the tail', () => { + // Long names make each entry exceed its fair share of the budget, so + // descriptions get truncated and low-priority entries get dropped. + const skills = Array.from({ length: 50 }, (_, i) => ({ + name: `${String(i).padStart(3, '0')}-${'n'.repeat(47)}`, + description: `description number ${i} `.repeat(30), // long on purpose + path: `/tmp/skill-${i}/SKILL.md`, + paths: [`/tmp/skill-${i}/SKILL.md`], + hash: `h${i}`, + scope: 'global' as const, + sourceRoot: '/tmp', + })) + + const section = renderSkillsSection(skills) + assert.ok(section) + assert.ok(section.length <= DEFAULT_SKILLS_BUDGET_CHARS) + assert.ok(section.includes('omitted due to context budget')) + assert.ok(section.includes('...'), 'truncated descriptions should end with ...') + }) + + test('omits all entries when budget cannot fit any entry', () => { + const skills = [ + { + name: 'x', + description: 'y', + path: '/tmp/x/SKILL.md', + paths: ['/tmp/x/SKILL.md'], + hash: 'h1', + scope: 'global' as const, + sourceRoot: '/tmp', + }, + ] + // Budget below the fixed header/rules cost: every entry is omitted. + const section = renderSkillsSection(skills, { budgetChars: 100 }) + assert.ok(section) + assert.ok(section.includes('1 more skills omitted')) + assert.ok(!section.includes('- x: y')) + }) + + test('returns null for empty skills', () => { + assert.strictEqual(renderSkillsSection([]), null) + }) +}) diff --git a/packages/core/src/runtime/skills.ts b/packages/core/src/skills/skills.ts similarity index 53% rename from packages/core/src/runtime/skills.ts rename to packages/core/src/skills/skills.ts index d35645c..fcd4af7 100644 --- a/packages/core/src/runtime/skills.ts +++ b/packages/core/src/skills/skills.ts @@ -1,13 +1,34 @@ import { access, readFile, readdir, stat } from 'node:fs/promises' import { constants as fsConstants } from 'node:fs' +import { createHash } from 'node:crypto' import { homedir } from 'node:os' -import { dirname, isAbsolute, join, resolve } from 'node:path' +import { dirname, isAbsolute, join, resolve, sep } from 'node:path' import fg from 'fast-glob' +export type SkillScope = 'project' | 'global' + export type SkillMetadata = { name: string description: string + /** Winning path (first by root priority) for the deduped skill. */ path: string + /** All absolute SKILL.md paths with identical content, winner first. */ + paths: string[] + /** sha256 of the SKILL.md content; dedup key. */ + hash: string + scope: SkillScope + /** Root directory the skill was discovered from. */ + sourceRoot: string +} + +/** Index over a deduped skill snapshot, used by read_skill and the CLI. */ +export type SkillIndex = { + list: SkillMetadata[] + /** name → all records (same name with different content coexists). */ + byName: Map + /** resolved absolute path → record (includes deduped-away copies). */ + byPath: Map + byHash: Map } type LoadSkillsOptions = { @@ -24,12 +45,18 @@ const DEFAULT_MAX_SKILLS = 200 const MAX_NAME_LEN = 64 const MAX_DESCRIPTION_LEN = 1024 -const SKILLS_USAGE_RULES = `- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths. +/** Context budget for the skills directory in the system prompt (fixed fallback). */ +export const DEFAULT_SKILLS_BUDGET_CHARS = 4096 + +/** Smallest entry worth keeping in the skills directory: "- a: b" shape. */ +const MIN_ENTRY_CHARS = 8 + +const SKILLS_USAGE_RULES = `- Discovery: The list above is the skills available in this session (name + description). To use a skill, call the \`read_skill\` tool with its name (or its SKILL.md path when names are ambiguous) to load the full SKILL.md instructions. - Trigger rules: If the user names a skill (with \`$SkillName\` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned. -- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback. +- Missing/blocked: If a named skill isn't in the list or the read_skill call fails, say so briefly and continue with the best fallback. - How to use a skill (progressive disclosure): - 1) After deciding to use a skill, open its \`SKILL.md\`. Read only enough to follow the workflow. - 2) When \`SKILL.md\` references relative paths (e.g., \`scripts/foo.py\`), resolve them relative to the skill directory listed above first, and only consider other paths if needed. + 1) After deciding to use a skill, call \`read_skill\` with the skill name. Read only enough to follow the workflow. + 2) When \`SKILL.md\` references relative paths (e.g., \`scripts/foo.py\`), resolve them relative to the skill directory returned by \`read_skill\` first, and only consider other paths if needed. 3) If \`SKILL.md\` points to extra folders such as \`references/\`, load only the specific files needed for the request; don't bulk-load everything. 4) If \`scripts/\` exist, prefer running or patching them instead of retyping large code blocks. 5) If \`assets/\` or templates exist, reuse them instead of recreating from scratch. @@ -88,7 +115,8 @@ function parseMultilineValue(frontmatter: string, key: string): string | null { const collected: string[] = [] for (const line of lines) { if (!inBlock) { - const match = line.match(new RegExp(`^${key}\\s*:\\s*[|>]\\s*$`)) + // YAML block scalars: "key: >", "key: >-", "key: |", "key: |-". + const match = line.match(new RegExp(`^${key}\\s*:\\s*[|>]-?\\s*$`)) if (match) { inBlock = true } @@ -121,7 +149,9 @@ function parseFrontmatterValue(frontmatter: string, key: string): string | null return normalizeValue(unquote(match[1])) } -function parseSkillFile(content: string, path: string): SkillMetadata | null { +type ParsedSkillFile = Pick + +function parseSkillFile(content: string, path: string): ParsedSkillFile | null { const frontmatter = extractFrontmatter(content) if (!frontmatter) { return null @@ -228,7 +258,13 @@ async function defaultSkillRoots(options: LoadSkillsOptions): Promise const projectRoot = await resolveProjectRoot(cwd) const roots: string[] = await projectDotSkillRoots(projectRoot) + // User-level global roots, ordered by priority (first wins on dedup): + // memo home first so skills_admin writes stay the winning paths, then the + // well-known Claude / Codex / Agents skill directories. roots.push(join(memoHome, 'skills')) + roots.push(join(homeDir, '.claude', 'skills')) + roots.push(join(homeDir, '.codex', 'skills')) + roots.push(join(homeDir, '.agents', 'skills')) return dedupePaths(roots) } @@ -247,9 +283,11 @@ async function resolveSkillRoots(options: LoadSkillsOptions): Promise export async function loadSkills(options: LoadSkillsOptions = {}): Promise { const roots = await resolveSkillRoots(options) + const projectRoot = await resolveProjectRoot(options.cwd ?? process.cwd()) const maxSkills = Math.max(1, options.maxSkills ?? DEFAULT_MAX_SKILLS) const skills: SkillMetadata[] = [] const seenPaths = new Set() + const byHash = new Map() for (const root of roots) { if (!(await existsAsDirectory(root))) { @@ -274,6 +312,7 @@ export async function loadSkills(options: LoadSkillsOptions = {}): Promise= maxSkills) { return skills } @@ -298,21 +353,135 @@ export async function loadSkills(options: LoadSkillsOptions = {}): Promise resolve(item))) + return skills.filter((skill) => skill.paths.some((alias) => active.has(resolve(alias)))) +} + +export function buildSkillIndex(skills: SkillMetadata[]): SkillIndex { + const byName = new Map() + const byPath = new Map() + const byHash = new Map() + for (const skill of skills) { + const nameMatches = byName.get(skill.name) ?? [] + nameMatches.push(skill) + byName.set(skill.name, nameMatches) + for (const alias of skill.paths) { + byPath.set(resolve(alias), skill) + } + byHash.set(skill.hash, skill) + } + return { list: skills, byName, byPath, byHash } +} + +export function findSkillByName(index: SkillIndex, name: string): SkillMetadata[] { + return index.byName.get(name) ?? [] +} + +export function findSkillByPath(index: SkillIndex, path: string): SkillMetadata | undefined { + return index.byPath.get(resolve(path)) +} + +/** Read the SKILL.md body with the frontmatter stripped. */ +export async function readSkillBody(record: SkillMetadata): Promise { + const content = await readFile(record.path, 'utf-8') + return stripFrontmatter(content) +} + +export function stripFrontmatter(content: string): string { + const lines = content.split(/\r?\n/) + if (lines[0]?.trim() !== '---') { + return content.trim() + } + let found = 0 + for (let i = 0; i < lines.length; i++) { + if (lines[i]?.trim() === '---') { + found += 1 + if (found === 2) { + return lines + .slice(i + 1) + .join('\n') + .trim() + } + } + } + return content.trim() +} + +export function renderSkillsSection(skills: SkillMetadata[], options: { budgetChars?: number } = {}): string | null { if (skills.length === 0) { return null } + const budget = options.budgetChars ?? DEFAULT_SKILLS_BUDGET_CHARS + + const intro = [ + '## Skills', + 'A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills available in this session. To use a skill, call the `read_skill` tool with its name (or its SKILL.md path when names are ambiguous) to load the full instructions.', + '### Available skills', + ] + const rules = ['### How to use skills', SKILLS_USAGE_RULES] + const entries = skills.map((skill) => `- ${skill.name}: ${skill.description}`) + + const render = (keptEntries: string[], omitted: number): string => { + const parts = [...intro, ...keptEntries] + if (omitted > 0) { + parts.push(`- (${omitted} more skills omitted due to context budget)`) + } + parts.push(...rules) + return parts.join('\n') + } - const lines: string[] = [] - lines.push('## Skills') - lines.push( - 'A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.', - ) - lines.push('### Available skills') - for (const skill of skills) { - lines.push(`- ${skill.name}: ${skill.description} (file: ${skill.path})`) + if (render(entries, 0).length <= budget) { + return render(entries, 0) + } + + // Over budget: truncate descriptions fairly, then drop low-priority entries from the tail. + const fixedCost = render([], 0).length + 1 + const entryBudget = Math.max(0, budget - fixedCost) + if (entryBudget <= 0) { + return render([], skills.length) + } + + const share = Math.floor(entryBudget / skills.length) + if (share >= MIN_ENTRY_CHARS) { + const truncated = skills.map((skill) => { + const maxDesc = Math.max(1, share - skill.name.length - 6) + if (skill.description.length <= maxDesc) { + return `- ${skill.name}: ${skill.description}` + } + return `- ${skill.name}: ${skill.description.slice(0, Math.max(1, maxDesc - 3)).trimEnd()}...` + }) + let kept = truncated.length + while (kept > 0 && render(truncated.slice(0, kept), skills.length - kept).length > budget) { + kept -= 1 + } + return render(truncated.slice(0, kept), skills.length - kept) + } + + let kept = entries.length + while (kept > 0 && render(entries.slice(0, kept), skills.length - kept).length > budget) { + kept -= 1 } - lines.push('### How to use skills') - lines.push(SKILLS_USAGE_RULES) - return lines.join('\n') + return render(entries.slice(0, kept), skills.length - kept) } diff --git a/packages/core/src/runtime/skills_admin.test.ts b/packages/core/src/skills/skills_admin.test.ts similarity index 100% rename from packages/core/src/runtime/skills_admin.test.ts rename to packages/core/src/skills/skills_admin.test.ts diff --git a/packages/core/src/runtime/skills_admin.ts b/packages/core/src/skills/skills_admin.ts similarity index 99% rename from packages/core/src/runtime/skills_admin.ts rename to packages/core/src/skills/skills_admin.ts index 697f6fe..e11af23 100644 --- a/packages/core/src/runtime/skills_admin.ts +++ b/packages/core/src/skills/skills_admin.ts @@ -2,7 +2,7 @@ import { access, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promise import { homedir } from 'node:os' import { basename, dirname, join, resolve } from 'node:path' import { loadMemoConfig, writeMemoConfig, type MemoConfig } from '../config/config.js' -import { normalizeWorkspacePath } from './workspace.js' +import { normalizeWorkspacePath } from '../utils/workspace.js' import type { SkillRecord } from '../api_types.js' type SkillScope = 'project' | 'global' diff --git a/packages/tools/src/approval/classifier.test.ts b/packages/core/src/tools/approval/classifier.test.ts similarity index 100% rename from packages/tools/src/approval/classifier.test.ts rename to packages/core/src/tools/approval/classifier.test.ts diff --git a/packages/tools/src/approval/classifier.ts b/packages/core/src/tools/approval/classifier.ts similarity index 100% rename from packages/tools/src/approval/classifier.ts rename to packages/core/src/tools/approval/classifier.ts diff --git a/packages/tools/src/approval/constants.ts b/packages/core/src/tools/approval/constants.ts similarity index 100% rename from packages/tools/src/approval/constants.ts rename to packages/core/src/tools/approval/constants.ts diff --git a/packages/tools/src/approval/fingerprint.ts b/packages/core/src/tools/approval/fingerprint.ts similarity index 63% rename from packages/tools/src/approval/fingerprint.ts rename to packages/core/src/tools/approval/fingerprint.ts index 475fd0f..5c88211 100644 --- a/packages/tools/src/approval/fingerprint.ts +++ b/packages/core/src/tools/approval/fingerprint.ts @@ -1,22 +1,9 @@ /** @file Tool request fingerprint generation */ import { createHash } from 'node:crypto' +import { stableStringify } from '@memo/core/utils/serialize' import type { ApprovalKey } from './types' -/** Stable serialize object (ensures same parameters generate same string) */ -export function stableStringify(value: unknown): string { - if (value === null || typeof value !== 'object') { - return JSON.stringify(value) - } - - if (Array.isArray(value)) { - return '[' + value.map((v) => stableStringify(v)).join(',') + ']' - } - - const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)) - return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}` -} - /** Generate tool request fingerprint */ export function generateFingerprint(toolName: string, params: unknown): ApprovalKey { const normalized = stableStringify(params) diff --git a/packages/tools/src/approval/index.ts b/packages/core/src/tools/approval/index.ts similarity index 76% rename from packages/tools/src/approval/index.ts rename to packages/core/src/tools/approval/index.ts index 89532f3..63b2162 100644 --- a/packages/tools/src/approval/index.ts +++ b/packages/core/src/tools/approval/index.ts @@ -9,9 +9,11 @@ export type { ApprovalKey, ApprovalMode, RiskLevel, + ToolActionErrorType, + ToolActionStatus, } from './types' export { createApprovalManager } from './manager' export { createToolClassifier } from './classifier' -export { generateFingerprint, stableStringify } from './fingerprint' +export { generateFingerprint, generatePartialFingerprint } from './fingerprint' export { DEFAULT_TOOL_RISK_LEVELS, RISK_LEVEL_ORDER } from './constants' diff --git a/packages/tools/src/approval/manager.test.ts b/packages/core/src/tools/approval/manager.test.ts similarity index 100% rename from packages/tools/src/approval/manager.test.ts rename to packages/core/src/tools/approval/manager.test.ts diff --git a/packages/tools/src/approval/manager.ts b/packages/core/src/tools/approval/manager.ts similarity index 100% rename from packages/tools/src/approval/manager.ts rename to packages/core/src/tools/approval/manager.ts diff --git a/packages/tools/src/approval/types.ts b/packages/core/src/tools/approval/types.ts similarity index 83% rename from packages/tools/src/approval/types.ts rename to packages/core/src/tools/approval/types.ts index 4c67db1..f004bcf 100644 --- a/packages/tools/src/approval/types.ts +++ b/packages/core/src/tools/approval/types.ts @@ -66,3 +66,15 @@ export interface ApprovalManager { /** Clear all authorizations (called when Session ends) */ dispose(): void } + +/** Tool action error category (moved here from tools/orchestrator). */ +export type ToolActionErrorType = + | 'approval_denied' + | 'policy_denied' + | 'sandbox_denied' + | 'tool_not_found' + | 'input_invalid' + | 'execution_failed' + +/** Tool action status ('success' or an error category). */ +export type ToolActionStatus = 'success' | ToolActionErrorType diff --git a/packages/tools/src/index.test.ts b/packages/core/src/tools/index.test.ts similarity index 100% rename from packages/tools/src/index.test.ts rename to packages/core/src/tools/index.test.ts diff --git a/packages/core/src/tools/index.ts b/packages/core/src/tools/index.ts new file mode 100644 index 0000000..d733926 --- /dev/null +++ b/packages/core/src/tools/index.ts @@ -0,0 +1,84 @@ +import type { Tool } from 'ai' +import { shellTool } from '@memo/core/tools/tools/shell' +import { shellCommandTool } from '@memo/core/tools/tools/shell_command' +import { execCommandTool } from '@memo/core/tools/tools/exec_command' +import { writeStdinTool } from '@memo/core/tools/tools/write_stdin' +import { applyPatchTool } from '@memo/core/tools/tools/apply_patch' +import { readTextFileTool } from '@memo/core/tools/tools/read_text_file' +import { readMediaFileTool } from '@memo/core/tools/tools/read_media_file' +import { readFilesTool } from '@memo/core/tools/tools/read_files' +import { writeFileTool } from '@memo/core/tools/tools/write_file' +import { editFileTool } from '@memo/core/tools/tools/edit_file' +import { listDirectoryTool } from '@memo/core/tools/tools/list_directory' +import { searchFilesTool } from '@memo/core/tools/tools/search_files' +import { + listMcpResourceTemplatesTool, + listMcpResourcesTool, + readMcpResourceTool, +} from '@memo/core/tools/tools/mcp_resources' +import { updatePlanTool } from '@memo/core/tools/tools/update_plan' +import { getMemoryTool } from '@memo/core/tools/tools/get_memory' +import { readSkillTool } from '@memo/core/tools/tools/read_skill' +import { webfetchTool } from '@memo/core/tools/tools/webfetch' +import { closeAgentTool, resumeAgentTool, sendInputTool, spawnAgentTool, waitTool } from '@memo/core/tools/tools/collab' + +function buildCodexTools(): Record { + const tools: Record = {} + const shellMode = process.env.MEMO_SHELL_TOOL_TYPE?.trim() || 'unified_exec' + const collabEnabled = process.env.MEMO_ENABLE_COLLAB_TOOLS !== '0' + const memoryToolEnabled = process.env.MEMO_ENABLE_MEMORY_TOOL !== '0' + + if (shellMode === 'shell') { + tools.shell = shellTool + } else if (shellMode === 'shell_command') { + tools.shell_command = shellCommandTool + } else if (shellMode === 'unified_exec') { + tools.exec_command = execCommandTool + tools.write_stdin = writeStdinTool + } else if (shellMode !== 'disabled') { + tools.exec_command = execCommandTool + tools.write_stdin = writeStdinTool + } + + tools.list_mcp_resources = listMcpResourcesTool + tools.list_mcp_resource_templates = listMcpResourceTemplatesTool + tools.read_mcp_resource = readMcpResourceTool + tools.update_plan = updatePlanTool + tools.read_skill = readSkillTool + tools.apply_patch = applyPatchTool + tools.read_text_file = readTextFileTool + tools.read_media_file = readMediaFileTool + tools.read_files = readFilesTool + tools.write_file = writeFileTool + tools.edit_file = editFileTool + tools.list_directory = listDirectoryTool + tools.search_files = searchFilesTool + + if (memoryToolEnabled) { + tools.get_memory = getMemoryTool + } + + tools.webfetch = webfetchTool + + if (collabEnabled) { + tools.spawn_agent = spawnAgentTool + tools.send_input = sendInputTool + tools.resume_agent = resumeAgentTool + tools.wait = waitTool + tools.close_agent = closeAgentTool + } + + return tools +} + +/** Exposed built-in tool collection (AI SDK ToolSet, keys are tool names). */ +export const TOOLKIT: Record = buildCodexTools() + +/** Tool array form, convenient for direct registration. */ +export const TOOL_LIST: Tool[] = Object.values(TOOLKIT) + +/** Built-in tools (already AI SDK Tool format, no adaptation needed). */ +export const NATIVE_TOOLS = TOOLKIT + +export * from '@memo/core/tools/approval' +export * from '@memo/core/tools/router' diff --git a/packages/core/src/tools/router/index.ts b/packages/core/src/tools/router/index.ts new file mode 100644 index 0000000..386e6ad --- /dev/null +++ b/packages/core/src/tools/router/index.ts @@ -0,0 +1,4 @@ +/** @file Tool router module exports (tools are standard AI SDK Tool objects). */ +export type { MCPServerConfig, McpClientConnection } from './types' +export { McpToolRegistry } from './mcp' +export type { McpOAuthSettings } from './mcp/oauth' diff --git a/packages/tools/src/router/mcp/cache_store.test.ts b/packages/core/src/tools/router/mcp/cache_store.test.ts similarity index 100% rename from packages/tools/src/router/mcp/cache_store.test.ts rename to packages/core/src/tools/router/mcp/cache_store.test.ts diff --git a/packages/tools/src/router/mcp/cache_store.ts b/packages/core/src/tools/router/mcp/cache_store.ts similarity index 95% rename from packages/tools/src/router/mcp/cache_store.ts rename to packages/core/src/tools/router/mcp/cache_store.ts index 0946852..458cc1f 100644 --- a/packages/tools/src/router/mcp/cache_store.ts +++ b/packages/core/src/tools/router/mcp/cache_store.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto' import { mkdir, readFile, rename, writeFile } from 'node:fs/promises' import { homedir } from 'node:os' import { dirname, join } from 'node:path' +import { stableStringify } from '@memo/core/utils/serialize' import type { MCPServerConfig } from '../types' const CACHE_FILE_NAME = 'mcp.json' @@ -69,20 +70,6 @@ function getCacheFilePath(): string { return join(resolveMemoHomeDir(), 'cache', CACHE_FILE_NAME) } -function stableStringify(value: unknown): string { - if (value === null || typeof value !== 'object') { - return JSON.stringify(value) - } - - if (Array.isArray(value)) { - return `[${value.map((item) => stableStringify(item)).join(',')}]` - } - - const object = value as Record - const keys = Object.keys(object).sort() - return `{${keys.map((key) => `${JSON.stringify(key)}:${stableStringify(object[key])}`).join(',')}}` -} - function configHash(config: MCPServerConfig): string { return createHash('sha256').update(stableStringify(config)).digest('hex') } diff --git a/packages/tools/src/router/mcp/context.ts b/packages/core/src/tools/router/mcp/context.ts similarity index 100% rename from packages/tools/src/router/mcp/context.ts rename to packages/core/src/tools/router/mcp/context.ts diff --git a/packages/tools/src/router/mcp/index.test.ts b/packages/core/src/tools/router/mcp/index.test.ts similarity index 88% rename from packages/tools/src/router/mcp/index.test.ts rename to packages/core/src/tools/router/mcp/index.test.ts index 828d449..aabe77d 100644 --- a/packages/tools/src/router/mcp/index.test.ts +++ b/packages/core/src/tools/router/mcp/index.test.ts @@ -16,21 +16,14 @@ function createConfig(): MCPServerConfig { function createConnection(serverName: string, toolName: string) { return { name: serverName, - client: { - callTool: async () => ({ content: [] }), - }, - transport: {} as any, - tools: [ - { - name: `${serverName}_${toolName}`, + client: {}, + tools: { + [toolName]: { description: `Tool from ${serverName}: ${toolName}`, - source: 'mcp' as const, - serverName, - originalName: toolName, - inputSchema: {}, - execute: async () => ({ content: [] }), + inputSchema: { jsonSchema: () => ({ type: 'object' }) }, + execute: async () => ({ type: 'text', value: '' }), }, - ], + }, } } diff --git a/packages/tools/src/router/mcp/index.ts b/packages/core/src/tools/router/mcp/index.ts similarity index 70% rename from packages/tools/src/router/mcp/index.ts rename to packages/core/src/tools/router/mcp/index.ts index d7d42f0..327b783 100644 --- a/packages/tools/src/router/mcp/index.ts +++ b/packages/core/src/tools/router/mcp/index.ts @@ -1,6 +1,6 @@ -/** @file MCP tool registry */ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' -import type { McpTool, ToolRegistry, MCPServerConfig } from '../types' +/** @file MCP tool registry (stores standard AI SDK Tool objects). */ +import { jsonSchema, type Tool } from 'ai' +import type { MCPServerConfig } from '../types' import { McpClientPool } from './pool' import { getGlobalMcpCacheStore, type CachedMcpToolDescriptor } from './cache_store' import { setActiveMcpCacheStore, setActiveMcpPool } from './context' @@ -11,7 +11,7 @@ export class McpToolRegistry { private pool: McpClientPool private serverToolNames: Map> = new Map() private refreshPromises: Map> = new Map() - private tools: Map = new Map() + private tools: Map = new Map() private cacheStore = getGlobalMcpCacheStore() private readonly shouldLog: boolean @@ -22,25 +22,25 @@ export class McpToolRegistry { this.shouldLog = !(process.stdout.isTTY && process.stdin.isTTY) } - private buildTool(serverName: string, config: MCPServerConfig, descriptor: CachedMcpToolDescriptor): McpTool { + /** Cached-descriptor placeholder tool: lazy-connects and delegates to the real MCP tool on execute. */ + private buildCachedTool(serverName: string, config: MCPServerConfig, descriptor: CachedMcpToolDescriptor): Tool { return { - name: `${serverName}_${descriptor.originalName}`, description: descriptor.description || `Tool from ${serverName}: ${descriptor.originalName}`, - source: 'mcp', - serverName, - originalName: descriptor.originalName, - inputSchema: (descriptor.inputSchema as any) ?? {}, - execute: async (input: unknown): Promise => { + inputSchema: descriptor.inputSchema + ? jsonSchema(descriptor.inputSchema as object) + : jsonSchema({ type: 'object' }), + execute: async (input, options) => { const connection = await this.pool.connect(serverName, config) - return connection.client.callTool({ - name: descriptor.originalName, - arguments: input as Record, - }) as Promise + const sdkTool = connection.tools[descriptor.originalName] + if (!sdkTool?.execute) { + return { type: 'error-text', value: `MCP tool not found: ${descriptor.originalName}` } + } + return sdkTool.execute(input, options) }, } } - private replaceServerTools(serverName: string, nextTools: McpTool[]) { + private replaceServerTools(serverName: string, nextTools: Record) { const prev = this.serverToolNames.get(serverName) if (prev) { for (const toolName of prev) { @@ -49,9 +49,9 @@ export class McpToolRegistry { } const next = new Set() - for (const tool of nextTools) { - this.tools.set(tool.name, tool) - next.add(tool.name) + for (const [toolName, tool] of Object.entries(nextTools)) { + this.tools.set(toolName, tool) + next.add(toolName) } this.serverToolNames.set(serverName, next) } @@ -60,10 +60,10 @@ export class McpToolRegistry { serverName: string, connection: Awaited>, ): CachedMcpToolDescriptor[] { - return connection.tools.map((tool) => ({ - originalName: tool.originalName, - description: tool.description || `Tool from ${serverName}: ${tool.originalName}`, - inputSchema: tool.inputSchema, + return Object.entries(connection.tools).map(([originalName, tool]) => ({ + originalName, + description: tool.description || `Tool from ${serverName}: ${originalName}`, + inputSchema: (tool.inputSchema as { jsonSchema?: () => unknown }).jsonSchema?.(), })) } @@ -85,10 +85,12 @@ export class McpToolRegistry { const connection = await this.pool.connect(serverName, config) const descriptors = this.connectionToDescriptors(serverName, connection) await this.cacheStore.setServerTools(serverName, config, descriptors) - const tools = descriptors.map((descriptor) => this.buildTool(serverName, config, descriptor)) + const tools = Object.fromEntries( + Object.entries(connection.tools).map(([name, tool]) => [`${serverName}_${name}`, tool]), + ) this.replaceServerTools(serverName, tools) if (this.shouldLog && mode === 'background') { - console.log(`[MCP] Refreshed '${serverName}' tools in background (${tools.length})`) + console.log(`[MCP] Refreshed '${serverName}' tools in background (${Object.keys(tools).length})`) } } catch (err) { if (this.shouldLog) { @@ -143,11 +145,16 @@ export class McpToolRegistry { for (const [serverName, config] of entries) { const cached = await this.cacheStore.getServerTools(serverName, config) if (cached) { - const tools = cached.tools.map((descriptor) => this.buildTool(serverName, config, descriptor)) + const tools = Object.fromEntries( + cached.tools.map((descriptor) => [ + `${serverName}_${descriptor.originalName}`, + this.buildCachedTool(serverName, config, descriptor), + ]), + ) this.replaceServerTools(serverName, tools) if (this.shouldLog) { console.log( - `[MCP] Loaded ${tools.length} cached tools for '${serverName}' (${cached.stale ? 'stale' : 'fresh'})`, + `[MCP] Loaded ${Object.keys(tools).length} cached tools for '${serverName}' (${cached.stale ? 'stale' : 'fresh'})`, ) } @@ -165,22 +172,18 @@ export class McpToolRegistry { } /** Get tool */ - get(name: string): McpTool | undefined { + get(name: string): Tool | undefined { return this.tools.get(name) } /** Get all tools */ - getAll(): McpTool[] { + getAll(): Tool[] { return Array.from(this.tools.values()) } - /** Convert to ToolRegistry format */ - toRegistry(): ToolRegistry { - const registry: ToolRegistry = {} - for (const [name, tool] of this.tools) { - registry[name] = tool - } - return registry + /** Convert to AI SDK ToolSet format */ + toToolSet(): Record { + return Object.fromEntries(this.tools) } /** Check if tool exists */ diff --git a/packages/tools/src/router/mcp/oauth.runtime.test.ts b/packages/core/src/tools/router/mcp/oauth.runtime.test.ts similarity index 98% rename from packages/tools/src/router/mcp/oauth.runtime.test.ts rename to packages/core/src/tools/router/mcp/oauth.runtime.test.ts index ca9d291..33d9454 100644 --- a/packages/tools/src/router/mcp/oauth.runtime.test.ts +++ b/packages/core/src/tools/router/mcp/oauth.runtime.test.ts @@ -11,8 +11,8 @@ const { authMock, spawnMock } = vi.hoisted(() => { } }) -vi.mock('@modelcontextprotocol/sdk/client/auth.js', async () => { - const actual = await vi.importActual('@modelcontextprotocol/sdk/client/auth.js') +vi.mock('@ai-sdk/mcp', async () => { + const actual = await vi.importActual('@ai-sdk/mcp') return { ...(actual as object), auth: authMock, diff --git a/packages/tools/src/router/mcp/oauth.test.ts b/packages/core/src/tools/router/mcp/oauth.test.ts similarity index 100% rename from packages/tools/src/router/mcp/oauth.test.ts rename to packages/core/src/tools/router/mcp/oauth.test.ts diff --git a/packages/tools/src/router/mcp/oauth.ts b/packages/core/src/tools/router/mcp/oauth.ts similarity index 98% rename from packages/tools/src/router/mcp/oauth.ts rename to packages/core/src/tools/router/mcp/oauth.ts index dccefdd..0458cf4 100644 --- a/packages/tools/src/router/mcp/oauth.ts +++ b/packages/core/src/tools/router/mcp/oauth.ts @@ -4,13 +4,14 @@ import { createServer } from 'node:http' import { mkdir, readFile, rename, writeFile } from 'node:fs/promises' import { homedir } from 'node:os' import { dirname, join } from 'node:path' -import { auth, type OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js' -import type { - OAuthClientInformationMixed, - OAuthClientMetadata, - OAuthTokens, -} from '@modelcontextprotocol/sdk/shared/auth.js' -import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' +import { + auth, + type OAuthClientInformation, + type OAuthClientMetadata, + type OAuthClientProvider, + type OAuthTokens, +} from '@ai-sdk/mcp' +import type { FetchFunction } from '@ai-sdk/provider-utils' import type { MCPServerConfig } from '../types' const OAUTH_FILE_VERSION = 1 @@ -33,7 +34,7 @@ export type McpOAuthSettings = { } export type McpOAuthCredential = { - clientInformation?: OAuthClientInformationMixed + clientInformation?: OAuthClientInformation tokens?: OAuthTokens } @@ -306,7 +307,7 @@ export async function openExternalUrl(url: string): Promise { }) } -function createServerBoundFetch(serverUrl: string, defaultHeaders: Record): FetchLike { +function createServerBoundFetch(serverUrl: string, defaultHeaders: Record): FetchFunction { return async (input, init) => { const requestUrl = typeof input === 'string' || input instanceof URL ? new URL(String(input), serverUrl) : new URL(input.url) @@ -408,7 +409,7 @@ class MemoOAuthClientProvider implements OAuthClientProvider { return this.credential.clientInformation } - async saveClientInformation(clientInformation: OAuthClientInformationMixed) { + async saveClientInformation(clientInformation: OAuthClientInformation) { await this.ensureLoaded() this.credential = { ...this.credential, diff --git a/packages/tools/src/router/mcp/pool.test.ts b/packages/core/src/tools/router/mcp/pool.test.ts similarity index 57% rename from packages/tools/src/router/mcp/pool.test.ts rename to packages/core/src/tools/router/mcp/pool.test.ts index 5e3ef06..d5abd11 100644 --- a/packages/tools/src/router/mcp/pool.test.ts +++ b/packages/core/src/tools/router/mcp/pool.test.ts @@ -3,70 +3,41 @@ import { afterEach, describe, expect, test, vi } from 'vitest' import type { MCPServerConfig } from '../types' const { - connectMock, - listToolsMock, + toolsMock, + createClientConfigMock, closeMock, createRuntimeMcpOAuthProviderMock, - streamableInstances, stdioInstances, UnauthorizedErrorMock, - StreamableHTTPErrorMock, } = vi.hoisted(() => { class UnauthorizedErrorMock extends Error {} - class StreamableHTTPErrorMock extends Error { - code: number - constructor(message: string, code: number) { - super(message) - this.code = code - } - } return { - connectMock: vi.fn(), - listToolsMock: vi.fn(), + toolsMock: vi.fn(), + createClientConfigMock: vi.fn(), closeMock: vi.fn(), createRuntimeMcpOAuthProviderMock: vi.fn(), - streamableInstances: [] as Array<{ url: URL; options: Record }>, stdioInstances: [] as Array<{ options: Record }>, UnauthorizedErrorMock, - StreamableHTTPErrorMock, } }) -vi.mock('@modelcontextprotocol/sdk/client/index.js', () => { - class MockClient { - async connect(transport: unknown) { - return connectMock(transport) - } - async listTools() { - return listToolsMock() - } - async close() { - return closeMock() - } - } +vi.mock('@ai-sdk/mcp', async () => { + const actual = await vi.importActual('@ai-sdk/mcp') return { - Client: MockClient, - } -}) - -vi.mock('@modelcontextprotocol/sdk/client/streamableHttp.js', () => { - class MockStreamableHTTPClientTransport { - url: URL - options: Record - constructor(url: URL, options: Record) { - this.url = url - this.options = options - streamableInstances.push(this) - } - } - return { - StreamableHTTPClientTransport: MockStreamableHTTPClientTransport, - StreamableHTTPError: StreamableHTTPErrorMock, + ...(actual as Record), + createMCPClient: async (config: unknown) => { + await createClientConfigMock(config) + return { + tools: async () => toolsMock(), + close: async () => closeMock(), + } + }, + UnauthorizedError: UnauthorizedErrorMock, } }) -vi.mock('@modelcontextprotocol/sdk/client/stdio.js', () => { - class MockStdioClientTransport { +vi.mock('@ai-sdk/mcp/mcp-stdio', () => { + class MockStdioMCPTransport { options: Record constructor(options: Record) { this.options = options @@ -74,13 +45,7 @@ vi.mock('@modelcontextprotocol/sdk/client/stdio.js', () => { } } return { - StdioClientTransport: MockStdioClientTransport, - } -}) - -vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => { - return { - UnauthorizedError: UnauthorizedErrorMock, + Experimental_StdioMCPTransport: MockStdioMCPTransport, } }) @@ -100,25 +65,21 @@ function httpConfig(extra?: Partial>): } } -afterEach(() => { - vi.restoreAllMocks() - connectMock.mockReset() - listToolsMock.mockReset() - closeMock.mockReset() - createRuntimeMcpOAuthProviderMock.mockReset() - streamableInstances.splice(0) - stdioInstances.splice(0) - delete process.env.MCP_TOKEN - delete process.env.BASE_ENV -}) - describe('mcp client pool', () => { + afterEach(() => { + toolsMock.mockReset() + createClientConfigMock.mockReset() + closeMock.mockReset() + createRuntimeMcpOAuthProviderMock.mockReset() + stdioInstances.splice(0) + delete process.env.MCP_TOKEN + }) + test('connects HTTP server with oauth settings and request headers', async () => { const authProvider = { kind: 'oauth-provider' } createRuntimeMcpOAuthProviderMock.mockResolvedValue(authProvider) - connectMock.mockResolvedValue(undefined) - listToolsMock.mockResolvedValue({ - tools: [{ name: 'search', description: 'Search docs', inputSchema: { type: 'object' } }], + toolsMock.mockResolvedValue({ + search: { description: 'Search docs', inputSchema: { type: 'object' } }, }) process.env.MCP_TOKEN = 'token-123' @@ -139,19 +100,19 @@ describe('mcp client pool', () => { config, settings: { memoHome: '/tmp/memo-home', storeMode: 'file', callbackPort: 33333 }, }) - expect(connectMock).toHaveBeenCalledTimes(1) - assert.strictEqual(streamableInstances.length, 1) - const transport = streamableInstances[0] - assert.strictEqual(transport?.url.toString(), 'https://example.com/mcp') - expect(transport?.options.authProvider).toEqual(authProvider) - expect(transport?.options.requestInit).toEqual({ - headers: { - 'X-Custom': 'value', - Authorization: 'Bearer token-123', - }, + expect(createClientConfigMock).toHaveBeenCalledTimes(1) + const transportConfig = createClientConfigMock.mock.calls[0]?.[0] as { + transport: { type: string; url: string; headers: Record; authProvider: unknown } + } + assert.strictEqual(transportConfig.transport.type, 'http') + assert.strictEqual(transportConfig.transport.url, 'https://example.com/mcp') + expect(transportConfig.transport.authProvider).toEqual(authProvider) + expect(transportConfig.transport.headers).toEqual({ + 'X-Custom': 'value', + Authorization: 'Bearer token-123', }) - assert.strictEqual(connection.tools.length, 1) - assert.strictEqual(connection.tools[0]?.name, 'remote_search') + assert.strictEqual(Object.keys(connection.tools).length, 1) + assert.ok(connection.tools['search']) }) test('reuses inflight connect promise for same server', async () => { @@ -159,8 +120,8 @@ describe('mcp client pool', () => { const connectPromise = new Promise((resolve) => { resolveConnect = resolve }) - connectMock.mockImplementation(() => connectPromise) - listToolsMock.mockResolvedValue({ tools: [] }) + createClientConfigMock.mockImplementation(() => connectPromise) + toolsMock.mockResolvedValue({}) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) const pool = new McpClientPool() @@ -172,58 +133,44 @@ describe('mcp client pool', () => { resolveConnect() const [left, right] = await Promise.all([first, second]) - expect(connectMock).toHaveBeenCalledTimes(1) + expect(createClientConfigMock).toHaveBeenCalledTimes(1) assert.strictEqual(left, right) }) test('includes login hint for unauthorized HTTP failures', async () => { - connectMock.mockRejectedValue(new UnauthorizedErrorMock('unauthorized')) + createClientConfigMock.mockRejectedValue(new UnauthorizedErrorMock('unauthorized')) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) const pool = new McpClientPool() - - await expect(pool.connect('remote', httpConfig())).rejects.toThrow('Run "memo mcp login remote".') - }) - - test('includes login hint for 403 streamable HTTP failures', async () => { - connectMock.mockRejectedValue(new StreamableHTTPErrorMock('forbidden', 403)) - createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) - - const pool = new McpClientPool() - await expect(pool.connect('remote', httpConfig())).rejects.toThrow('Run "memo mcp login remote".') }) test('does not include login hint for non-auth failures', async () => { - connectMock.mockRejectedValue(new Error('network timeout')) + createClientConfigMock.mockRejectedValue(new Error('connection refused')) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) const pool = new McpClientPool() - await expect(pool.connect('remote', httpConfig())).rejects.toThrow( - 'Failed to connect via streamable_http (network timeout).', + 'Failed to connect via streamable_http (connection refused).', ) }) test('closes client when listing tools fails', async () => { - connectMock.mockResolvedValue(undefined) - listToolsMock.mockRejectedValue(new Error('list failed')) - closeMock.mockResolvedValue(undefined) + toolsMock.mockRejectedValue(new Error('list failed')) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) const pool = new McpClientPool() - await expect(pool.connect('remote', httpConfig())).rejects.toThrow('list failed') expect(closeMock).toHaveBeenCalledTimes(1) }) test('connects stdio server with merged env and explicit stderr mode', async () => { - connectMock.mockResolvedValue(undefined) - listToolsMock.mockResolvedValue({ tools: [] }) + toolsMock.mockResolvedValue({}) + createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) process.env.BASE_ENV = 'base' const pool = new McpClientPool() - await pool.connect('local', { + await pool.connect('remote', { command: 'node', args: ['server.js'], env: { LOCAL_ENV: 'local' }, @@ -238,14 +185,12 @@ describe('mcp client pool', () => { const env = transport?.options.env as Record assert.strictEqual(env.LOCAL_ENV, 'local') assert.strictEqual(env.BASE_ENV, 'base') - delete process.env.BASE_ENV }) test('closeAll logs close failures and clears connected clients', async () => { - connectMock.mockResolvedValue(undefined) - listToolsMock.mockResolvedValue({ tools: [] }) - closeMock.mockRejectedValue(new Error('close failed')) + toolsMock.mockResolvedValue({}) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) + closeMock.mockRejectedValue(new Error('close failed')) const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) const pool = new McpClientPool() @@ -253,23 +198,22 @@ describe('mcp client pool', () => { assert.strictEqual(pool.size, 1) await pool.closeAll() - assert.strictEqual(pool.size, 0) expect(consoleSpy).toHaveBeenCalled() + consoleSpy.mockRestore() }) test('tracks known servers from configs and active connections', async () => { - connectMock.mockResolvedValue(undefined) - listToolsMock.mockResolvedValue({ tools: [] }) + toolsMock.mockResolvedValue({}) createRuntimeMcpOAuthProviderMock.mockResolvedValue(null) const pool = new McpClientPool() pool.setServerConfigs({ configured: httpConfig() }) - assert.strictEqual(pool.hasServer('configured'), true) + await pool.connect('connected', httpConfig()) - await pool.connect('connected', httpConfig({ url: 'https://example.com/other' })) + assert.strictEqual(pool.hasServer('configured'), true) + assert.strictEqual(pool.hasServer('connected'), true) const names = pool.getKnownServerNames() - expect(names).toContain('configured') expect(names).toContain('connected') }) diff --git a/packages/tools/src/router/mcp/pool.ts b/packages/core/src/tools/router/mcp/pool.ts similarity index 57% rename from packages/tools/src/router/mcp/pool.ts rename to packages/core/src/tools/router/mcp/pool.ts index f9490c8..06bd92c 100644 --- a/packages/tools/src/router/mcp/pool.ts +++ b/packages/core/src/tools/router/mcp/pool.ts @@ -1,14 +1,9 @@ -/** @file MCP Client 连接池管理 */ -import { Client } from '@modelcontextprotocol/sdk/client/index.js' -import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' -import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' -import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' -import { StreamableHTTPError } from '@modelcontextprotocol/sdk/client/streamableHttp.js' +/** @file MCP Client 连接池管理（基于 @ai-sdk/mcp） */ +import { createMCPClient, UnauthorizedError, type MCPClient, type MCPClientConfig } from '@ai-sdk/mcp' +import { Experimental_StdioMCPTransport } from '@ai-sdk/mcp/mcp-stdio' import type { MCPServerConfig, McpClientConnection } from '../types' import { createRuntimeMcpOAuthProvider, type McpOAuthSettings } from './oauth' -type ClientTransport = StdioClientTransport | StreamableHTTPClientTransport - function mergeProcessEnv(env?: Record): Record | undefined { if (!env) return undefined const merged: Record = { @@ -19,25 +14,6 @@ function mergeProcessEnv(env?: Record): Record | return Object.fromEntries(entries) } -/** 创建标准化的 MCP Client */ -function createMcpClient(): Client { - return new Client( - { - name: 'memo-code-cli-client', - version: '1.0.0', - }, - { - capabilities: {}, - }, - ) -} - -/** 构建 HTTP 请求的 headers */ -function buildRequestInit(headers?: Record): RequestInit | undefined { - if (!headers || Object.keys(headers).length === 0) return undefined - return { headers } -} - function resolveHttpHeaders(config: Extract) { const headers = { ...(config.http_headers ?? config.headers), @@ -51,63 +27,43 @@ function resolveHttpHeaders(config: Extract) { return headers } -/** 通过 HTTP 连接 MCP Server */ -async function connectOverHttp( - name: string, - config: Extract, - oauthSettings: McpOAuthSettings | undefined, -): Promise<{ client: Client; transport: ClientTransport }> { - const baseUrl = new URL(config.url) - const requestInit = buildRequestInit(resolveHttpHeaders(config)) - const authProvider = await createRuntimeMcpOAuthProvider({ - serverName: name, - config, - settings: oauthSettings, - }) - - try { - const client = createMcpClient() - const transport = new StreamableHTTPClientTransport(baseUrl, { - requestInit, - ...(authProvider ? { authProvider } : {}), - }) - await client.connect(transport) - return { client, transport } - } catch (streamErr) { - const authHint = isAuthFailure(streamErr) ? ` Run "memo mcp login ${name}".` : '' - const message = `Failed to connect via streamable_http (${(streamErr as Error).message}).${authHint}` - const error = new Error(message) - ;(error as any).cause = streamErr - throw error - } -} - -/** 根据配置建立连接 */ +/** 根据配置建立 AI SDK MCP 客户端 */ async function connectWithConfig( name: string, config: MCPServerConfig, oauthSettings: McpOAuthSettings | undefined, -): Promise<{ client: Client; transport: ClientTransport }> { +): Promise { if ('url' in config) { - return connectOverHttp(name, config, oauthSettings) + const authProvider = await createRuntimeMcpOAuthProvider({ + serverName: name, + config, + settings: oauthSettings, + }) + const transport: MCPClientConfig['transport'] = { + type: 'http', + url: config.url, + headers: resolveHttpHeaders(config), + ...(authProvider ? { authProvider } : {}), + } + try { + return await createMCPClient({ transport }) + } catch (streamErr) { + const authHint = isAuthFailure(streamErr) ? ` Run "memo mcp login ${name}".` : '' + const message = `Failed to connect via streamable_http (${(streamErr as Error).message}).${authHint}` + const error = new Error(message) + ;(error as any).cause = streamErr + throw error + } } // stdio 类型 - const stdioOptions: { - command: string - args?: string[] - env?: Record - stderr?: 'inherit' | 'pipe' | 'ignore' - } = { + const transport = new Experimental_StdioMCPTransport({ command: config.command, args: config.args, env: mergeProcessEnv(config.env), stderr: config.stderr ?? (process.stdout.isTTY && process.stdin.isTTY ? 'ignore' : undefined), - } - const transport = new StdioClientTransport(stdioOptions as any) - const client = createMcpClient() - await client.connect(transport) - return { client, transport } + }) + return createMCPClient({ transport }) } /** MCP Client 连接池 */ @@ -130,7 +86,7 @@ export class McpClientPool { * Connect to specified MCP Server * @param name - server name (key in configuration) * @param config - server configuration - * @returns connection info (contains client, transport, and tool list) + * @returns connection info (AI SDK MCP client + tool set) */ async connect(name: string, config?: MCPServerConfig): Promise { if (config) { @@ -154,28 +110,15 @@ export class McpClientPool { } const pending = (async () => { - // Establish new connection - const { client, transport } = await connectWithConfig(name, effectiveConfig, this.oauthSettings) + const client = await connectWithConfig(name, effectiveConfig, this.oauthSettings) try { - // Get tool list - const toolsResult = await client.listTools() - - // Build McpTool array (execute not filled yet, handled by Registry) + // Get tool set (AI SDK Tools with own execute). + const tools = await client.tools() const connection: McpClientConnection = { name, client, - transport, - tools: (toolsResult.tools || []).map((t) => ({ - name: `${name}_${t.name}`, - description: t.description || `Tool from ${name}: ${t.name}`, - source: 'mcp' as const, - serverName: name, - originalName: t.name, - inputSchema: t.inputSchema as any, - // execute 会在 registry 中绑定 - execute: async () => ({ content: [] }), - })), + tools, } this.connections.set(name, connection) @@ -223,18 +166,18 @@ export class McpClientPool { description: string serverName: string originalName: string - inputSchema: any - client: Client + inputSchema: unknown + client: MCPClient }[] = [] for (const conn of this.connections.values()) { - for (const tool of conn.tools) { + for (const [originalName, tool] of Object.entries(conn.tools)) { allTools.push({ - name: tool.name, - description: tool.description, - serverName: tool.serverName, - originalName: tool.originalName, - inputSchema: tool.inputSchema, + name: `${conn.name}_${originalName}`, + description: tool.description ?? `Tool from ${conn.name}: ${originalName}`, + serverName: conn.name, + originalName, + inputSchema: (tool.inputSchema as { jsonSchema?: () => unknown }).jsonSchema?.(), client: conn.client, }) } @@ -266,9 +209,6 @@ export class McpClientPool { function isAuthFailure(error: unknown): boolean { if (error instanceof UnauthorizedError) return true - if (error instanceof StreamableHTTPError) { - return error.code === 401 || error.code === 403 - } const message = (error as Error)?.message?.toLowerCase() ?? '' return ( message.includes('unauthorized') || diff --git a/packages/core/src/tools/router/types.ts b/packages/core/src/tools/router/types.ts new file mode 100644 index 0000000..73893c9 --- /dev/null +++ b/packages/core/src/tools/router/types.ts @@ -0,0 +1,27 @@ +/** @file MCP configuration and connection types (tools are standard AI SDK Tool objects). */ + +/** MCP Server configuration (reuses definition from config.ts) */ +export type MCPServerConfig = + | { + type?: 'stdio' + command: string + args?: string[] + env?: Record + /** Subprocess stderr behavior (silent in TTY by default). */ + stderr?: 'inherit' | 'pipe' | 'ignore' + } + | { + type?: 'streamable_http' + url: string + headers?: Record + http_headers?: Record + bearer_token_env_var?: string + } + +/** MCP Client connection info (AI SDK MCP client + tool set). */ +export interface McpClientConnection { + name: string + client: import('@ai-sdk/mcp').MCPClient + /** originalName → AI SDK Tool (own execute, JSON-RPC under the hood). */ + tools: Record +} diff --git a/packages/tools/src/runtime/context.ts b/packages/core/src/tools/runtime/context.ts similarity index 100% rename from packages/tools/src/runtime/context.ts rename to packages/core/src/tools/runtime/context.ts diff --git a/packages/core/src/tools/runtime/step_gate.test.ts b/packages/core/src/tools/runtime/step_gate.test.ts new file mode 100644 index 0000000..1c7cc74 --- /dev/null +++ b/packages/core/src/tools/runtime/step_gate.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from 'vitest' +import { createStepGate, type StepPermit } from '@memo/core/tools/runtime/step_gate' + +function release(permit: StepPermit) { + if (!permit.skipped) permit.release() +} + +describe('createStepGate', () => { + test('grants the first exclusive acquire immediately', async () => { + const gate = createStepGate() + const permit = await gate.acquire(true) + + expect(permit.skipped).toBe(false) + release(permit) + }) + + test('runs shared acquires together and gives queued exclusive work priority', async () => { + const gate = createStepGate() + const firstShared = await gate.acquire(false) + const secondShared = await gate.acquire(false) + const order: string[] = [] + let exclusiveStarted = false + let lateSharedStarted = false + + const exclusivePromise = gate.acquire(true).then((permit) => { + exclusiveStarted = true + order.push('exclusive') + return permit + }) + const lateSharedPromise = gate.acquire(false).then((permit) => { + lateSharedStarted = true + order.push('late-shared') + return permit + }) + + await Promise.resolve() + expect(exclusiveStarted).toBe(false) + expect(lateSharedStarted).toBe(false) + + release(firstShared) + await Promise.resolve() + expect(exclusiveStarted).toBe(false) + + release(secondShared) + const exclusive = await exclusivePromise + expect(order).toEqual(['exclusive']) + expect(lateSharedStarted).toBe(false) + + release(exclusive) + const lateShared = await lateSharedPromise + expect(order).toEqual(['exclusive', 'late-shared']) + release(lateShared) + }) + + test('serializes exclusive acquires in FIFO order', async () => { + const gate = createStepGate() + const first = await gate.acquire(true) + const order: number[] = [] + + const secondPromise = gate.acquire(true).then((permit) => { + order.push(2) + return permit + }) + const thirdPromise = gate.acquire(true).then((permit) => { + order.push(3) + return permit + }) + + release(first) + const second = await secondPromise + expect(order).toEqual([2]) + + release(second) + const third = await thirdPromise + expect(order).toEqual([2, 3]) + release(third) + }) + + test('skips queued and future acquires after denial', async () => { + const gate = createStepGate() + const running = await gate.acquire(true) + const queuedPromise = gate.acquire(false) + + gate.markDenied() + + await expect(queuedPromise).resolves.toEqual({ skipped: true }) + await expect(gate.acquire(false)).resolves.toEqual({ skipped: true }) + release(running) + }) +}) diff --git a/packages/core/src/tools/runtime/step_gate.ts b/packages/core/src/tools/runtime/step_gate.ts new file mode 100644 index 0000000..ec414ef --- /dev/null +++ b/packages/core/src/tools/runtime/step_gate.ts @@ -0,0 +1,86 @@ +/** @file Per-streamText-call tool execution gate: serializes mutating tools, skips after denial. */ + +export type StepPermit = + | { skipped: true } + | { + skipped: false + /** Must be called after the tool finishes (finally). */ + release: () => void + } + +export interface StepGate { + /** + * Acquire execution permission. + * - exclusive (mutating or non-parallel tools): runs alone, FIFO order. + * - shared (read-only parallel tools): runs concurrently unless an exclusive tool is queued/running. + * - after markDenied, every acquire returns { skipped: true }. + */ + acquire(exclusive: boolean): Promise + /** Deny the batch: subsequent tools in this step are skipped. */ + markDenied(): void +} + +export function createStepGate(): StepGate { + let denied = false + let runningShared = 0 + let runningExclusive = false + let queue: Array<{ exclusive: boolean; resolve: (permit: StepPermit) => void }> = [] + + function drain() { + if (denied) { + const pending = queue + queue = [] + for (const waiter of pending) waiter.resolve({ skipped: true }) + return + } + if (runningExclusive || queue.length === 0) return + + const first = queue[0] + if (first?.exclusive) { + if (runningShared > 0) return + queue.shift() + runningExclusive = true + let released = false + first.resolve({ + skipped: false, + release: () => { + if (released) return + released = true + runningExclusive = false + drain() + }, + }) + return + } + + while (queue[0] && !queue[0].exclusive) { + const waiter = queue.shift() + if (!waiter) break + runningShared += 1 + let released = false + waiter.resolve({ + skipped: false, + release: () => { + if (released) return + released = true + runningShared -= 1 + drain() + }, + }) + } + } + + return { + async acquire(exclusive) { + if (denied) return { skipped: true } + return new Promise((resolve) => { + queue.push({ exclusive, resolve }) + drain() + }) + }, + markDenied() { + denied = true + drain() + }, + } +} diff --git a/packages/tools/src/runtime/tool_output_limits.ts b/packages/core/src/tools/runtime/tool_output_limits.ts similarity index 100% rename from packages/tools/src/runtime/tool_output_limits.ts rename to packages/core/src/tools/runtime/tool_output_limits.ts diff --git a/packages/core/src/tools/sdk_tools.test.ts b/packages/core/src/tools/sdk_tools.test.ts new file mode 100644 index 0000000..2ff3d6a --- /dev/null +++ b/packages/core/src/tools/sdk_tools.test.ts @@ -0,0 +1,204 @@ +import { afterEach, describe, expect, test, vi } from 'vitest' +import { jsonSchema, tool, type Tool, type ToolExecutionOptions, type ToolSet } from 'ai' +import { TOOL_SKIPPED_DISABLED_MESSAGE, wrapToolSetWithRuntime } from '@memo/core/tools/sdk_tools' +import type { ToolExecutionContext } from '@memo/core/tools/sdk_tools' +import type { ApprovalManager } from '@memo/core/tools/approval' + +const TOOL_SKIPPED_AFTER_REJECTION_MESSAGE = 'Skipped tool execution after previous rejection.' + +function makeCtx(overrides: Partial = {}): ToolExecutionContext { + return { + approvalManager: { + isDangerousMode: false, + getRiskLevel: () => 'execute', + check: () => ({ needApproval: false, decision: 'auto-execute' }), + recordDecision: () => {}, + isGranted: () => false, + clearOnceApprovals: () => {}, + dispose: () => {}, + }, + approvalHooks: {}, + toolsDisabled: false, + gate: { acquire: async () => ({ skipped: false, release: () => {} }), markDenied: () => {} }, + ...overrides, + } +} + +function makeEcho(execute: Tool['execute'] = async () => ({ type: 'text', value: 'ok' })): Tool { + return tool({ + description: 'echo', + inputSchema: jsonSchema({ type: 'object' }), + execute, + }) +} + +async function runWrapped(tools: ToolSet, name: string, input: unknown, ctx: ToolExecutionContext) { + const wrapped = wrapToolSetWithRuntime(tools) + const wrappedTool = wrapped?.[name] + expect(wrappedTool?.execute).toBeDefined() + const options = { experimental_context: ctx } as unknown as ToolExecutionOptions + return wrappedTool?.execute?.(input, options) +} + +afterEach(() => { + delete process.env.MEMO_TOOL_RESULT_MAX_CHARS +}) + +describe('wrapToolSetWithRuntime', () => { + test('returns undefined for empty tool set', () => { + expect(wrapToolSetWithRuntime({})).toBeUndefined() + }) + + test('executes the underlying tool and returns its output', async () => { + const execute = vi.fn(async () => ({ type: 'text', value: 'ok' })) + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', { text: 'hi' }, makeCtx()) + expect(output).toEqual({ type: 'text', value: 'ok' }) + expect(execute).toHaveBeenCalledWith({ text: 'hi' }, expect.anything()) + }) + + test('returns the disabled sentinel (error-text) without executing when tools are disabled', async () => { + const execute = vi.fn(async () => ({ type: 'text', value: 'must not run' })) + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', {}, makeCtx({ toolsDisabled: true })) + expect(output).toEqual({ type: 'error-text', value: TOOL_SKIPPED_DISABLED_MESSAGE }) + expect(execute).not.toHaveBeenCalled() + }) + + test('denies via approval manager: execution-denied + gate.markDenied', async () => { + const markDenied = vi.fn() + const requestApproval = vi.fn(async () => 'deny' as const) + const onApprovalRequest = vi.fn() + const onApprovalResponse = vi.fn() + const recordDecision = vi.fn() + const execute = vi.fn(async () => ({ type: 'text', value: 'must not run' })) + const ctx = makeCtx({ + approvalManager: { + ...makeCtx().approvalManager, + check: () => ({ + needApproval: true as const, + fingerprint: 'fp-1', + riskLevel: 'execute' as const, + reason: 'risky', + toolName: 'echo', + params: { text: 'hi' }, + }), + recordDecision, + }, + approvalHooks: { onApprovalRequest, onApprovalResponse, requestApproval }, + gate: { acquire: async () => ({ skipped: false, release: () => {} }), markDenied }, + }) + + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', {}, ctx) + + expect(output).toEqual({ type: 'execution-denied', reason: 'User denied tool execution: echo' }) + expect(onApprovalRequest).toHaveBeenCalledOnce() + expect(requestApproval).toHaveBeenCalledOnce() + expect(recordDecision).toHaveBeenCalledWith('fp-1', 'deny') + expect(onApprovalResponse).toHaveBeenCalledWith({ fingerprint: 'fp-1', decision: 'deny' }) + expect(markDenied).toHaveBeenCalledOnce() + expect(execute).not.toHaveBeenCalled() + }) + + test('records granted approvals and skips the check on subsequent calls', async () => { + const recordDecision = vi.fn() + let checkCalls = 0 + const approvalManager: ApprovalManager = { + isDangerousMode: false, + getRiskLevel: () => 'execute', + check: () => { + checkCalls += 1 + if (checkCalls === 1) { + return { + needApproval: true as const, + fingerprint: 'fp-2', + riskLevel: 'execute' as const, + reason: 'risky', + toolName: 'echo', + params: {}, + } + } + return { needApproval: false as const, decision: 'auto-execute' as const } + }, + recordDecision, + isGranted: () => checkCalls > 1, + clearOnceApprovals: () => {}, + dispose: () => {}, + } + const ctx = makeCtx({ + approvalManager, + approvalHooks: { requestApproval: async () => 'once' as const }, + }) + + await runWrapped({ echo: makeEcho() }, 'echo', {}, ctx) + await runWrapped({ echo: makeEcho() }, 'echo', {}, ctx) + + expect(checkCalls).toBe(2) + expect(recordDecision).toHaveBeenCalledWith('fp-2', 'once') + }) + + test('acquires the gate exclusively for mutating tools', async () => { + const acquire = vi.fn(async () => ({ skipped: false, release: () => {} })) + const ctx = makeCtx({ gate: { acquire, markDenied: () => {} } }) + const mutating = tool({ + description: 'write', + inputSchema: jsonSchema({ type: 'object' }), + metadata: { memo: { isMutating: true } }, + execute: async () => ({ type: 'text', value: 'written' }), + }) + await runWrapped({ write: mutating }, 'write', {}, ctx) + expect(acquire).toHaveBeenCalledWith(true) + }) + + test('acquires the gate shared for read-only parallel tools', async () => { + const acquire = vi.fn(async () => ({ skipped: false, release: () => {} })) + const ctx = makeCtx({ gate: { acquire, markDenied: () => {} } }) + await runWrapped({ echo: makeEcho() }, 'echo', {}, ctx) + expect(acquire).toHaveBeenCalledWith(false) + }) + + test('skips with a text notice when the gate rejects (previous denial in step)', async () => { + const execute = vi.fn(async () => ({ type: 'text', value: 'must not run' })) + const ctx = makeCtx({ + gate: { acquire: async () => ({ skipped: true }), markDenied: () => {} }, + }) + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', {}, ctx) + expect(output).toEqual({ type: 'text', value: TOOL_SKIPPED_AFTER_REJECTION_MESSAGE }) + expect(execute).not.toHaveBeenCalled() + }) + + test('truncates oversized text output into a system_hint', async () => { + process.env.MEMO_TOOL_RESULT_MAX_CHARS = '10' + const execute = async () => ({ type: 'text', value: 'x'.repeat(100) }) + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', {}, makeCtx()) + expect(output?.type).toBe('text') + expect((output as { value: string }).value).toContain('system_hint') + expect((output as { value: string }).value).toContain('tool_output_omitted') + }) + + test('wraps execute errors into error-text', async () => { + const execute = async () => { + throw new Error('boom') + } + const output = await runWrapped({ echo: makeEcho(execute) }, 'echo', {}, makeCtx()) + expect(output).toEqual({ type: 'error-text', value: 'Tool execution failed: boom' }) + }) + + test('rethrows when the abort signal is aborted', async () => { + const controller = new AbortController() + controller.abort() + const execute = async () => { + throw new Error('aborted upstream') + } + const wrapped = wrapToolSetWithRuntime({ echo: makeEcho(execute) }) + const options = { + experimental_context: makeCtx(), + abortSignal: controller.signal, + } as unknown as ToolExecutionOptions + await expect(wrapped?.echo?.execute?.({}, options)).rejects.toThrow('aborted upstream') + }) + + test('passes through tools without an execute function untouched', () => { + const providerTool = { type: 'provider' as const, id: 'mock.echo', args: {} } + const wrapped = wrapToolSetWithRuntime({ provider: providerTool as unknown as Tool }) + expect(wrapped?.provider).toBe(providerTool) + }) +}) diff --git a/packages/core/src/tools/sdk_tools.ts b/packages/core/src/tools/sdk_tools.ts new file mode 100644 index 0000000..2b67bbd --- /dev/null +++ b/packages/core/src/tools/sdk_tools.ts @@ -0,0 +1,108 @@ +/** @file Runtime wrapping for standard AI SDK tools: approval + truncation wrapper on execute. */ +import type { Tool, ToolExecutionOptions, ToolSet } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ApprovalDecision, ApprovalManager, ApprovalRequest } from '@memo/core/tools/approval' +import { getMaxToolResultChars } from '@memo/core/tools/runtime/tool_output_limits' +import type { StepGate } from '@memo/core/tools/runtime/step_gate' +import type { SkillIndex } from '@memo/core/skills/skills' + +const TOOL_SKIPPED_AFTER_REJECTION_MESSAGE = 'Skipped tool execution after previous rejection.' +export const TOOL_SKIPPED_DISABLED_MESSAGE = 'Tool execution skipped: tools are disabled in current permission mode.' + +/** Approval UI hooks. */ +export type ToolApprovalHooks = { + onApprovalRequest?: (request: ApprovalRequest) => Promise | void + onApprovalResponse?: (payload: { fingerprint: string; decision: ApprovalDecision }) => Promise | void + requestApproval?: (request: ApprovalRequest) => Promise +} + +/** Per-call context passed to tool execute wrappers via streamText experimental_context. */ +export type ToolExecutionContext = { + approvalManager: ApprovalManager + approvalHooks: ToolApprovalHooks + toolsDisabled: boolean + /** Fresh per streamText call. */ + gate: StepGate + /** Deduped skill snapshot for the current session (read_skill). */ + skillIndex?: SkillIndex +} + +function escapeXmlAttr(value: string) { + return value.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>') +} + +function buildOversizeHintXml(toolName: string, actualChars: number, maxChars: number) { + return `Tool output too long, automatically omitted. Please narrow the scope or add limit parameters and try again.` +} + +function guardToolResultOutput(toolName: string, result: ToolResultOutput): ToolResultOutput { + const maxChars = getMaxToolResultChars() + const actualChars = + result.type === 'text' || result.type === 'error-text' + ? result.value.length + : result.type === 'json' || result.type === 'error-json' + ? JSON.stringify(result.value).length + : 0 + if (actualChars <= maxChars) return result + return { type: 'text', value: buildOversizeHintXml(toolName, actualChars, maxChars) } +} + +/** + * Wrap a standard AI SDK ToolSet with the runtime execute wrapper (approval gate, skip, + * truncation). The per-call context is read from `options.experimental_context`, so the + * wrapper is built once at assembly time and reused across streamText calls. + */ +export function wrapToolSetWithRuntime(tools: ToolSet): ToolSet | undefined { + const entries = Object.entries(tools) + if (entries.length === 0) return undefined + return Object.fromEntries(entries.map(([name, tool]) => [name, wrapTool(name, tool)])) +} + +function wrapTool(name: string, tool: Tool): Tool { + const execute = tool.execute + if (typeof execute !== 'function') return tool + const meta = tool.metadata?.memo as { isMutating?: boolean; supportsParallelToolCalls?: boolean } | undefined + const exclusive = meta?.isMutating === true || meta?.supportsParallelToolCalls === false + return { + ...tool, + execute: async (input, options: ToolExecutionOptions) => { + const ctx = options.experimental_context as ToolExecutionContext + if (ctx.toolsDisabled) return { type: 'error-text', value: TOOL_SKIPPED_DISABLED_MESSAGE } + const permit = await ctx.gate.acquire(exclusive) + if (permit.skipped) return { type: 'text', value: TOOL_SKIPPED_AFTER_REJECTION_MESSAGE } + + try { + // Approval (white-list → classifier → fingerprint cache), UI decision awaited inline. + const check = ctx.approvalManager.check(name, input) + if (check.needApproval) { + const request = { + toolName: check.toolName, + params: check.params, + fingerprint: check.fingerprint, + riskLevel: check.riskLevel, + reason: check.reason, + } + await ctx.approvalHooks.onApprovalRequest?.(request) + const decision = ctx.approvalHooks.requestApproval + ? await ctx.approvalHooks.requestApproval(request) + : 'deny' + ctx.approvalManager.recordDecision(check.fingerprint, decision) + await ctx.approvalHooks.onApprovalResponse?.({ fingerprint: check.fingerprint, decision }) + if (decision === 'deny') { + ctx.gate.markDenied() + return { type: 'execution-denied', reason: `User denied tool execution: ${name}` } + } + } + + // Execute + truncate. Input validation is handled by the SDK inputSchema. + const raw = await execute(input, options) + return guardToolResultOutput(name, raw as ToolResultOutput) + } catch (err) { + if (options.abortSignal?.aborted) throw err + return { type: 'error-text', value: `Tool execution failed: ${(err as Error).message}` } + } finally { + permit.release() + } + }, + } +} diff --git a/packages/tools/src/tools/apply_patch.test.ts b/packages/core/src/tools/tools/apply_patch.test.ts similarity index 92% rename from packages/tools/src/tools/apply_patch.test.ts rename to packages/core/src/tools/tools/apply_patch.test.ts index df1e495..1c4c19c 100644 --- a/packages/tools/src/tools/apply_patch.test.ts +++ b/packages/core/src/tools/tools/apply_patch.test.ts @@ -1,10 +1,13 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { access, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { afterAll, beforeAll, describe, test } from 'vitest' -import { runWithRuntimeContext } from '@memo/tools/runtime/context' -import { applyPatchTool } from '@memo/tools/tools/apply_patch' +import { runWithRuntimeContext } from '@memo/core/tools/runtime/context' +import { applyPatchTool } from '@memo/core/tools/tools/apply_patch' let tempDir: string let prevWritableRoots: string | undefined @@ -24,32 +27,29 @@ async function readText(path: string) { } } -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } -function assertPatchOk(result: { isError?: boolean; content?: Array<{ type: string; text?: string }> }) { +function assertPatchOk(result: ToolOutput) { const payload = textPayload(result) - assert.ok(!result.isError, payload) + assert.ok(result.type !== 'error-text', payload) } -function assertPatchError( - result: { isError?: boolean; content?: Array<{ type: string; text?: string }> }, - includes?: string, -) { - assert.strictEqual(result.isError, true) +function assertPatchError(result: ToolOutput, includes?: string) { + assert.strictEqual(result.type, 'error-text') if (includes) { assert.ok(textPayload(result).includes(includes), textPayload(result)) } } async function executePatch(input: string) { - return runWithRuntimeContext({ cwd: tempDir }, () => applyPatchTool.execute({ input })) + return runWithRuntimeContext({ cwd: tempDir }, () => runTool(applyPatchTool, { input })) } async function executePatchIn(cwd: string, input: string) { - return runWithRuntimeContext({ cwd }, () => applyPatchTool.execute({ input })) + return runWithRuntimeContext({ cwd }, () => runTool(applyPatchTool, { input })) } beforeAll(async () => { @@ -67,6 +67,10 @@ afterAll(async () => { await rm(tempDir, { recursive: true, force: true }) }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('apply_patch tool (structured patch)', () => { test('applies add/update/delete operations in one patch', async () => { await writeFile(join(tempDir, 'modify.txt'), 'line1\nline2\n', 'utf8') @@ -365,7 +369,7 @@ describe('apply_patch tool (structured patch)', () => { }) test('validates required input field', async () => { - const result = await applyPatchTool.execute({} as never) + const result = await runTool(applyPatchTool, {} as never) assertPatchError(result, 'apply_patch invalid input') }) }) diff --git a/packages/tools/src/tools/apply_patch.ts b/packages/core/src/tools/tools/apply_patch.ts similarity index 98% rename from packages/tools/src/tools/apply_patch.ts rename to packages/core/src/tools/tools/apply_patch.ts index e44aec7..c2f73bf 100644 --- a/packages/tools/src/tools/apply_patch.ts +++ b/packages/core/src/tools/tools/apply_patch.ts @@ -1,10 +1,10 @@ import { mkdir, readFile, realpath, rm, writeFile } from 'node:fs/promises' import { basename, dirname, isAbsolute, join } from 'node:path' import { z } from 'zod' -import { getRuntimeCwd } from '@memo/tools/runtime/context' -import { textResult } from '@memo/tools/tools/mcp' -import { normalizePath, writePathDenyReason } from '@memo/tools/tools/helpers' -import { defineMcpTool } from '@memo/tools/tools/types' +import { getRuntimeCwd } from '@memo/core/tools/runtime/context' +import { textResult } from '@memo/core/tools/tools/mcp' +import { normalizePath, writePathDenyReason } from '@memo/core/tools/tools/helpers' +import { tool } from 'ai' const BEGIN_PATCH_MARKER = '*** Begin Patch' const END_PATCH_MARKER = '*** End Patch' @@ -91,8 +91,6 @@ It is important to remember: - File references can only be relative, NEVER ABSOLUTE. ` -type ApplyPatchInput = z.infer - type AddFileHunk = { type: 'add' path: string @@ -704,12 +702,11 @@ function formatParseError(err: ApplyPatchParseError): string { return `Invalid patch: ${err.message}` } -export const applyPatchTool = defineMcpTool({ - name: 'apply_patch', +export const applyPatchTool = tool({ description: APPLY_PATCH_DESCRIPTION, inputSchema: APPLY_PATCH_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async (input) => { const parsed = APPLY_PATCH_INPUT_SCHEMA.safeParse(input) if (!parsed.success) { diff --git a/packages/tools/src/tools/codex_tools.test.ts b/packages/core/src/tools/tools/codex_tools.test.ts similarity index 76% rename from packages/tools/src/tools/codex_tools.test.ts rename to packages/core/src/tools/tools/codex_tools.test.ts index 149de7c..6d69575 100644 --- a/packages/tools/src/tools/codex_tools.test.ts +++ b/packages/core/src/tools/tools/codex_tools.test.ts @@ -1,18 +1,21 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { access, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { afterAll, beforeAll, describe, test } from 'vitest' -import { runWithRuntimeContext } from '@memo/tools/runtime/context' -import { execCommandTool } from '@memo/tools/tools/exec_command' -import { writeStdinTool } from '@memo/tools/tools/write_stdin' -import { applyPatchTool } from '@memo/tools/tools/apply_patch' -import { readTextFileTool } from '@memo/tools/tools/read_text_file' -import { readFilesTool } from '@memo/tools/tools/read_files' -import { listDirectoryTool } from '@memo/tools/tools/list_directory' -import { searchFilesTool } from '@memo/tools/tools/search_files' -import { updatePlanTool } from '@memo/tools/tools/update_plan' -import { getMemoryTool } from '@memo/tools/tools/get_memory' +import { runWithRuntimeContext } from '@memo/core/tools/runtime/context' +import { execCommandTool } from '@memo/core/tools/tools/exec_command' +import { writeStdinTool } from '@memo/core/tools/tools/write_stdin' +import { applyPatchTool } from '@memo/core/tools/tools/apply_patch' +import { readTextFileTool } from '@memo/core/tools/tools/read_text_file' +import { readFilesTool } from '@memo/core/tools/tools/read_files' +import { listDirectoryTool } from '@memo/core/tools/tools/list_directory' +import { searchFilesTool } from '@memo/core/tools/tools/search_files' +import { updatePlanTool } from '@memo/core/tools/tools/update_plan' +import { getMemoryTool } from '@memo/core/tools/tools/get_memory' let tempDir: string let prevWritableRoots: string | undefined @@ -34,9 +37,9 @@ async function readText(path: string) { } } -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } function outputPayload(text: string) { @@ -75,16 +78,20 @@ afterAll(async () => { await rm(tempDir, { recursive: true, force: true }) }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('codex shell family', () => { test('exec_command runs command and returns formatted output', async () => { - const result = await execCommandTool.execute({ cmd: 'echo hello-codex' }) + const result = await runTool(execCommandTool, { cmd: 'echo hello-codex' }) const text = textPayload(result) assert.ok(text.includes('Output:'), 'should contain output section') assert.ok(text.includes('hello-codex'), 'should include command output') }) test('exec_command blocks dangerous shell command with xml hint', async () => { - const result = await execCommandTool.execute({ cmd: 'rm -rf /' }) + const result = await runTool(execCommandTool, { cmd: 'rm -rf /' }) const text = textPayload(result) assert.ok(text.startsWith(' { }) test('write_stdin continues interactive session', async () => { - const started = await execCommandTool.execute({ + const started = await runTool(execCommandTool, { cmd: 'read line; echo "$line"', yield_time_ms: 50, }) @@ -102,7 +109,7 @@ describe('codex shell family', () => { assert.ok(match, `expected running session id, got: ${startedText}`) const sessionId = Number(match?.[1]) - const resumed = await writeStdinTool.execute({ + const resumed = await runTool(writeStdinTool, { session_id: sessionId, chars: 'interactive-ok\n', yield_time_ms: 1000, @@ -113,7 +120,7 @@ describe('codex shell family', () => { }) test('write_stdin blocks dangerous input and keeps session alive', async () => { - const started = await execCommandTool.execute({ + const started = await runTool(execCommandTool, { cmd: 'read line; echo "$line"', yield_time_ms: 50, }) @@ -122,7 +129,7 @@ describe('codex shell family', () => { assert.ok(match, `expected running session id, got: ${startedText}`) const sessionId = Number(match?.[1]) - const blocked = await writeStdinTool.execute({ + const blocked = await runTool(writeStdinTool, { session_id: sessionId, chars: 'rm -rf /\n', yield_time_ms: 50, @@ -131,7 +138,7 @@ describe('codex shell family', () => { assert.ok(blockedText.startsWith(' { }) test('write_stdin can fetch unread output tail after truncation', async () => { - const started = await execCommandTool.execute({ + const started = await runTool(execCommandTool, { cmd: `node -e "process.stdout.write('X'.repeat(5000)); setTimeout(() => {}, 2000)"`, yield_time_ms: 300, max_output_tokens: 10, @@ -153,7 +160,7 @@ describe('codex shell family', () => { let firstChunk = outputPayload(startedText) if (firstChunk.length === 0) { for (let i = 0; i < 5; i += 1) { - const retry = await writeStdinTool.execute({ + const retry = await runTool(writeStdinTool, { session_id: sessionId, yield_time_ms: 100, max_output_tokens: 10, @@ -164,7 +171,7 @@ describe('codex shell family', () => { } assert.strictEqual(firstChunk.length, 40) - const next = await writeStdinTool.execute({ + const next = await runTool(writeStdinTool, { session_id: sessionId, yield_time_ms: 100, max_output_tokens: 2000, @@ -178,10 +185,10 @@ describe('codex shell family', () => { test('exec_command rejects when active session cap is exceeded', async () => { const results = [] for (let i = 0; i < 70; i += 1) { - results.push(await execCommandTool.execute({ cmd: 'sleep 2', yield_time_ms: 0 })) + results.push(await runTool(execCommandTool, { cmd: 'sleep 2', yield_time_ms: 0 })) } const overflow = results.find( - (result) => result.isError && textPayload(result).includes('too many active sessions'), + (result) => result.type === 'error-text' && textPayload(result).includes('too many active sessions'), ) assert.ok(overflow, 'expected active-session cap error') @@ -196,7 +203,7 @@ describe('codex file/search family', () => { await writeFile(target, 'alpha beta alpha', 'utf8') const singleRes = await runWithRuntimeContext({ cwd: tempDir }, () => - applyPatchTool.execute({ + runTool(applyPatchTool, { input: [ '*** Begin Patch', '*** Update File: patched.txt', @@ -207,11 +214,11 @@ describe('codex file/search family', () => { ].join('\n'), }), ) - assert.ok(!singleRes.isError) + assert.ok(singleRes.type === 'text') assert.strictEqual(await readText(target), 'A beta alpha\n') const batchRes = await runWithRuntimeContext({ cwd: tempDir }, () => - applyPatchTool.execute({ + runTool(applyPatchTool, { input: [ '*** Begin Patch', '*** Update File: patched.txt', @@ -222,13 +229,13 @@ describe('codex file/search family', () => { ].join('\n'), }), ) - assert.ok(!batchRes.isError) + assert.ok(batchRes.type === 'text') assert.strictEqual(await readText(target), 'A B A\n') }) test('read_text_file requires valid path in allowed roots', async () => { - const result = await readTextFileTool.execute({ path: '/tmp/not-allowed.txt' }) - assert.strictEqual(result.isError, true) + const result = await runTool(readTextFileTool, { path: '/tmp/not-allowed.txt' }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Access denied')) }) @@ -237,7 +244,7 @@ describe('codex file/search family', () => { await mkdir(nested, { recursive: true }) await writeFile(join(nested, 'a.txt'), 'a', 'utf8') - const result = await listDirectoryTool.execute({ path: nested }) + const result = await runTool(listDirectoryTool, { path: nested }) const text = textPayload(result) assert.ok(text.includes('[FILE] a.txt'), 'should include file label') }) @@ -250,7 +257,7 @@ describe('codex file/search family', () => { await writeFile(fileA, 'A', 'utf8') await writeFile(fileB, 'B', 'utf8') - const result = await readFilesTool.execute({ paths: [fileA, fileB] }) + const result = await runTool(readFilesTool, { paths: [fileA, fileB] }) const text = textPayload(result) assert.ok(text.includes(`${fileA}:\nA`)) assert.ok(text.includes(`${fileB}:\nB`)) @@ -262,7 +269,7 @@ describe('codex file/search family', () => { await writeFile(join(searchRoot, 'm1.txt'), 'needle-here', 'utf8') await writeFile(join(searchRoot, 'm2.md'), 'nothing', 'utf8') - const result = await searchFilesTool.execute({ pattern: '**/*.txt', path: searchRoot }) + const result = await runTool(searchFilesTool, { pattern: '**/*.txt', path: searchRoot }) const text = textPayload(result) assert.ok(text.includes('m1.txt'), 'should include file name') assert.ok(!text.includes('m2.md')) @@ -271,20 +278,20 @@ describe('codex file/search family', () => { describe('codex workflow/context tools', () => { test('update_plan rejects multiple in_progress items', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { plan: [ { step: 'a', status: 'in_progress' }, { step: 'b', status: 'in_progress' }, ], }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('in_progress')) }) test('get_memory reads from MEMO_HOME Agents.md', async () => { const memoryPath = join(tempDir, 'Agents.md') await writeFile(memoryPath, '## Memo Added Memories\n\n- prefers concise answers\n', 'utf8') - const result = await getMemoryTool.execute({ memory_id: 'thread-1' }) + const result = await runTool(getMemoryTool, { memory_id: 'thread-1' }) const text = textPayload(result) assert.ok(text.includes('prefers concise answers')) }) diff --git a/packages/tools/src/tools/collab.test.ts b/packages/core/src/tools/tools/collab.test.ts similarity index 67% rename from packages/tools/src/tools/collab.test.ts rename to packages/core/src/tools/tools/collab.test.ts index c011391..09eb625 100644 --- a/packages/tools/src/tools/collab.test.ts +++ b/packages/core/src/tools/tools/collab.test.ts @@ -1,4 +1,7 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { mkdir, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' @@ -10,7 +13,7 @@ import { sendInputTool, spawnAgentTool, waitTool, -} from '@memo/tools/tools/collab' +} from '@memo/core/tools/tools/collab' let tempDir: string let prevCommand: string | undefined @@ -22,9 +25,9 @@ async function makeTempDir(prefix: string) { return dir } -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } beforeAll(async () => { @@ -91,16 +94,20 @@ afterAll(async () => { await rm(tempDir, { recursive: true, force: true }) }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('collab tools', () => { test('spawn + wait reaches completed status and returns final map payload', async () => { - const spawnResult = await spawnAgentTool.execute({ message: 'echo:hello' }) - assert.strictEqual(spawnResult.isError, false) + const spawnResult = await runTool(spawnAgentTool, { message: 'echo:hello' }) + assert.strictEqual(spawnResult.type, 'text') const spawned = JSON.parse(textPayload(spawnResult)) assert.strictEqual(spawned.status, 'running') assert.ok(typeof spawned.agent_id === 'string' && spawned.agent_id.length > 0) - const waitResult = await waitTool.execute({ ids: [spawned.agent_id], timeout_ms: 10_000 }) - assert.strictEqual(waitResult.isError, false) + const waitResult = await runTool(waitTool, { ids: [spawned.agent_id], timeout_ms: 10_000 }) + assert.strictEqual(waitResult.type, 'text') const waited = JSON.parse(textPayload(waitResult)) assert.strictEqual(waited.timed_out, false) assert.strictEqual(waited.status[spawned.agent_id], 'completed') @@ -112,36 +119,36 @@ describe('collab tools', () => { }) test('close_agent marks closed and resume_agent restores pre-close status', async () => { - const spawnResult = await spawnAgentTool.execute({ message: 'echo:first' }) + const spawnResult = await runTool(spawnAgentTool, { message: 'echo:first' }) const spawned = JSON.parse(textPayload(spawnResult)) const agentId = spawned.agent_id as string - await waitTool.execute({ ids: [agentId], timeout_ms: 10_000 }) + await runTool(waitTool, { ids: [agentId], timeout_ms: 10_000 }) - const closeResult = await closeAgentTool.execute({ id: agentId }) + const closeResult = await runTool(closeAgentTool, { id: agentId }) const closed = JSON.parse(textPayload(closeResult)) assert.strictEqual(closed.status, 'closed') - const sendWhileClosed = await sendInputTool.execute({ + const sendWhileClosed = await runTool(sendInputTool, { id: agentId, message: 'echo:second', }) - assert.strictEqual(sendWhileClosed.isError, true) + assert.strictEqual(sendWhileClosed.type, 'error-text') assert.ok(textPayload(sendWhileClosed).includes('resume_agent')) - const resumeResult = await resumeAgentTool.execute({ id: agentId }) + const resumeResult = await runTool(resumeAgentTool, { id: agentId }) const resumed = JSON.parse(textPayload(resumeResult)) assert.strictEqual(resumed.status, 'completed') - const sendAfterResume = await sendInputTool.execute({ + const sendAfterResume = await runTool(sendInputTool, { id: agentId, message: 'echo:second', }) - assert.strictEqual(sendAfterResume.isError, false) + assert.strictEqual(sendAfterResume.type, 'text') }) test('wait returns not_found immediately for unknown agents', async () => { - const waitResult = await waitTool.execute({ + const waitResult = await runTool(waitTool, { ids: ['missing-agent-id'], timeout_ms: 10_000, }) @@ -154,27 +161,27 @@ describe('collab tools', () => { test('spawn_agent respects MEMO_SUBAGENT_MAX_AGENTS limit', async () => { process.env.MEMO_SUBAGENT_MAX_AGENTS = '1' - const first = await spawnAgentTool.execute({ message: 'sleep:5000' }) - assert.strictEqual(first.isError, false) + const first = await runTool(spawnAgentTool, { message: 'sleep:5000' }) + assert.strictEqual(first.type, 'text') - const second = await spawnAgentTool.execute({ message: 'echo:blocked' }) - assert.strictEqual(second.isError, true) + const second = await runTool(spawnAgentTool, { message: 'echo:blocked' }) + assert.strictEqual(second.type, 'error-text') assert.ok(textPayload(second).includes('concurrency limit')) }) test('wait validates timeout and mutating tools report missing agents', async () => { - const invalidTimeout = await waitTool.execute({ ids: ['missing'], timeout_ms: 0 }) - assert.strictEqual(invalidTimeout.isError, true) + const invalidTimeout = await runTool(waitTool, { ids: ['missing'], timeout_ms: 0 }) + assert.strictEqual(invalidTimeout.type, 'error-text') assert.ok(textPayload(invalidTimeout).includes('timeout_ms')) - const sendResult = await sendInputTool.execute({ id: 'missing', message: 'x' }) - assert.strictEqual(sendResult.isError, true) + const sendResult = await runTool(sendInputTool, { id: 'missing', message: 'x' }) + assert.strictEqual(sendResult.type, 'error-text') assert.ok(textPayload(sendResult).includes('agent not found')) - const closeResult = await closeAgentTool.execute({ id: 'missing' }) - assert.strictEqual(closeResult.isError, true) + const closeResult = await runTool(closeAgentTool, { id: 'missing' }) + assert.strictEqual(closeResult.type, 'error-text') - const resumeResult = await resumeAgentTool.execute({ id: 'missing' }) - assert.strictEqual(resumeResult.isError, true) + const resumeResult = await runTool(resumeAgentTool, { id: 'missing' }) + assert.strictEqual(resumeResult.type, 'error-text') }) }) diff --git a/packages/tools/src/tools/collab.ts b/packages/core/src/tools/tools/collab.ts similarity index 93% rename from packages/tools/src/tools/collab.ts rename to packages/core/src/tools/tools/collab.ts index e17e7c7..8d1f2db 100644 --- a/packages/tools/src/tools/collab.ts +++ b/packages/core/src/tools/tools/collab.ts @@ -2,9 +2,9 @@ import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' import { existsSync } from 'node:fs' import { resolve } from 'node:path' import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { getRuntimeCwd } from '@memo/tools/runtime/context' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { getRuntimeCwd } from '@memo/core/tools/runtime/context' type AgentStatus = 'running' | 'completed' | 'errored' | 'closed' type WaitStatus = AgentStatus | 'not_found' @@ -81,12 +81,6 @@ const CLOSE_AGENT_INPUT_SCHEMA = z }) .strict() -type SpawnInput = z.infer -type SendInput = z.infer -type ResumeInput = z.infer -type WaitInput = z.infer -type CloseInput = z.infer - function nowIso() { return new Date().toISOString() } @@ -345,12 +339,11 @@ export async function __resetCollabStateForTests() { agents.clear() } -export const spawnAgentTool = defineMcpTool({ - name: 'spawn_agent', +export const spawnAgentTool = tool({ description: 'Spawn a sub-agent for a well-scoped task and return the agent id.', inputSchema: SPAWN_AGENT_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async ({ message }) => { const trimmed = message.trim() if (!trimmed) { @@ -392,12 +385,11 @@ export const spawnAgentTool = defineMcpTool({ }, }) -export const sendInputTool = defineMcpTool({ - name: 'send_input', +export const sendInputTool = tool({ description: 'Send a message to an existing agent.', inputSchema: SEND_INPUT_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async ({ id, message, interrupt }) => { const record = agents.get(id) if (!record) return buildMissingAgentError(id) @@ -440,12 +432,11 @@ export const sendInputTool = defineMcpTool({ }, }) -export const resumeAgentTool = defineMcpTool({ - name: 'resume_agent', +export const resumeAgentTool = tool({ description: 'Resume a previously closed agent by id.', inputSchema: RESUME_AGENT_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async ({ id }) => { const record = agents.get(id) if (!record) return buildMissingAgentError(id) @@ -468,12 +459,11 @@ export const resumeAgentTool = defineMcpTool({ }, }) -export const waitTool = defineMcpTool({ - name: 'wait', +export const waitTool = tool({ description: 'Wait for agent statuses and return current snapshots.', inputSchema: WAIT_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: false } }, + execute: async ({ ids, timeout_ms }) => { const resolvedTimeout = clampWaitTimeout(timeout_ms) if (resolvedTimeout === null) { @@ -541,12 +531,11 @@ export const waitTool = defineMcpTool({ }, }) -export const closeAgentTool = defineMcpTool({ - name: 'close_agent', +export const closeAgentTool = tool({ description: 'Close an agent and return its last known status.', inputSchema: CLOSE_AGENT_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async ({ id }) => { const record = agents.get(id) if (!record) return buildMissingAgentError(id) diff --git a/packages/tools/src/tools/command_guard.test.ts b/packages/core/src/tools/tools/command_guard.test.ts similarity index 98% rename from packages/tools/src/tools/command_guard.test.ts rename to packages/core/src/tools/tools/command_guard.test.ts index 3dd0817..ca340d8 100644 --- a/packages/tools/src/tools/command_guard.test.ts +++ b/packages/core/src/tools/tools/command_guard.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert' import { describe, test } from 'vitest' -import { detectDangerousCommand, guardDangerousCommand } from '@memo/tools/tools/command_guard' +import { detectDangerousCommand, guardDangerousCommand } from '@memo/core/tools/tools/command_guard' describe('command guard', () => { test('detects dangerous delete and disk mutation commands', () => { diff --git a/packages/tools/src/tools/command_guard.ts b/packages/core/src/tools/tools/command_guard.ts similarity index 100% rename from packages/tools/src/tools/command_guard.ts rename to packages/core/src/tools/tools/command_guard.ts diff --git a/packages/tools/src/tools/edit_file.ts b/packages/core/src/tools/tools/edit_file.ts similarity index 69% rename from packages/tools/src/tools/edit_file.ts rename to packages/core/src/tools/tools/edit_file.ts index 6869af9..097c6e8 100644 --- a/packages/tools/src/tools/edit_file.ts +++ b/packages/core/src/tools/tools/edit_file.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { applyFileEdits, validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { applyFileEdits, validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const EDIT_FILE_INPUT_SCHEMA = z .object({ @@ -21,14 +21,11 @@ const EDIT_FILE_INPUT_SCHEMA = z }) .strict() -type EditFileInput = z.infer - -export const editFileTool = defineMcpTool({ - name: 'edit_file', +export const editFileTool = tool({ description: 'Apply ordered edit operations to a text file and return a unified diff (dryRun previews only).', inputSchema: EDIT_FILE_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/tools/src/tools/exec_command.test.ts b/packages/core/src/tools/tools/exec_command.test.ts similarity index 75% rename from packages/tools/src/tools/exec_command.test.ts rename to packages/core/src/tools/tools/exec_command.test.ts index 247537b..e0720ec 100644 --- a/packages/tools/src/tools/exec_command.test.ts +++ b/packages/core/src/tools/tools/exec_command.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' import { describe, test, vi, beforeEach, afterEach, expect } from 'vitest' import { execCommandTool } from './exec_command' import { flattenText } from './mcp' @@ -13,6 +15,10 @@ vi.mock('./exec_runtime', async () => { import { startExecSession } from './exec_runtime' +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('exec_command tool', () => { beforeEach(() => { vi.resetAllMocks() @@ -26,9 +32,9 @@ describe('exec_command tool', () => { test('executes command and returns output', async () => { vi.mocked(startExecSession).mockResolvedValue('test output') - const result = await execCommandTool.execute({ cmd: 'echo hello' }) + const result = await runTool(execCommandTool, { cmd: 'echo hello' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), 'test output') expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -42,18 +48,18 @@ describe('exec_command tool', () => { const multiLineOutput = 'Line 1\nLine 2\nLine 3' vi.mocked(startExecSession).mockResolvedValue(multiLineOutput) - const result = await execCommandTool.execute({ cmd: 'printf "Line 1\nLine 2\nLine 3"' }) + const result = await runTool(execCommandTool, { cmd: 'printf "Line 1\nLine 2\nLine 3"' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), multiLineOutput) }) test('handles empty output', async () => { vi.mocked(startExecSession).mockResolvedValue('') - const result = await execCommandTool.execute({ cmd: 'true' }) + const result = await runTool(execCommandTool, { cmd: 'true' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), '') }) @@ -61,9 +67,9 @@ describe('exec_command tool', () => { const unicodeOutput = '你好世界 🌍 Привет мир' vi.mocked(startExecSession).mockResolvedValue(unicodeOutput) - const result = await execCommandTool.execute({ cmd: 'echo "你好世界 🌍 Привет мир"' }) + const result = await runTool(execCommandTool, { cmd: 'echo "你好世界 🌍 Привет мир"' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), unicodeOutput) }) }) @@ -72,7 +78,7 @@ describe('exec_command tool', () => { test('passes optional workdir parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'pwd', workdir: '/tmp' }) + await runTool(execCommandTool, { cmd: 'pwd', workdir: '/tmp' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -84,7 +90,7 @@ describe('exec_command tool', () => { test('passes optional shell parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'echo test', shell: '/bin/zsh' }) + await runTool(execCommandTool, { cmd: 'echo test', shell: '/bin/zsh' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -96,7 +102,7 @@ describe('exec_command tool', () => { test('passes different shell types', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'echo test', shell: '/usr/bin/fish' }) + await runTool(execCommandTool, { cmd: 'echo test', shell: '/usr/bin/fish' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -108,7 +114,7 @@ describe('exec_command tool', () => { test('passes login parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'whoami', login: true }) + await runTool(execCommandTool, { cmd: 'whoami', login: true }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -120,7 +126,7 @@ describe('exec_command tool', () => { test('passes tty parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'top', tty: true }) + await runTool(execCommandTool, { cmd: 'top', tty: true }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -132,7 +138,7 @@ describe('exec_command tool', () => { test('passes yield_time_ms parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'sleep 1', yield_time_ms: 10000 }) + await runTool(execCommandTool, { cmd: 'sleep 1', yield_time_ms: 10000 }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -144,7 +150,7 @@ describe('exec_command tool', () => { test('passes max_output_tokens parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('truncated output') - await execCommandTool.execute({ cmd: 'cat large_file.txt', max_output_tokens: 1000 }) + await runTool(execCommandTool, { cmd: 'cat large_file.txt', max_output_tokens: 1000 }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -156,7 +162,7 @@ describe('exec_command tool', () => { test('passes sandbox_permissions parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'ls', sandbox_permissions: 'require_escalated' }) + await runTool(execCommandTool, { cmd: 'ls', sandbox_permissions: 'require_escalated' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -168,7 +174,7 @@ describe('exec_command tool', () => { test('passes justification parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ + await runTool(execCommandTool, { cmd: 'cat config', justification: 'Reading config for debugging', }) @@ -183,7 +189,7 @@ describe('exec_command tool', () => { test('passes prefix_rule parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await execCommandTool.execute({ cmd: 'ls -la', prefix_rule: ['safe'] }) + await runTool(execCommandTool, { cmd: 'ls -la', prefix_rule: ['safe'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -197,9 +203,9 @@ describe('exec_command tool', () => { test('handles execution errors gracefully', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command failed')) - const result = await execCommandTool.execute({ cmd: 'invalid-command' }) + const result = await runTool(execCommandTool, { cmd: 'invalid-command' }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('exec_command failed')) assert.ok(flattenText(result).includes('command failed')) }) @@ -207,27 +213,27 @@ describe('exec_command tool', () => { test('handles command not found errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('ENOENT: no such file or directory')) - const result = await execCommandTool.execute({ cmd: 'definitely-not-a-command' }) + const result = await runTool(execCommandTool, { cmd: 'definitely-not-a-command' }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('exec_command failed')) }) test('handles permission denied errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('EACCES: permission denied')) - const result = await execCommandTool.execute({ cmd: '/root/protected' }) + const result = await runTool(execCommandTool, { cmd: '/root/protected' }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('exec_command failed')) }) test('handles timeout errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command timed out after 30000ms')) - const result = await execCommandTool.execute({ cmd: 'sleep 60', timeout_ms: 1000 }) + const result = await runTool(execCommandTool, { cmd: 'sleep 60', timeout_ms: 1000 }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('exec_command failed')) }) }) @@ -236,7 +242,7 @@ describe('exec_command tool', () => { test('handles complex command with arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('result') - await execCommandTool.execute({ + await runTool(execCommandTool, { cmd: 'grep -r "pattern" /path/to/search --include="*.js" -l', }) @@ -250,7 +256,7 @@ describe('exec_command tool', () => { test('handles command with quotes', async () => { vi.mocked(startExecSession).mockResolvedValue('quoted') - await execCommandTool.execute({ cmd: 'echo "hello world"' }) + await runTool(execCommandTool, { cmd: 'echo "hello world"' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -262,7 +268,7 @@ describe('exec_command tool', () => { test('handles command with backticks', async () => { vi.mocked(startExecSession).mockResolvedValue('backtick') - await execCommandTool.execute({ cmd: 'echo `date`' }) + await runTool(execCommandTool, { cmd: 'echo `date`' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/tools/src/tools/exec_command.ts b/packages/core/src/tools/tools/exec_command.ts similarity index 74% rename from packages/tools/src/tools/exec_command.ts rename to packages/core/src/tools/tools/exec_command.ts index babfa7a..4661163 100644 --- a/packages/tools/src/tools/exec_command.ts +++ b/packages/core/src/tools/tools/exec_command.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { startExecSession } from '@memo/tools/tools/exec_runtime' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { startExecSession } from '@memo/core/tools/tools/exec_runtime' const EXEC_COMMAND_INPUT_SCHEMA = z .object({ @@ -18,15 +18,12 @@ const EXEC_COMMAND_INPUT_SCHEMA = z }) .strict() -type ExecCommandInput = z.infer - -export const execCommandTool = defineMcpTool({ - name: 'exec_command', +export const execCommandTool = tool({ description: 'Runs a command in a PTY-like managed session, returning output or a session ID for ongoing interaction.', inputSchema: EXEC_COMMAND_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: true } }, + execute: async (input) => { try { const content = await startExecSession({ diff --git a/packages/tools/src/tools/exec_runtime.test.ts b/packages/core/src/tools/tools/exec_runtime.test.ts similarity index 100% rename from packages/tools/src/tools/exec_runtime.test.ts rename to packages/core/src/tools/tools/exec_runtime.test.ts diff --git a/packages/tools/src/tools/exec_runtime.ts b/packages/core/src/tools/tools/exec_runtime.ts similarity index 99% rename from packages/tools/src/tools/exec_runtime.ts rename to packages/core/src/tools/tools/exec_runtime.ts index a42ed36..9dac548 100644 --- a/packages/tools/src/tools/exec_runtime.ts +++ b/packages/core/src/tools/tools/exec_runtime.ts @@ -1,8 +1,8 @@ import { spawn } from 'node:child_process' import { EventEmitter } from 'node:events' import { resolve } from 'node:path' -import { guardDangerousCommand, splitStdinLines, trimPendingStdinBuffer } from '@memo/tools/tools/command_guard' -import { getRuntimeCwd } from '@memo/tools/runtime/context' +import { guardDangerousCommand, splitStdinLines, trimPendingStdinBuffer } from '@memo/core/tools/tools/command_guard' +import { getRuntimeCwd } from '@memo/core/tools/runtime/context' const DEFAULT_EXEC_YIELD_TIME_MS = 10_000 const DEFAULT_WRITE_YIELD_TIME_MS = 250 diff --git a/packages/tools/src/tools/filesystem/lib.ts b/packages/core/src/tools/tools/filesystem/lib.ts similarity index 100% rename from packages/tools/src/tools/filesystem/lib.ts rename to packages/core/src/tools/tools/filesystem/lib.ts diff --git a/packages/tools/src/tools/filesystem/path-utils.ts b/packages/core/src/tools/tools/filesystem/path-utils.ts similarity index 100% rename from packages/tools/src/tools/filesystem/path-utils.ts rename to packages/core/src/tools/tools/filesystem/path-utils.ts diff --git a/packages/tools/src/tools/filesystem/path-validation.ts b/packages/core/src/tools/tools/filesystem/path-validation.ts similarity index 100% rename from packages/tools/src/tools/filesystem/path-validation.ts rename to packages/core/src/tools/tools/filesystem/path-validation.ts diff --git a/packages/tools/src/tools/filesystem/roots.ts b/packages/core/src/tools/tools/filesystem/roots.ts similarity index 97% rename from packages/tools/src/tools/filesystem/roots.ts rename to packages/core/src/tools/tools/filesystem/roots.ts index 1fc7b72..c9bd5ef 100644 --- a/packages/tools/src/tools/filesystem/roots.ts +++ b/packages/core/src/tools/tools/filesystem/roots.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'node:fs' import path from 'node:path' -import { getRuntimeCwd } from '@memo/tools/runtime/context' +import { getRuntimeCwd } from '@memo/core/tools/runtime/context' import { expandHome, normalizePath } from './path-utils' const FS_ALLOWED_ROOTS_ENV = 'MEMO_FS_ALLOWED_ROOTS' diff --git a/packages/tools/src/tools/filesystem_tools.test.ts b/packages/core/src/tools/tools/filesystem_tools.test.ts similarity index 66% rename from packages/tools/src/tools/filesystem_tools.test.ts rename to packages/core/src/tools/tools/filesystem_tools.test.ts index 02013e5..3774719 100644 --- a/packages/tools/src/tools/filesystem_tools.test.ts +++ b/packages/core/src/tools/tools/filesystem_tools.test.ts @@ -1,21 +1,23 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' import { basename, join } from 'node:path' import { tmpdir } from 'node:os' import { afterEach, beforeEach, describe, test } from 'vitest' -import { readTextFileTool } from '@memo/tools/tools/read_text_file' -import { readMediaFileTool } from '@memo/tools/tools/read_media_file' -import { readFilesTool } from '@memo/tools/tools/read_files' -import { writeFileTool } from '@memo/tools/tools/write_file' -import { editFileTool } from '@memo/tools/tools/edit_file' -import { listDirectoryTool } from '@memo/tools/tools/list_directory' -import { searchFilesTool } from '@memo/tools/tools/search_files' - -type ToolResult = { content?: Array<{ type: string; text?: string }>; isError?: boolean } - -function textPayload(result: ToolResult): string { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +import { z } from 'zod' +import { readTextFileTool } from '@memo/core/tools/tools/read_text_file' +import { readMediaFileTool } from '@memo/core/tools/tools/read_media_file' +import { readFilesTool } from '@memo/core/tools/tools/read_files' +import { writeFileTool } from '@memo/core/tools/tools/write_file' +import { editFileTool } from '@memo/core/tools/tools/edit_file' +import { listDirectoryTool } from '@memo/core/tools/tools/list_directory' +import { searchFilesTool } from '@memo/core/tools/tools/search_files' + +function textPayload(result: ToolOutput): string { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } let rootDir = '' @@ -40,33 +42,37 @@ afterEach(async () => { await rm(outsideDir, { recursive: true, force: true }) }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('filesystem tools', () => { test('tool schemas reject invalid empty path input', () => { - const readValidation = readTextFileTool.validateInput?.({ path: '' }) - assert.strictEqual(readValidation?.ok, false) + const readSchema = readTextFileTool.inputSchema as z.ZodTypeAny + assert.strictEqual(readSchema.safeParse({ path: '' }).success, false) - const listValidation = listDirectoryTool.validateInput?.({ path: '' }) - assert.strictEqual(listValidation?.ok, false) + const listSchema = listDirectoryTool.inputSchema as z.ZodTypeAny + assert.strictEqual(listSchema.safeParse({ path: '' }).success, false) }) test('read_text_file reads full content and supports head/tail', async () => { const filePath = join(rootDir, 'a.txt') await writeFile(filePath, 'line1\nline2\nline3\n', 'utf8') - const full = await readTextFileTool.execute({ path: filePath }) - assert.strictEqual(full.isError, false) + const full = await runTool(readTextFileTool, { path: filePath }) + assert.strictEqual(full.type, 'text') assert.strictEqual(textPayload(full), 'line1\nline2\nline3\n') - const head = await readTextFileTool.execute({ path: filePath, head: 2 }) - assert.strictEqual(head.isError, false) + const head = await runTool(readTextFileTool, { path: filePath, head: 2 }) + assert.strictEqual(head.type, 'text') assert.strictEqual(textPayload(head), 'line1\nline2') - const tail = await readTextFileTool.execute({ path: filePath, tail: 2 }) - assert.strictEqual(tail.isError, false) + const tail = await runTool(readTextFileTool, { path: filePath, tail: 2 }) + assert.strictEqual(tail.type, 'text') assert.strictEqual(textPayload(tail), 'line3\n') - const invalid = await readTextFileTool.execute({ path: filePath, head: 1, tail: 1 }) - assert.strictEqual(invalid.isError, true) + const invalid = await runTool(readTextFileTool, { path: filePath, head: 1, tail: 1 }) + assert.strictEqual(invalid.type, 'error-text') assert.ok(textPayload(invalid).includes('Cannot specify both head and tail')) }) @@ -74,8 +80,8 @@ describe('filesystem tools', () => { const filePath = join(rootDir, 'img.png') await writeFile(filePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) - const result = await readMediaFileTool.execute({ path: filePath }) - assert.strictEqual(result.isError, false) + const result = await runTool(readMediaFileTool, { path: filePath }) + assert.strictEqual(result.type, 'text') const payload = JSON.parse(textPayload(result)) as { type: string @@ -95,8 +101,8 @@ describe('filesystem tools', () => { await writeFile(first, 'one', 'utf8') await writeFile(second, 'two', 'utf8') - const result = await readFilesTool.execute({ paths: [first, missing, second] }) - assert.strictEqual(result.isError, false) + const result = await runTool(readFilesTool, { paths: [first, missing, second] }) + assert.strictEqual(result.type, 'text') const text = textPayload(result) assert.ok(text.includes(`${first}:\none`)) assert.ok(text.includes(`${missing}: Error -`)) @@ -106,20 +112,20 @@ describe('filesystem tools', () => { test('write_file writes and overwrites content', async () => { const filePath = join(rootDir, 'write.txt') - const first = await writeFileTool.execute({ path: filePath, content: 'alpha' }) - assert.strictEqual(first.isError, false) + const first = await runTool(writeFileTool, { path: filePath, content: 'alpha' }) + assert.strictEqual(first.type, 'text') assert.ok(textPayload(first).includes('Successfully wrote')) assert.strictEqual(await readFile(filePath, 'utf8'), 'alpha') - const second = await writeFileTool.execute({ path: filePath, content: 'beta' }) - assert.strictEqual(second.isError, false) + const second = await runTool(writeFileTool, { path: filePath, content: 'beta' }) + assert.strictEqual(second.type, 'text') assert.strictEqual(await readFile(filePath, 'utf8'), 'beta') }) test('write_file fails when parent directory is missing', async () => { const missingParentPath = join(rootDir, 'missing', 'write.txt') - const result = await writeFileTool.execute({ path: missingParentPath, content: 'alpha' }) - assert.strictEqual(result.isError, true) + const result = await runTool(writeFileTool, { path: missingParentPath, content: 'alpha' }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Parent directory does not exist')) }) @@ -127,24 +133,24 @@ describe('filesystem tools', () => { const filePath = join(rootDir, 'edit.ts') await writeFile(filePath, 'a\n b\n c\n', 'utf8') - const dryRun = await editFileTool.execute({ + const dryRun = await runTool(editFileTool, { path: filePath, dryRun: true, edits: [{ oldText: ' b', newText: ' bb' }], }) - assert.strictEqual(dryRun.isError, false) + assert.strictEqual(dryRun.type, 'text') const dryText = textPayload(dryRun) assert.ok(dryText.includes('```diff')) assert.strictEqual(await readFile(filePath, 'utf8'), 'a\n b\n c\n') - const applied = await editFileTool.execute({ + const applied = await runTool(editFileTool, { path: filePath, edits: [ { oldText: 'a', newText: 'aa' }, { oldText: ' c', newText: ' cc' }, ], }) - assert.strictEqual(applied.isError, false) + assert.strictEqual(applied.type, 'text') assert.strictEqual(await readFile(filePath, 'utf8'), 'aa\n b\n cc\n') }) @@ -152,11 +158,11 @@ describe('filesystem tools', () => { const filePath = join(rootDir, 'edit-error.ts') await writeFile(filePath, 'alpha\nbeta\n', 'utf8') - const result = await editFileTool.execute({ + const result = await runTool(editFileTool, { path: filePath, edits: [{ oldText: 'missing', newText: 'value' }], }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Could not find exact match for edit')) }) @@ -164,11 +170,11 @@ describe('filesystem tools', () => { const filePath = join(rootDir, 'edit-crlf.ts') await writeFile(filePath, 'line1\r\nline2\r\nline3\r\n', 'utf8') - const result = await editFileTool.execute({ + const result = await runTool(editFileTool, { path: filePath, edits: [{ oldText: 'line1\nline2', newText: 'line1\nline2-updated' }], }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.ok(textPayload(result).includes('```diff')) assert.strictEqual(await readFile(filePath, 'utf8'), 'line1\nline2-updated\nline3\n') }) @@ -178,8 +184,8 @@ describe('filesystem tools', () => { await mkdir(nested, { recursive: true }) await writeFile(join(rootDir, 'file.txt'), 'x', 'utf8') - const result = await listDirectoryTool.execute({ path: rootDir }) - assert.strictEqual(result.isError, false) + const result = await runTool(listDirectoryTool, { path: rootDir }) + assert.strictEqual(result.type, 'text') const text = textPayload(result) assert.ok(text.includes('[DIR] nested')) @@ -193,18 +199,18 @@ describe('filesystem tools', () => { await writeFile(join(rootDir, 'skip.log'), 's', 'utf8') await writeFile(join(srcDir, 'inside.txt'), 'i', 'utf8') - const matched = await searchFilesTool.execute({ + const matched = await runTool(searchFilesTool, { path: rootDir, pattern: '**/*.txt', excludePatterns: ['src/**'], }) - assert.strictEqual(matched.isError, false) + assert.strictEqual(matched.type, 'text') const text = textPayload(matched) assert.ok(text.includes(join(rootDir, 'keep.txt'))) assert.ok(!text.includes(join(srcDir, 'inside.txt'))) - const none = await searchFilesTool.execute({ path: rootDir, pattern: '**/*.md' }) - assert.strictEqual(none.isError, false) + const none = await runTool(searchFilesTool, { path: rootDir, pattern: '**/*.md' }) + assert.strictEqual(none.type, 'text') assert.strictEqual(textPayload(none), 'No matches found') }) @@ -212,8 +218,8 @@ describe('filesystem tools', () => { const outsideFile = join(outsideDir, 'outside.txt') await writeFile(outsideFile, 'outside', 'utf8') - const result = await readTextFileTool.execute({ path: outsideFile }) - assert.strictEqual(result.isError, true) + const result = await runTool(readTextFileTool, { path: outsideFile }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Access denied - path outside allowed directories')) }) @@ -222,9 +228,9 @@ describe('filesystem tools', () => { await writeFile(outsideFile, 'outside', 'utf8') const traversalPath = join('..', basename(outsideDir), 'outside-traversal.txt') - const result = await readTextFileTool.execute({ path: traversalPath }) + const result = await runTool(readTextFileTool, { path: traversalPath }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Access denied - path outside allowed directories')) }) @@ -243,8 +249,8 @@ describe('filesystem tools', () => { throw error } - const result = await readTextFileTool.execute({ path: linkedPath }) - assert.strictEqual(result.isError, true) + const result = await runTool(readTextFileTool, { path: linkedPath }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('Access denied - symlink target outside allowed directories')) }) }) diff --git a/packages/tools/src/tools/get_memory.test.ts b/packages/core/src/tools/tools/get_memory.test.ts similarity index 63% rename from packages/tools/src/tools/get_memory.test.ts rename to packages/core/src/tools/tools/get_memory.test.ts index 9a89f1e..27e2253 100644 --- a/packages/tools/src/tools/get_memory.test.ts +++ b/packages/core/src/tools/tools/get_memory.test.ts @@ -1,9 +1,12 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { mkdir, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { tmpdir } from 'node:os' import { afterAll, beforeAll, describe, test } from 'vitest' -import { getMemoryTool } from '@memo/tools/tools/get_memory' +import { getMemoryTool } from '@memo/core/tools/tools/get_memory' let tempDir: string let prevMemoHome: string | undefined @@ -14,9 +17,9 @@ async function makeTempDir(prefix: string) { return dir } -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } beforeAll(async () => { @@ -34,10 +37,14 @@ afterAll(async () => { await rm(tempDir, { recursive: true, force: true }) }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('get_memory tool', () => { test('returns missing error when Agents.md does not exist', async () => { - const result = await getMemoryTool.execute({ memory_id: 'missing-thread' }) - assert.strictEqual(result.isError, true) + const result = await runTool(getMemoryTool, { memory_id: 'missing-thread' }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('memory not found')) }) @@ -45,8 +52,8 @@ describe('get_memory tool', () => { const memoryPath = join(tempDir, 'Agents.md') await writeFile(memoryPath, '## Memo Added Memories\n\n- prefers concise output\n', 'utf8') - const result = await getMemoryTool.execute({ memory_id: 'thread-1' }) - assert.ok(!result.isError) + const result = await runTool(getMemoryTool, { memory_id: 'thread-1' }) + assert.ok(result.type === 'text') const parsed = JSON.parse(textPayload(result)) assert.strictEqual(parsed.memory_id, 'thread-1') diff --git a/packages/tools/src/tools/get_memory.ts b/packages/core/src/tools/tools/get_memory.ts similarity index 77% rename from packages/tools/src/tools/get_memory.ts rename to packages/core/src/tools/tools/get_memory.ts index d23d96c..76b6494 100644 --- a/packages/tools/src/tools/get_memory.ts +++ b/packages/core/src/tools/tools/get_memory.ts @@ -2,8 +2,8 @@ import { readFile } from 'node:fs/promises' import { homedir } from 'node:os' import { join } from 'node:path' import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' const GET_MEMORY_INPUT_SCHEMA = z .object({ @@ -11,19 +11,16 @@ const GET_MEMORY_INPUT_SCHEMA = z }) .strict() -type GetMemoryInput = z.infer - function resolveMemoryPath() { const base = process.env.MEMO_HOME?.trim() || join(homedir(), '.memo') return join(base, 'Agents.md') } -export const getMemoryTool = defineMcpTool({ - name: 'get_memory', +export const getMemoryTool = tool({ description: 'Loads the stored memory payload for a memory_id.', inputSchema: GET_MEMORY_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async ({ memory_id }) => { try { const memoryPath = resolveMemoryPath() diff --git a/packages/tools/src/tools/helpers.test.ts b/packages/core/src/tools/tools/helpers.test.ts similarity index 99% rename from packages/tools/src/tools/helpers.test.ts rename to packages/core/src/tools/tools/helpers.test.ts index 790f8b5..ce62395 100644 --- a/packages/tools/src/tools/helpers.test.ts +++ b/packages/core/src/tools/tools/helpers.test.ts @@ -10,7 +10,7 @@ import { isWritePathAllowed, normalizePath, writePathDenyReason, -} from '@memo/tools/tools/helpers' +} from '@memo/core/tools/tools/helpers' const tempDirs: string[] = [] diff --git a/packages/tools/src/tools/helpers.ts b/packages/core/src/tools/tools/helpers.ts similarity index 97% rename from packages/tools/src/tools/helpers.ts rename to packages/core/src/tools/tools/helpers.ts index 7c699b0..d619b9f 100644 --- a/packages/tools/src/tools/helpers.ts +++ b/packages/core/src/tools/tools/helpers.ts @@ -3,8 +3,8 @@ import { homedir } from 'node:os' import { existsSync, statSync, realpathSync } from 'node:fs' import { readFile } from 'node:fs/promises' import ignore from 'ignore' -import { getRuntimeCwd } from '@memo/tools/runtime/context' -import { getMaxToolResultChars, getMaxToolResultLines } from '@memo/tools/runtime/tool_output_limits' +import { getRuntimeCwd } from '@memo/core/tools/runtime/context' +import { getMaxToolResultChars, getMaxToolResultLines } from '@memo/core/tools/runtime/tool_output_limits' /** * 生成标准化的绝对路径，避免因工作目录差异导致的路径错误。 diff --git a/packages/tools/src/tools/list_directory.ts b/packages/core/src/tools/tools/list_directory.ts similarity index 65% rename from packages/tools/src/tools/list_directory.ts rename to packages/core/src/tools/tools/list_directory.ts index f981a19..51bb94d 100644 --- a/packages/tools/src/tools/list_directory.ts +++ b/packages/core/src/tools/tools/list_directory.ts @@ -1,9 +1,9 @@ import fs from 'node:fs/promises' import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const LIST_DIRECTORY_INPUT_SCHEMA = z .object({ @@ -11,14 +11,11 @@ const LIST_DIRECTORY_INPUT_SCHEMA = z }) .strict() -type ListDirectoryInput = z.infer - -export const listDirectoryTool = defineMcpTool({ - name: 'list_directory', +export const listDirectoryTool = tool({ description: 'List direct children of a directory using [DIR]/[FILE] labels.', inputSchema: LIST_DIRECTORY_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/core/src/tools/tools/mcp.test.ts b/packages/core/src/tools/tools/mcp.test.ts new file mode 100644 index 0000000..94fad4a --- /dev/null +++ b/packages/core/src/tools/tools/mcp.test.ts @@ -0,0 +1,134 @@ +import assert from 'node:assert' +import { describe, test } from 'vitest' +import { textResult, flattenText } from './mcp' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' + +describe('mcp helpers', () => { + describe('textResult', () => { + test('creates successful text result', () => { + const result = textResult('hello world') + if (result.type === 'text') { + if (result.type === 'text') { + assert.strictEqual(result.value, 'hello world') + } + } + assert.strictEqual(result.type, 'text') + }) + + test('creates error text result', () => { + const result = textResult('error message', true) + if (result.type === 'error-text') { + assert.strictEqual(result.value, 'error message') + } + assert.strictEqual(result.type, 'error-text') + }) + + test('handles empty string', () => { + const result = textResult('') + if (result.type === 'text') { + assert.strictEqual(result.value, '') + } + assert.strictEqual(result.type, 'text') + }) + + test('handles unicode content', () => { + const result = textResult('你好世界 🌍 Привет') + if (result.type === 'text') { + assert.strictEqual(result.value, '你好世界 🌍 Привет') + } + }) + + test('handles multi-line content', () => { + const result = textResult('line1\nline2\nline3') + if (result.type === 'text') { + assert.strictEqual(result.value, 'line1\nline2\nline3') + } + }) + + test('handles special characters', () => { + const result = textResult('\n') + if (result.type === 'text') { + assert.strictEqual(result.value, '\n') + } + }) + + test('handles very long content', () => { + const longContent = 'x'.repeat(100000) + const result = textResult(longContent) + if (result.type === 'text') { + assert.strictEqual(result.value.length, 100000) + } + }) + + test('handles JSON-like content', () => { + const result = textResult('{"key": "value", "nested": {"a": 1}}') + if (result.type === 'text') { + assert.ok(result.value.includes('"key"')) + } + }) + }) + + describe('flattenText', () => { + test('extracts text from single content item', () => { + const result = textResult('single line') + assert.strictEqual(flattenText(result), 'single line') + }) + + test('joins multiple text content items', () => { + const result: ToolOutput = { type: 'text', value: 'line1\nline2' } + assert.strictEqual(flattenText(result), 'line1\nline2') + }) + + test('ignores non-text content', () => { + const result: ToolOutput = { type: 'text', value: 'visible\nalso visible' } + assert.strictEqual(flattenText(result), 'visible\nalso visible') + }) + + test('handles empty result', () => { + const result: ToolOutput = { type: 'text', value: '' } + assert.strictEqual(flattenText(result), '') + }) + + test('handles empty text', () => { + const result: ToolOutput = { type: 'text', value: '' } + assert.strictEqual(flattenText(result), '') + }) + + test('handles json output', () => { + const result: ToolOutput = { type: 'json', value: { a: 1 } } + assert.strictEqual(flattenText(result), '{"a":1}') + }) + + test('handles execution-denied output', () => { + const result: ToolOutput = { type: 'execution-denied', reason: 'denied' } + assert.strictEqual(flattenText(result), 'denied') + }) + + test('preserves exact text including whitespace', () => { + const result: ToolOutput = { type: 'text', value: ' leading spaces\ntrailing spaces \n\ttab\t' } + const output = flattenText(result) + assert.ok(output.includes(' leading spaces')) + assert.ok(output.includes('trailing spaces ')) + assert.ok(output.includes('\ttab\t')) + }) + + test('handles isError flag correctly', () => { + const errorResult = textResult('error message', true) + assert.strictEqual(errorResult.type, 'error-text') + + const successResult = textResult('success message', false) + assert.strictEqual(successResult.type, 'text') + }) + + test('handles many content items', () => { + const result: ToolOutput = { + type: 'text', + value: Array.from({ length: 100 }, (_, i) => `line${i}`).join('\n'), + } + const output = flattenText(result) + assert.ok(output.includes('line0')) + assert.ok(output.includes('line99')) + assert.strictEqual(output.split('\n').length, 100) + }) + }) +}) diff --git a/packages/core/src/tools/tools/mcp.ts b/packages/core/src/tools/tools/mcp.ts new file mode 100644 index 0000000..be0f304 --- /dev/null +++ b/packages/core/src/tools/tools/mcp.ts @@ -0,0 +1,17 @@ +import type { ToolResultOutput } from '@ai-sdk/provider-utils' + +/** Standard AI SDK tool output shape. */ +export type ToolOutput = ToolResultOutput + +/** Quick constructor for text-based tool output. */ +export function textResult(text: string, isError = false): ToolOutput { + return isError ? { type: 'error-text', value: text } : { type: 'text', value: text } +} + +/** Flatten tool output to string for observation display. */ +export function flattenText(result: ToolResultOutput): string { + if (result.type === 'text' || result.type === 'error-text') return result.value + if (result.type === 'json' || result.type === 'error-json') return JSON.stringify(result.value) + if (result.type === 'execution-denied') return result.reason ?? '' + return '(no tool output)' +} diff --git a/packages/tools/src/tools/mcp_resources.test.ts b/packages/core/src/tools/tools/mcp_resources.test.ts similarity index 82% rename from packages/tools/src/tools/mcp_resources.test.ts rename to packages/core/src/tools/tools/mcp_resources.test.ts index bf40f2e..643b5cf 100644 --- a/packages/tools/src/tools/mcp_resources.test.ts +++ b/packages/core/src/tools/tools/mcp_resources.test.ts @@ -1,19 +1,22 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { tmpdir } from 'node:os' import { join } from 'node:path' import { mkdtemp, readFile, rm } from 'node:fs/promises' import { afterEach, describe, test, vi } from 'vitest' -import { setActiveMcpCacheStore, setActiveMcpPool } from '@memo/tools/router/mcp/context' +import { setActiveMcpCacheStore, setActiveMcpPool } from '@memo/core/tools/router/mcp/context' import { __resetMcpResourceCacheForTests, listMcpResourceTemplatesTool, listMcpResourcesTool, readMcpResourceTool, -} from '@memo/tools/tools/mcp_resources' +} from '@memo/core/tools/tools/mcp_resources' -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' } afterEach(() => { @@ -23,11 +26,15 @@ afterEach(() => { vi.useRealTimers() }) +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('mcp resource tools', () => { test('returns error when MCP pool is missing', async () => { setActiveMcpPool(null) - const result = await listMcpResourcesTool.execute({}) - assert.strictEqual(result.isError, true) + const result = await runTool(listMcpResourcesTool, {}) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('not initialized')) }) @@ -54,10 +61,10 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const first = await listMcpResourcesTool.execute({ server: 'alpha', cursor: 'c1' }) - const second = await listMcpResourcesTool.execute({ server: 'alpha', cursor: 'c1' }) + const first = await runTool(listMcpResourcesTool, { server: 'alpha', cursor: 'c1' }) + const second = await runTool(listMcpResourcesTool, { server: 'alpha', cursor: 'c1' }) - assert.ok(!first.isError) + assert.ok(first.type === 'text') assert.deepStrictEqual(capturedCursor, { cursor: 'c1' }) assert.strictEqual(callCount, 1) @@ -90,13 +97,13 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const first = await listMcpResourcesTool.execute({ server: 'alpha' }) - const second = await listMcpResourcesTool.execute({ server: 'alpha' }) + const first = await runTool(listMcpResourcesTool, { server: 'alpha' }) + const second = await runTool(listMcpResourcesTool, { server: 'alpha' }) assert.strictEqual(callCount, 1) assert.strictEqual(textPayload(first), textPayload(second)) vi.advanceTimersByTime(15_001) - const third = await listMcpResourcesTool.execute({ server: 'alpha' }) + const third = await runTool(listMcpResourcesTool, { server: 'alpha' }) assert.strictEqual(callCount, 2) assert.notStrictEqual(textPayload(first), textPayload(third)) }) @@ -123,8 +130,8 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const first = listMcpResourcesTool.execute({ server: 'alpha' }) - const second = listMcpResourcesTool.execute({ server: 'alpha' }) + const first = runTool(listMcpResourcesTool, { server: 'alpha' }) + const second = runTool(listMcpResourcesTool, { server: 'alpha' }) for (let i = 0; i < 5 && callCount === 0; i += 1) { await Promise.resolve() } @@ -160,7 +167,7 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const result = await listMcpResourcesTool.execute({}) + const result = await runTool(listMcpResourcesTool, {}) const parsed = JSON.parse(textPayload(result)) assert.strictEqual(parsed.resources[0].server, 'alpha') @@ -191,8 +198,8 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const result = await listMcpResourcesTool.execute({}) - assert.ok(!result.isError) + const result = await runTool(listMcpResourcesTool, {}) + assert.ok(result.type === 'text') const parsed = JSON.parse(textPayload(result)) assert.strictEqual(parsed.resources.length, 1) @@ -208,11 +215,11 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const resourcesResult = await listMcpResourcesTool.execute({ cursor: 'x' }) - assert.strictEqual(resourcesResult.isError, true) + const resourcesResult = await runTool(listMcpResourcesTool, { cursor: 'x' }) + assert.strictEqual(resourcesResult.type, 'error-text') - const templatesResult = await listMcpResourceTemplatesTool.execute({ cursor: 'x' }) - assert.strictEqual(templatesResult.isError, true) + const templatesResult = await runTool(listMcpResourceTemplatesTool, { cursor: 'x' }) + assert.strictEqual(templatesResult.type, 'error-text') }) test('caches list resource templates for same key', async () => { @@ -234,8 +241,8 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const first = await listMcpResourceTemplatesTool.execute({ server: 'alpha' }) - const second = await listMcpResourceTemplatesTool.execute({ server: 'alpha' }) + const first = await runTool(listMcpResourceTemplatesTool, { server: 'alpha' }) + const second = await runTool(listMcpResourceTemplatesTool, { server: 'alpha' }) assert.strictEqual(callCount, 1) assert.strictEqual(textPayload(first), textPayload(second)) @@ -262,9 +269,9 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const first = await readMcpResourceTool.execute({ server: 'alpha', uri: 'memo://a' }) - const second = await readMcpResourceTool.execute({ server: 'alpha', uri: 'memo://a' }) - assert.ok(!first.isError) + const first = await runTool(readMcpResourceTool, { server: 'alpha', uri: 'memo://a' }) + const second = await runTool(readMcpResourceTool, { server: 'alpha', uri: 'memo://a' }) + assert.ok(first.type === 'text') assert.strictEqual(callCount, 1) assert.deepStrictEqual(capturedUri, { uri: 'memo://a' }) const parsed = JSON.parse(textPayload(first)) @@ -280,8 +287,8 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - const result = await readMcpResourceTool.execute({ server: 'none', uri: 'memo://x' }) - assert.strictEqual(result.isError, true) + const result = await runTool(readMcpResourceTool, { server: 'none', uri: 'memo://x' }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('MCP server not found')) }) @@ -316,7 +323,7 @@ describe('mcp resource tools', () => { } setActiveMcpPool(pool as any) - await listMcpResourcesTool.execute({ server: 'alpha' }) + await runTool(listMcpResourcesTool, { server: 'alpha' }) assert.strictEqual(callCount, 1) await new Promise((resolve) => setTimeout(resolve, 220)) @@ -329,7 +336,7 @@ describe('mcp resource tools', () => { assert.ok(Object.keys(parsed.responses ?? {}).some((k) => k.startsWith('list_resources:'))) __resetMcpResourceCacheForTests() - await listMcpResourcesTool.execute({ server: 'alpha' }) + await runTool(listMcpResourcesTool, { server: 'alpha' }) assert.strictEqual(callCount, 1) } finally { if (originalMemoHome === undefined) { diff --git a/packages/tools/src/tools/mcp_resources.ts b/packages/core/src/tools/tools/mcp_resources.ts similarity index 93% rename from packages/tools/src/tools/mcp_resources.ts rename to packages/core/src/tools/tools/mcp_resources.ts index f0f5f71..fd063be 100644 --- a/packages/tools/src/tools/mcp_resources.ts +++ b/packages/core/src/tools/tools/mcp_resources.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { getGlobalMcpCacheStore, resetGlobalMcpCacheStoreForTests } from '@memo/tools/router/mcp/cache_store' -import { getActiveMcpCacheStore, getActiveMcpPool } from '@memo/tools/router/mcp/context' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { getGlobalMcpCacheStore, resetGlobalMcpCacheStoreForTests } from '@memo/core/tools/router/mcp/cache_store' +import { getActiveMcpCacheStore, getActiveMcpPool } from '@memo/core/tools/router/mcp/context' const LIST_MCP_RESOURCES_INPUT_SCHEMA = z .object({ @@ -25,10 +25,6 @@ const READ_MCP_RESOURCE_INPUT_SCHEMA = z }) .strict() -type ListResourcesInput = z.infer -type ListResourceTemplatesInput = z.infer -type ReadResourceInput = z.infer - type PoolLike = { get?: (name: string) => any getAll?: () => any[] @@ -126,12 +122,11 @@ export function __resetMcpResourceCacheForTests() { resetGlobalMcpCacheStoreForTests() } -export const listMcpResourcesTool = defineMcpTool({ - name: 'list_mcp_resources', +export const listMcpResourcesTool = tool({ description: 'Lists resources provided by MCP servers. Prefer resources over web search when possible.', inputSchema: LIST_MCP_RESOURCES_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async ({ server, cursor }) => { try { const pool = getPoolOrThrow() @@ -218,13 +213,12 @@ export const listMcpResourcesTool = defineMcpTool({ }, }) -export const listMcpResourceTemplatesTool = defineMcpTool({ - name: 'list_mcp_resource_templates', +export const listMcpResourceTemplatesTool = tool({ description: 'Lists resource templates provided by MCP servers. Prefer resource templates over web search when possible.', inputSchema: LIST_MCP_RESOURCE_TEMPLATES_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async ({ server, cursor }) => { try { const pool = getPoolOrThrow() @@ -311,12 +305,11 @@ export const listMcpResourceTemplatesTool = defineMcpTool({ - name: 'read_mcp_resource', +export const readMcpResourceTool = tool({ description: 'Read a specific resource from an MCP server given the server name and resource URI.', inputSchema: READ_MCP_RESOURCE_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async ({ server, uri }) => { try { const pool = getPoolOrThrow() diff --git a/packages/tools/src/tools/read_files.ts b/packages/core/src/tools/tools/read_files.ts similarity index 71% rename from packages/tools/src/tools/read_files.ts rename to packages/core/src/tools/tools/read_files.ts index d11ae44..dc3890b 100644 --- a/packages/tools/src/tools/read_files.ts +++ b/packages/core/src/tools/tools/read_files.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { readFileContent, validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { readFileContent, validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const READ_FILES_INPUT_SCHEMA = z .object({ @@ -10,15 +10,12 @@ const READ_FILES_INPUT_SCHEMA = z }) .strict() -type ReadFilesInput = z.infer - -export const readFilesTool = defineMcpTool({ - name: 'read_files', +export const readFilesTool = tool({ description: 'Read multiple text files in one call. Per-file failures are returned inline without aborting the batch.', inputSchema: READ_FILES_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/tools/src/tools/read_media_file.ts b/packages/core/src/tools/tools/read_media_file.ts similarity index 74% rename from packages/tools/src/tools/read_media_file.ts rename to packages/core/src/tools/tools/read_media_file.ts index 48cbbe9..bedf88d 100644 --- a/packages/tools/src/tools/read_media_file.ts +++ b/packages/core/src/tools/tools/read_media_file.ts @@ -1,10 +1,10 @@ import { readFile } from 'node:fs/promises' import path from 'node:path' import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const READ_MEDIA_FILE_INPUT_SCHEMA = z .object({ @@ -12,8 +12,6 @@ const READ_MEDIA_FILE_INPUT_SCHEMA = z }) .strict() -type ReadMediaFileInput = z.infer - const MIME_TYPES: Record = { '.png': 'image/png', '.jpg': 'image/jpeg', @@ -28,12 +26,11 @@ const MIME_TYPES: Record = { '.flac': 'audio/flac', } -export const readMediaFileTool = defineMcpTool({ - name: 'read_media_file', +export const readMediaFileTool = tool({ description: 'Read an image or audio file and return base64 payload metadata as JSON text.', inputSchema: READ_MEDIA_FILE_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/core/src/tools/tools/read_skill.test.ts b/packages/core/src/tools/tools/read_skill.test.ts new file mode 100644 index 0000000..af22d6a --- /dev/null +++ b/packages/core/src/tools/tools/read_skill.test.ts @@ -0,0 +1,157 @@ +import assert from 'node:assert' +import { mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, test } from 'vitest' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import { buildSkillIndex, loadSkills } from '@memo/core/skills/skills' +import type { SkillIndex } from '@memo/core/skills/skills' +import { readSkillTool } from '@memo/core/tools/tools/read_skill' + +async function makeTempDir(prefix: string) { + const dir = join(tmpdir(), `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2)}`) + await mkdir(dir, { recursive: true }) + return dir +} + +async function removeDir(path: string) { + await rm(path, { recursive: true, force: true }) +} + +async function writeSkill(skillRoot: string, skillName: string, description: string, body?: string) { + const skillDir = join(skillRoot, skillName) + const skillPath = join(skillDir, 'SKILL.md') + await mkdir(skillDir, { recursive: true }) + await writeFile( + skillPath, + `--- +name: ${skillName} +description: ${description} +--- +${body ?? `# ${skillName}\n`}`, + 'utf-8', + ) + return skillPath +} + +async function buildIndexFor({ projectRoot, homeDir }: { projectRoot: string; homeDir: string }): Promise { + const memoHome = join(homeDir, '.memo') + await mkdir(homeDir, { recursive: true }) + await mkdir(memoHome, { recursive: true }) + await writeFile(join(projectRoot, '.git'), 'gitdir: test\n', 'utf-8') + const discovered = await loadSkills({ cwd: projectRoot, homeDir, memoHome }) + return buildSkillIndex(discovered) +} + +function callTool(input: unknown, index: SkillIndex): Promise { + return readSkillTool.execute!( + input as never, + { + experimental_context: { skillIndex: index }, + } as never, + ) as Promise +} + +describe('read_skill tool', () => { + test('reads a skill body with frontmatter stripped and reports skill_directory', async () => { + const sandbox = await makeTempDir('memo-read-skill') + const projectRoot = join(sandbox, 'repo') + await mkdir(projectRoot, { recursive: true }) + await writeSkill( + join(projectRoot, '.agents', 'skills'), + 'fmt', + 'formatting helper', + '# Fmt\n\nRun `scripts/format.sh` relative to this skill directory.\n', + ) + try { + const index = await buildIndexFor({ projectRoot, homeDir: join(sandbox, 'home') }) + const result = await callTool({ name: 'fmt' }, index) + assert.strictEqual(result.type, 'text') + const payload = JSON.parse(result.value) + assert.strictEqual(payload.name, 'fmt') + assert.ok(payload.skill_directory.endsWith(join('.agents', 'skills', 'fmt'))) + assert.ok(payload.content.includes('scripts/format.sh')) + assert.ok(!payload.content.includes('description: formatting helper')) + } finally { + await removeDir(sandbox) + } + }) + + test('returns not-found with available names', async () => { + const sandbox = await makeTempDir('memo-read-skill-missing') + const projectRoot = join(sandbox, 'repo') + await mkdir(projectRoot, { recursive: true }) + await writeSkill(join(projectRoot, '.agents', 'skills'), 'only-one', 'only skill') + try { + const index = await buildIndexFor({ projectRoot, homeDir: join(sandbox, 'home') }) + const result = await callTool({ name: 'nope' }, index) + assert.strictEqual(result.type, 'error-text') + assert.ok(result.value.includes('nope')) + assert.ok(result.value.includes('only-one')) + } finally { + await removeDir(sandbox) + } + }) + + test('reports ambiguity with candidate paths when a name matches multiple skills', async () => { + const sandbox = await makeTempDir('memo-read-skill-ambiguous') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + await mkdir(projectRoot, { recursive: true }) + await writeSkill(join(projectRoot, '.agents', 'skills'), 'clash', 'project version') + await writeSkill(join(homeDir, '.claude', 'skills'), 'clash', 'global version') + try { + const index = await buildIndexFor({ projectRoot, homeDir }) + const result = await callTool({ name: 'clash' }, index) + assert.strictEqual(result.type, 'error-text') + assert.ok(result.value.includes('ambiguous')) + assert.ok(result.value.includes('.agents/skills/clash/SKILL.md')) + assert.ok(result.value.includes('.claude/skills/clash/SKILL.md')) + } finally { + await removeDir(sandbox) + } + }) + + test('resolves a path even when it points at a deduped-away copy', async () => { + const sandbox = await makeTempDir('memo-read-skill-path') + const projectRoot = join(sandbox, 'repo') + const homeDir = join(sandbox, 'home') + await mkdir(projectRoot, { recursive: true }) + const projectPath = await writeSkill(join(projectRoot, '.agents', 'skills'), 'shared', 'same skill') + const homePath = await writeSkill(join(homeDir, '.claude', 'skills'), 'shared', 'same skill') + try { + const index = await buildIndexFor({ projectRoot, homeDir }) + const result = await callTool({ path: homePath }, index) + assert.strictEqual(result.type, 'text') + const payload = JSON.parse(result.value) + assert.strictEqual(payload.name, 'shared') + assert.ok(payload.skill_directory.endsWith(join('.agents', 'skills', 'shared')), 'winner directory') + void projectPath + } finally { + await removeDir(sandbox) + } + }) + + test('falls back to a fresh scan when context has no skillIndex', async () => { + const sandbox = await makeTempDir('memo-read-skill-fallback') + const homeDir = join(sandbox, 'home') + const memoHome = join(homeDir, '.memo') + await mkdir(memoHome, { recursive: true }) + await writeSkill(join(memoHome, 'skills'), 'fallback', 'fallback skill') + process.env.MEMO_HOME = memoHome + try { + const result = (await readSkillTool.execute!( + { name: 'fallback' } as never, + { + experimental_context: {}, + } as never, + )) as ToolResultOutput + assert.strictEqual(result.type, 'text') + const payload = JSON.parse(result.value) + assert.strictEqual(payload.name, 'fallback') + } finally { + delete process.env.MEMO_HOME + await removeDir(sandbox) + } + }) +}) diff --git a/packages/core/src/tools/tools/read_skill.ts b/packages/core/src/tools/tools/read_skill.ts new file mode 100644 index 0000000..2912137 --- /dev/null +++ b/packages/core/src/tools/tools/read_skill.ts @@ -0,0 +1,67 @@ +import { dirname } from 'node:path' +import { z } from 'zod' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { buildSkillIndex, findSkillByName, findSkillByPath, loadSkills, readSkillBody } from '@memo/core/skills/skills' +import type { SkillIndex, SkillMetadata } from '@memo/core/skills/skills' + +const READ_SKILL_INPUT_SCHEMA = z + .object({ + name: z.string().min(1).optional(), + path: z.string().min(1).optional(), + }) + .strict() + .refine((input) => Boolean(input.name) !== Boolean(input.path), { + message: 'Provide exactly one of name or path', + }) + +export const readSkillTool = tool({ + description: + 'Loads the full SKILL.md of a skill listed in the Skills directory (frontmatter stripped). Pass the skill name from the directory, or the exact SKILL.md path when names are ambiguous. Resolve relative paths (scripts/, references/) against the returned skill_directory. Very long skills may be truncated.', + inputSchema: READ_SKILL_INPUT_SCHEMA, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + + execute: async ({ name, path }, options) => { + const ctx = options.experimental_context as { skillIndex?: SkillIndex } | undefined + const index = ctx?.skillIndex ?? buildSkillIndex(await loadSkills()) + + let record: SkillMetadata | undefined + if (path) { + record = findSkillByPath(index, path) + if (!record) { + return textResult(`skill not found for path=${path}. Use a name from the Skills directory.`, true) + } + } else if (name) { + const matches = findSkillByName(index, name) + if (matches.length === 0) { + const available = index.list.map((skill) => skill.name).join(', ') + return textResult(`skill not found: ${name}. Available skills: ${available || '(none)'}`, true) + } + if (matches.length > 1) { + const candidates = matches.map((skill) => skill.paths.join(' | ')).join(', ') + return textResult( + `skill name "${name}" is ambiguous, matches: ${candidates}. Pass the exact SKILL.md path instead.`, + true, + ) + } + record = matches[0] + } + + try { + const body = await readSkillBody(record!) + return textResult( + JSON.stringify( + { + name: record!.name, + skill_directory: dirname(record!.path), + content: body, + }, + null, + 2, + ), + ) + } catch (err) { + return textResult(`failed to read skill: ${(err as Error).message}`, true) + } + }, +}) diff --git a/packages/tools/src/tools/read_text_file.ts b/packages/core/src/tools/tools/read_text_file.ts similarity index 74% rename from packages/tools/src/tools/read_text_file.ts rename to packages/core/src/tools/tools/read_text_file.ts index ed1ddf0..3f5de04 100644 --- a/packages/tools/src/tools/read_text_file.ts +++ b/packages/core/src/tools/tools/read_text_file.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { headFile, readFileContent, tailFile, validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { headFile, readFileContent, tailFile, validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const READ_TEXT_FILE_INPUT_SCHEMA = z .object({ @@ -12,14 +12,11 @@ const READ_TEXT_FILE_INPUT_SCHEMA = z }) .strict() -type ReadTextFileInput = z.infer - -export const readTextFileTool = defineMcpTool({ - name: 'read_text_file', +export const readTextFileTool = tool({ description: 'Read the complete file content as text, optionally with head/tail line limits.', inputSchema: READ_TEXT_FILE_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { if (input.head && input.tail) { return textResult('Cannot specify both head and tail parameters simultaneously', true) diff --git a/packages/tools/src/tools/search_files.ts b/packages/core/src/tools/tools/search_files.ts similarity index 66% rename from packages/tools/src/tools/search_files.ts rename to packages/core/src/tools/tools/search_files.ts index 15ab590..5de14af 100644 --- a/packages/tools/src/tools/search_files.ts +++ b/packages/core/src/tools/tools/search_files.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { searchFilesWithValidation, validatePath } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { searchFilesWithValidation, validatePath } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const SEARCH_FILES_INPUT_SCHEMA = z .object({ @@ -12,14 +12,11 @@ const SEARCH_FILES_INPUT_SCHEMA = z }) .strict() -type SearchFilesInput = z.infer - -export const searchFilesTool = defineMcpTool({ - name: 'search_files', +export const searchFilesTool = tool({ description: 'Recursively search files and directories by glob pattern within allowed directories.', inputSchema: SEARCH_FILES_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/tools/src/tools/shell.test.ts b/packages/core/src/tools/tools/shell.test.ts similarity index 77% rename from packages/tools/src/tools/shell.test.ts rename to packages/core/src/tools/tools/shell.test.ts index eedd41f..608f2a0 100644 --- a/packages/tools/src/tools/shell.test.ts +++ b/packages/core/src/tools/tools/shell.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' import { describe, test, vi, beforeEach, afterEach, expect } from 'vitest' import { shellTool } from './shell' import { flattenText } from './mcp' @@ -13,6 +15,10 @@ vi.mock('./exec_runtime', async () => { import { startExecSession } from './exec_runtime' +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('shell tool', () => { beforeEach(() => { vi.resetAllMocks() @@ -26,9 +32,9 @@ describe('shell tool', () => { test('joins argv and executes command', async () => { vi.mocked(startExecSession).mockResolvedValue('test output') - const result = await shellTool.execute({ command: ['echo', 'hello'] }) + const result = await runTool(shellTool, { command: ['echo', 'hello'] }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), 'test output') expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -41,7 +47,7 @@ describe('shell tool', () => { test('handles single argument', async () => { vi.mocked(startExecSession).mockResolvedValue('single') - await shellTool.execute({ command: ['echo', 'single'] }) + await runTool(shellTool, { command: ['echo', 'single'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -53,7 +59,7 @@ describe('shell tool', () => { test('handles many arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('many') - await shellTool.execute({ command: ['echo', 'a', 'b', 'c', 'd', 'e'] }) + await runTool(shellTool, { command: ['echo', 'a', 'b', 'c', 'd', 'e'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -67,7 +73,7 @@ describe('shell tool', () => { test('quotes arguments with spaces', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'hello world'] }) + await runTool(shellTool, { command: ['echo', 'hello world'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -79,7 +85,7 @@ describe('shell tool', () => { test('quotes arguments with single quotes', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', "it's working"] }) + await runTool(shellTool, { command: ['echo', "it's working"] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -91,7 +97,7 @@ describe('shell tool', () => { test('handles empty arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', ''] }) + await runTool(shellTool, { command: ['echo', ''] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -103,7 +109,7 @@ describe('shell tool', () => { test('quotes arguments with dollar signs', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '$HOME'] }) + await runTool(shellTool, { command: ['echo', '$HOME'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -115,7 +121,7 @@ describe('shell tool', () => { test('quotes arguments with backticks', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '`date`'] }) + await runTool(shellTool, { command: ['echo', '`date`'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -127,7 +133,7 @@ describe('shell tool', () => { test('quotes arguments with newlines', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'line1\nline2'] }) + await runTool(shellTool, { command: ['echo', 'line1\nline2'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -139,7 +145,7 @@ describe('shell tool', () => { test('quotes arguments with backslashes', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'path\\to\\file'] }) + await runTool(shellTool, { command: ['echo', 'path\\to\\file'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -151,7 +157,7 @@ describe('shell tool', () => { test('quotes arguments with semicolons', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'a;b;c'] }) + await runTool(shellTool, { command: ['echo', 'a;b;c'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -163,7 +169,7 @@ describe('shell tool', () => { test('quotes arguments with pipes', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'a|b'] }) + await runTool(shellTool, { command: ['echo', 'a|b'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -175,7 +181,7 @@ describe('shell tool', () => { test('quotes arguments with wildcards', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '*.txt'] }) + await runTool(shellTool, { command: ['echo', '*.txt'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -187,7 +193,7 @@ describe('shell tool', () => { test('quotes arguments with angle brackets', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', ''] }) + await runTool(shellTool, { command: ['echo', ''] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -199,7 +205,7 @@ describe('shell tool', () => { test('quotes arguments with ampersands', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'A&B'] }) + await runTool(shellTool, { command: ['echo', 'A&B'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -211,7 +217,7 @@ describe('shell tool', () => { test('does not quote safe arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ + await runTool(shellTool, { command: ['echo', 'hello_world', 'file.txt', '/usr/local/bin'], }) @@ -225,7 +231,7 @@ describe('shell tool', () => { test('handles mixed safe and unsafe arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', 'safe', 'hello world', 'unsafe'] }) + await runTool(shellTool, { command: ['echo', 'safe', 'hello world', 'unsafe'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -239,7 +245,7 @@ describe('shell tool', () => { test('passes optional workdir parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['pwd'], workdir: '/tmp' }) + await runTool(shellTool, { command: ['pwd'], workdir: '/tmp' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -251,7 +257,7 @@ describe('shell tool', () => { test('passes optional timeout_ms parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['sleep', '1'], timeout_ms: 5000 }) + await runTool(shellTool, { command: ['sleep', '1'], timeout_ms: 5000 }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -266,36 +272,36 @@ describe('shell tool', () => { test('handles execution errors gracefully', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command failed')) - const result = await shellTool.execute({ command: ['invalid-command'] }) + const result = await runTool(shellTool, { command: ['invalid-command'] }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell failed')) }) test('handles command not found errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('ENOENT')) - const result = await shellTool.execute({ command: ['definitely-not-real'] }) + const result = await runTool(shellTool, { command: ['definitely-not-real'] }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell failed')) }) test('handles permission denied errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('EACCES: permission denied')) - const result = await shellTool.execute({ command: ['/protected/path'] }) + const result = await runTool(shellTool, { command: ['/protected/path'] }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell failed')) }) test('handles timeout errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command timed out')) - const result = await shellTool.execute({ command: ['sleep', '100'], timeout_ms: 100 }) + const result = await runTool(shellTool, { command: ['sleep', '100'], timeout_ms: 100 }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell failed')) }) }) @@ -304,7 +310,7 @@ describe('shell tool', () => { test('handles unicode arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '你好世界'] }) + await runTool(shellTool, { command: ['echo', '你好世界'] }) expect(startExecSession).toHaveBeenCalled() }) @@ -312,7 +318,7 @@ describe('shell tool', () => { test('handles emoji in arguments', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '🌍🚀'] }) + await runTool(shellTool, { command: ['echo', '🌍🚀'] }) expect(startExecSession).toHaveBeenCalled() }) @@ -320,7 +326,7 @@ describe('shell tool', () => { test('handles unicode with spaces', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellTool.execute({ command: ['echo', '你好 世界'] }) + await runTool(shellTool, { command: ['echo', '你好 世界'] }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/tools/src/tools/shell.ts b/packages/core/src/tools/tools/shell.ts similarity index 79% rename from packages/tools/src/tools/shell.ts rename to packages/core/src/tools/tools/shell.ts index c3c0c17..4bcded6 100644 --- a/packages/tools/src/tools/shell.ts +++ b/packages/core/src/tools/tools/shell.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { startExecSession } from '@memo/tools/tools/exec_runtime' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { startExecSession } from '@memo/core/tools/tools/exec_runtime' const SHELL_INPUT_SCHEMA = z .object({ @@ -14,8 +14,6 @@ const SHELL_INPUT_SCHEMA = z }) .strict() -type ShellInput = z.infer - const SAFE_SHELL_ARG = /^[A-Za-z0-9_./:@%+-]+$/ function shellQuote(part: string) { @@ -28,12 +26,11 @@ function shellJoin(argv: string[]) { return argv.map((part) => shellQuote(part)).join(' ') } -export const shellTool = defineMcpTool({ - name: 'shell', +export const shellTool = tool({ description: 'Runs a shell command (argv form) and returns output.', inputSchema: SHELL_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: true } }, + execute: async ({ command, workdir, timeout_ms }) => { try { const content = await startExecSession({ diff --git a/packages/tools/src/tools/shell_command.test.ts b/packages/core/src/tools/tools/shell_command.test.ts similarity index 74% rename from packages/tools/src/tools/shell_command.test.ts rename to packages/core/src/tools/tools/shell_command.test.ts index 3fe0ca7..0c6b810 100644 --- a/packages/tools/src/tools/shell_command.test.ts +++ b/packages/core/src/tools/tools/shell_command.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' import { describe, test, vi, beforeEach, afterEach, expect } from 'vitest' import { shellCommandTool } from './shell_command' import { flattenText } from './mcp' @@ -13,6 +15,10 @@ vi.mock('./exec_runtime', async () => { import { startExecSession } from './exec_runtime' +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('shell_command tool', () => { beforeEach(() => { vi.resetAllMocks() @@ -26,9 +32,9 @@ describe('shell_command tool', () => { test('executes command and returns output', async () => { vi.mocked(startExecSession).mockResolvedValue('test output') - const result = await shellCommandTool.execute({ command: 'echo hello' }) + const result = await runTool(shellCommandTool, { command: 'echo hello' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), 'test output') expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -42,20 +48,20 @@ describe('shell_command tool', () => { const multiLineOutput = 'line1\nline2\nline3' vi.mocked(startExecSession).mockResolvedValue(multiLineOutput) - const result = await shellCommandTool.execute({ + const result = await runTool(shellCommandTool, { command: 'printf "line1\nline2\nline3"', }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), multiLineOutput) }) test('handles empty output', async () => { vi.mocked(startExecSession).mockResolvedValue('') - const result = await shellCommandTool.execute({ command: 'true' }) + const result = await runTool(shellCommandTool, { command: 'true' }) - assert.strictEqual(result.isError, false) + assert.strictEqual(result.type, 'text') assert.strictEqual(flattenText(result), '') }) }) @@ -64,7 +70,7 @@ describe('shell_command tool', () => { test('passes optional workdir parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellCommandTool.execute({ command: 'pwd', workdir: '/tmp' }) + await runTool(shellCommandTool, { command: 'pwd', workdir: '/tmp' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -76,7 +82,7 @@ describe('shell_command tool', () => { test('passes optional login parameter', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellCommandTool.execute({ command: 'whoami', login: true }) + await runTool(shellCommandTool, { command: 'whoami', login: true }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -88,7 +94,7 @@ describe('shell_command tool', () => { test('passes login=false explicitly', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellCommandTool.execute({ command: 'echo test', login: false }) + await runTool(shellCommandTool, { command: 'echo test', login: false }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -100,7 +106,7 @@ describe('shell_command tool', () => { test('passes timeout_ms as yield_time_ms and execution_timeout_ms', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellCommandTool.execute({ command: 'sleep 1', timeout_ms: 5000 }) + await runTool(shellCommandTool, { command: 'sleep 1', timeout_ms: 5000 }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -113,7 +119,7 @@ describe('shell_command tool', () => { test('handles zero timeout_ms', async () => { vi.mocked(startExecSession).mockResolvedValue('output') - await shellCommandTool.execute({ command: 'echo test', timeout_ms: 0 }) + await runTool(shellCommandTool, { command: 'echo test', timeout_ms: 0 }) expect(startExecSession).toHaveBeenCalled() }) @@ -123,27 +129,27 @@ describe('shell_command tool', () => { test('handles execution errors gracefully', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command failed')) - const result = await shellCommandTool.execute({ command: 'invalid-command' }) + const result = await runTool(shellCommandTool, { command: 'invalid-command' }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell_command failed')) }) test('includes original error message', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('ENOENT: no such file')) - const result = await shellCommandTool.execute({ command: 'nonexistent-cmd' }) + const result = await runTool(shellCommandTool, { command: 'nonexistent-cmd' }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('ENOENT')) }) test('handles timeout errors', async () => { vi.mocked(startExecSession).mockRejectedValue(new Error('command timed out after 5000ms')) - const result = await shellCommandTool.execute({ command: 'sleep 10', timeout_ms: 1000 }) + const result = await runTool(shellCommandTool, { command: 'sleep 10', timeout_ms: 1000 }) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(flattenText(result).includes('shell_command failed')) }) }) @@ -152,7 +158,7 @@ describe('shell_command tool', () => { test('handles commands with pipes', async () => { vi.mocked(startExecSession).mockResolvedValue('filtered output') - await shellCommandTool.execute({ command: 'cat file.txt | grep pattern' }) + await runTool(shellCommandTool, { command: 'cat file.txt | grep pattern' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -164,7 +170,7 @@ describe('shell_command tool', () => { test('handles commands with redirects', async () => { vi.mocked(startExecSession).mockResolvedValue('') - await shellCommandTool.execute({ command: 'echo hello > /tmp/output.txt' }) + await runTool(shellCommandTool, { command: 'echo hello > /tmp/output.txt' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -176,7 +182,7 @@ describe('shell_command tool', () => { test('handles commands with environment variables', async () => { vi.mocked(startExecSession).mockResolvedValue('test-value') - await shellCommandTool.execute({ command: 'echo $MY_VAR' }) + await runTool(shellCommandTool, { command: 'echo $MY_VAR' }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ @@ -189,7 +195,7 @@ describe('shell_command tool', () => { const longCmd = Array(100).fill('echo test &&').join(' ') + ' echo done' vi.mocked(startExecSession).mockResolvedValue('done') - await shellCommandTool.execute({ command: longCmd }) + await runTool(shellCommandTool, { command: longCmd }) expect(startExecSession).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/tools/src/tools/shell_command.ts b/packages/core/src/tools/tools/shell_command.ts similarity index 74% rename from packages/tools/src/tools/shell_command.ts rename to packages/core/src/tools/tools/shell_command.ts index ce17460..51eff19 100644 --- a/packages/tools/src/tools/shell_command.ts +++ b/packages/core/src/tools/tools/shell_command.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { startExecSession } from '@memo/tools/tools/exec_runtime' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { startExecSession } from '@memo/core/tools/tools/exec_runtime' const SHELL_COMMAND_INPUT_SCHEMA = z .object({ @@ -15,14 +15,11 @@ const SHELL_COMMAND_INPUT_SCHEMA = z }) .strict() -type ShellCommandInput = z.infer - -export const shellCommandTool = defineMcpTool({ - name: 'shell_command', +export const shellCommandTool = tool({ description: 'Runs a shell command and returns its output. Always set workdir when possible.', inputSchema: SHELL_COMMAND_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: true } }, + execute: async ({ command, workdir, login, timeout_ms }) => { try { const content = await startExecSession({ diff --git a/packages/tools/src/tools/shell_update_plan.test.ts b/packages/core/src/tools/tools/shell_update_plan.test.ts similarity index 71% rename from packages/tools/src/tools/shell_update_plan.test.ts rename to packages/core/src/tools/tools/shell_update_plan.test.ts index 5028104..9e9a347 100644 --- a/packages/tools/src/tools/shell_update_plan.test.ts +++ b/packages/core/src/tools/tools/shell_update_plan.test.ts @@ -1,41 +1,48 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { describe, test } from 'vitest' -import { shellTool } from '@memo/tools/tools/shell' -import { shellCommandTool } from '@memo/tools/tools/shell_command' -import { writeStdinTool } from '@memo/tools/tools/write_stdin' -import { updatePlanTool } from '@memo/tools/tools/update_plan' - -function textPayload(result: { content?: Array<{ type: string; text?: string }> }) { - const first = result.content?.find((item) => item.type === 'text') - return first?.text ?? '' +import { shellTool } from '@memo/core/tools/tools/shell' +import { shellCommandTool } from '@memo/core/tools/tools/shell_command' +import { writeStdinTool } from '@memo/core/tools/tools/write_stdin' +import { updatePlanTool } from '@memo/core/tools/tools/update_plan' + +function textPayload(result: ToolOutput) { + if (result.type === 'text' || result.type === 'error-text') return result.value ?? '' + return '' +} + +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput } describe('shell wrappers and update_plan', () => { test('shell tool executes argv command form', async () => { - const result = await shellTool.execute({ + const result = await runTool(shellTool, { command: ['echo', 'shell-wrapper-ok'], }) const text = textPayload(result) - assert.ok(!result.isError) + assert.ok(result.type === 'text') assert.ok(text.includes('shell-wrapper-ok')) }) test('shell tool quotes dangerous shell metacharacters in argv', async () => { const literal = '$HOME $(echo hacked);`date`' - const result = await shellTool.execute({ + const result = await runTool(shellTool, { command: ['printf', '%s', literal], }) const text = textPayload(result) - assert.ok(!result.isError) + assert.ok(result.type === 'text') assert.ok(text.includes(literal)) assert.ok(text.includes('$(echo hacked)')) assert.ok(text.includes('`date`')) }) test('shell tool blocks dangerous argv command', async () => { - const result = await shellTool.execute({ + const result = await runTool(shellTool, { command: ['mkfs.ext4', '/dev/sda'], }) @@ -47,32 +54,32 @@ describe('shell wrappers and update_plan', () => { test('shell tool enforces timeout_ms as execution deadline', async () => { const startedAt = Date.now() - const result = await shellTool.execute({ + const result = await runTool(shellTool, { command: ['sleep', '2'], timeout_ms: 100, }) const elapsedMs = Date.now() - startedAt const text = textPayload(result) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(text.includes('timed out')) assert.ok(elapsedMs < 1_500) }) test('shell_command executes script command form', async () => { - const result = await shellCommandTool.execute({ + const result = await runTool(shellCommandTool, { command: 'echo shell-command-ok', login: false, timeout_ms: 1000, }) const text = textPayload(result) - assert.ok(!result.isError) + assert.ok(result.type === 'text') assert.ok(text.includes('shell-command-ok')) }) test('shell_command blocks dangerous script command', async () => { - const result = await shellCommandTool.execute({ + const result = await runTool(shellCommandTool, { command: 'dd if=/dev/zero of=/dev/sda bs=1M', login: false, timeout_ms: 1000, @@ -86,7 +93,7 @@ describe('shell wrappers and update_plan', () => { test('shell_command enforces timeout_ms as execution deadline', async () => { const startedAt = Date.now() - const result = await shellCommandTool.execute({ + const result = await runTool(shellCommandTool, { command: 'sleep 2; echo too-late', login: false, timeout_ms: 100, @@ -94,19 +101,19 @@ describe('shell wrappers and update_plan', () => { const elapsedMs = Date.now() - startedAt const text = textPayload(result) - assert.strictEqual(result.isError, true) + assert.strictEqual(result.type, 'error-text') assert.ok(text.includes('timed out')) assert.ok(elapsedMs < 1_500) }) test('write_stdin fails for unknown session id', async () => { - const result = await writeStdinTool.execute({ session_id: 999999, chars: 'noop' }) - assert.strictEqual(result.isError, true) + const result = await runTool(writeStdinTool, { session_id: 999999, chars: 'noop' }) + assert.strictEqual(result.type, 'error-text') assert.ok(textPayload(result).includes('session_id 999999 not found')) }) test('update_plan rejects 1-step task', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { explanation: 'simple task', plan: [{ step: 'read_file package.json', status: 'pending' }], }) @@ -119,7 +126,7 @@ describe('shell wrappers and update_plan', () => { }) test('update_plan rejects 2-step task', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { explanation: '2-step task', plan: [ { step: 'step one', status: 'pending' }, @@ -132,7 +139,7 @@ describe('shell wrappers and update_plan', () => { }) test('update_plan rejects 3-step task', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { explanation: '3-step task', plan: [ { step: 'step one', status: 'completed' }, @@ -146,7 +153,7 @@ describe('shell wrappers and update_plan', () => { }) test('update_plan accepts 4-step task', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { explanation: 'complex task', plan: [ { step: 'step one', status: 'pending' }, @@ -156,13 +163,13 @@ describe('shell wrappers and update_plan', () => { ], }) - assert.ok(!result.isError) + assert.ok(result.type === 'text') const parsed = JSON.parse(textPayload(result)) assert.strictEqual(parsed.message, 'Plan updated') }) test('update_plan rejects too many in_progress', async () => { - const result = await updatePlanTool.execute({ + const result = await runTool(updatePlanTool, { explanation: 'invalid plan', plan: [ { step: 'step one', status: 'in_progress' }, @@ -170,7 +177,7 @@ describe('shell wrappers and update_plan', () => { ], }) - assert.ok(result.isError) + assert.ok(result.type === 'error-text') assert.ok(textPayload(result).includes('At most one step can be in_progress')) }) }) diff --git a/packages/tools/src/tools/update_plan.ts b/packages/core/src/tools/tools/update_plan.ts similarity index 81% rename from packages/tools/src/tools/update_plan.ts rename to packages/core/src/tools/tools/update_plan.ts index 885f79c..d5f518b 100644 --- a/packages/tools/src/tools/update_plan.ts +++ b/packages/core/src/tools/tools/update_plan.ts @@ -1,6 +1,6 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' const PLAN_ITEM_SCHEMA = z .object({ @@ -16,16 +16,13 @@ const UPDATE_PLAN_INPUT_SCHEMA = z }) .strict() -type UpdatePlanInput = z.infer - let currentPlan: UpdatePlanInput['plan'] = [] -export const updatePlanTool = defineMcpTool({ - name: 'update_plan', +export const updatePlanTool = tool({ description: 'Updates the task plan. At most one step can be in_progress at a time.', inputSchema: UPDATE_PLAN_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: false } }, + execute: async ({ explanation, plan }) => { const inProgressCount = plan.filter((item) => item.status === 'in_progress').length if (inProgressCount > 1) { diff --git a/packages/tools/src/tools/webfetch.test.ts b/packages/core/src/tools/tools/webfetch.test.ts similarity index 76% rename from packages/tools/src/tools/webfetch.test.ts rename to packages/core/src/tools/tools/webfetch.test.ts index f2a8a91..107ea59 100644 --- a/packages/tools/src/tools/webfetch.test.ts +++ b/packages/core/src/tools/tools/webfetch.test.ts @@ -1,12 +1,16 @@ import assert from 'node:assert' +import type { Tool, ToolExecutionOptions } from 'ai' +import type { ToolResultOutput } from '@ai-sdk/provider-utils' +import type { ToolOutput } from '@memo/core/tools/tools/mcp' import { afterEach, beforeEach, describe, test, vi } from 'vitest' +import { z } from 'zod' vi.mock('node:dns/promises', () => ({ lookup: vi.fn(), })) import { lookup } from 'node:dns/promises' -import { webfetchTool } from '@memo/tools/tools/webfetch' +import { webfetchTool } from '@memo/core/tools/tools/webfetch' const dnsLookupMock = vi.mocked(lookup) const WEBFETCH_ENV_KEYS = [ @@ -17,10 +21,10 @@ const WEBFETCH_ENV_KEYS = [ 'MEMO_WEBFETCH_BLOCK_PRIVATE_NET', ] -type ToolResult = { isError?: boolean; content?: Array<{ type: string; text?: string }> } +type ToolResult = ToolOutput function textPayload(result: ToolResult) { - return result.content?.find((item) => item.type === 'text')?.text ?? '' + return result.type === 'text' || result.type === 'error-text' ? result.value : '' } function installFetchMock( @@ -40,6 +44,10 @@ function installFetchMock( return fetchMock } +async function runTool(tool: Tool, input: unknown): Promise { + return (await tool.execute!(input, {} as ToolExecutionOptions)) as ToolResultOutput +} + describe('webfetch tool', () => { let originalFetch: typeof globalThis.fetch const previousEnv: Record = {} @@ -64,22 +72,22 @@ describe('webfetch tool', () => { }) test('requires url', async () => { - const res = webfetchTool.validateInput?.({ url: '' }) - assert.ok(res && !res.ok) + const schema = webfetchTool.inputSchema as z.ZodTypeAny + assert.strictEqual(schema.safeParse({ url: '' }).success, false) }) test('rejects unsupported protocol', async () => { - const res = await webfetchTool.execute({ url: 'file:///etc/hosts' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'file:///etc/hosts' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('Unsupported protocol')) }) test('rejects invalid proxy protocol', async () => { - const res = await webfetchTool.execute({ + const res = await runTool(webfetchTool, { url: 'https://example.com', proxy_url: 'socks5://127.0.0.1:1080', }) - assert.strictEqual(res.isError, true) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('Unsupported proxy protocol')) }) @@ -91,9 +99,9 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/html; charset=utf-8' }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com' }) + const res = await runTool(webfetchTool, { url: 'https://example.com' }) const text = textPayload(res) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.ok(text.includes('Contents of https://example.com/')) assert.ok(text.includes('Hello')) assert.ok(text.includes('World')) @@ -108,9 +116,9 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/html' }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com', raw: true }) + const res = await runTool(webfetchTool, { url: 'https://example.com', raw: true }) const text = textPayload(res) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.ok(text.includes('cannot be simplified to markdown')) assert.ok(text.includes('

Hello

')) }) @@ -123,9 +131,9 @@ describe('webfetch tool', () => { headers: { 'content-type': 'application/json' }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) const text = textPayload(res) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.ok(text.includes('cannot be simplified to markdown')) assert.ok(text.includes('{"key":"value"}')) }) @@ -138,13 +146,13 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/plain' }, }), ]) - const res = await webfetchTool.execute({ + const res = await runTool(webfetchTool, { url: 'https://example.com/data', start_index: 2, max_length: 4, }) const text = textPayload(res) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.ok(text.includes('cdef')) assert.ok(text.includes('start_index of 6')) }) @@ -157,11 +165,11 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/plain' }, }), ]) - const res = await webfetchTool.execute({ + const res = await runTool(webfetchTool, { url: 'https://example.com/data', start_index: 99, }) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.ok(textPayload(res).includes('No more content available.')) }) @@ -170,22 +178,22 @@ describe('webfetch tool', () => { new Response('not found', { status: 404 }), new Response('ok', { status: 200, headers: { 'content-type': 'text/plain' } }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) - assert.strictEqual(res.isError, false) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) + assert.strictEqual(res.type, 'text') assert.ok(textPayload(res).includes('Contents of https://example.com/data')) }) test('robots 403 blocks autonomous fetching', async () => { installFetchMock([new Response('blocked', { status: 403 })]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('autonomous fetching is not allowed')) }) test('robots disallow blocks autonomous fetching', async () => { installFetchMock([new Response('User-agent: *\nDisallow: /', { status: 200 })]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes("site's robots.txt")) }) @@ -197,16 +205,16 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/plain' }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) - assert.strictEqual(res.isError, false) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) + assert.strictEqual(res.type, 'text') assert.strictEqual(fetchMock.mock.calls.length, 1) }) test('blocks localhost target', async () => { const fetchMock = vi.fn() Object.assign(globalThis, { fetch: fetchMock as unknown as typeof globalThis.fetch }) - const res = await webfetchTool.execute({ url: 'http://localhost:8080/private' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'http://localhost:8080/private' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('Blocked private or local network host')) assert.strictEqual(fetchMock.mock.calls.length, 0) }) @@ -215,8 +223,8 @@ describe('webfetch tool', () => { dnsLookupMock.mockResolvedValue([{ address: '10.0.0.12', family: 4 }] as never) const fetchMock = vi.fn() Object.assign(globalThis, { fetch: fetchMock as unknown as typeof globalThis.fetch }) - const res = await webfetchTool.execute({ url: 'https://example.com/private' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/private' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('resolved to 10.0.0.12')) assert.strictEqual(fetchMock.mock.calls.length, 0) }) @@ -231,8 +239,8 @@ describe('webfetch tool', () => { headers: { 'content-type': 'text/plain' }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/data' }) - assert.strictEqual(res.isError, false) + const res = await runTool(webfetchTool, { url: 'https://example.com/data' }) + assert.strictEqual(res.type, 'text') }) test('returns timeout error when request aborts', async () => { @@ -250,8 +258,8 @@ describe('webfetch tool', () => { }) }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/slow' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/slow' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('timeout or aborted')) }) @@ -267,8 +275,8 @@ describe('webfetch tool', () => { }, }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/large' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/large' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('response body too large')) }) @@ -277,8 +285,8 @@ describe('webfetch tool', () => { new Response('User-agent: *\nAllow: /', { status: 200 }), new Response('not found', { status: 404, headers: { 'content-type': 'text/plain' } }), ]) - const res = await webfetchTool.execute({ url: 'https://example.com/missing' }) - assert.strictEqual(res.isError, true) + const res = await runTool(webfetchTool, { url: 'https://example.com/missing' }) + assert.strictEqual(res.type, 'error-text') assert.ok(textPayload(res).includes('status code 404')) }) @@ -287,11 +295,11 @@ describe('webfetch tool', () => { new Response('User-agent: *\nAllow: /', { status: 200 }), new Response('ok', { status: 200, headers: { 'content-type': 'text/plain' } }), ]) - const res = await webfetchTool.execute({ + const res = await runTool(webfetchTool, { url: 'https://example.com/data', proxy_url: 'http://proxy.example.com:8080', }) - assert.strictEqual(res.isError, false) + assert.strictEqual(res.type, 'text') assert.strictEqual(fetchMock.mock.calls.length, 2) const firstInit = fetchMock.mock.calls[0]?.[1] as RequestInit & { dispatcher?: unknown } const secondInit = fetchMock.mock.calls[1]?.[1] as RequestInit & { dispatcher?: unknown } diff --git a/packages/tools/src/tools/webfetch.ts b/packages/core/src/tools/tools/webfetch.ts similarity index 97% rename from packages/tools/src/tools/webfetch.ts rename to packages/core/src/tools/tools/webfetch.ts index 69c6345..f16bc50 100644 --- a/packages/tools/src/tools/webfetch.ts +++ b/packages/core/src/tools/tools/webfetch.ts @@ -7,8 +7,8 @@ import robotsParser from 'robots-parser' import TurndownService from 'turndown' import { ProxyAgent, type Dispatcher } from 'undici' import { z } from 'zod' -import { textResult } from '@memo/tools/tools/mcp' -import { defineMcpTool } from '@memo/tools/tools/types' +import { textResult } from '@memo/core/tools/tools/mcp' +import { tool } from 'ai' const WEBFETCH_INPUT_SCHEMA = z .object({ @@ -20,8 +20,6 @@ const WEBFETCH_INPUT_SCHEMA = z }) .strict() -type WebFetchInput = z.infer - const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']) const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]) const MAX_REDIRECTS = 10 @@ -335,13 +333,12 @@ function closeDispatcher(dispatcher: Dispatcher | undefined) { /** * Webfetch v2: paged HTTP fetch with optional markdown extraction and robots/security policy checks. */ -export const webfetchTool = defineMcpTool({ - name: 'webfetch', +export const webfetchTool = tool({ description: 'Fetch a URL, optionally simplify HTML to markdown, and return paged content with robots-aware policy checks.', inputSchema: WEBFETCH_INPUT_SCHEMA, - supportsParallelToolCalls: true, - isMutating: false, + metadata: { memo: { supportsParallelToolCalls: true, isMutating: false } }, + execute: async (input) => { let url: URL try { diff --git a/packages/tools/src/tools/write_file.ts b/packages/core/src/tools/tools/write_file.ts similarity index 61% rename from packages/tools/src/tools/write_file.ts rename to packages/core/src/tools/tools/write_file.ts index 57a3913..019b96b 100644 --- a/packages/tools/src/tools/write_file.ts +++ b/packages/core/src/tools/tools/write_file.ts @@ -1,8 +1,8 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { validatePath, writeFileContent } from '@memo/tools/tools/filesystem/lib' -import { resolveAllowedDirectories } from '@memo/tools/tools/filesystem/roots' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { validatePath, writeFileContent } from '@memo/core/tools/tools/filesystem/lib' +import { resolveAllowedDirectories } from '@memo/core/tools/tools/filesystem/roots' const WRITE_FILE_INPUT_SCHEMA = z .object({ @@ -11,14 +11,11 @@ const WRITE_FILE_INPUT_SCHEMA = z }) .strict() -type WriteFileInput = z.infer - -export const writeFileTool = defineMcpTool({ - name: 'write_file', +export const writeFileTool = tool({ description: 'Create or overwrite a file with UTF-8 content using atomic replace semantics.', inputSchema: WRITE_FILE_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async (input) => { try { const allowedDirectories = await resolveAllowedDirectories() diff --git a/packages/tools/src/tools/write_stdin.ts b/packages/core/src/tools/tools/write_stdin.ts similarity index 67% rename from packages/tools/src/tools/write_stdin.ts rename to packages/core/src/tools/tools/write_stdin.ts index 4ce186a..295443b 100644 --- a/packages/tools/src/tools/write_stdin.ts +++ b/packages/core/src/tools/tools/write_stdin.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import { defineMcpTool } from '@memo/tools/tools/types' -import { textResult } from '@memo/tools/tools/mcp' -import { writeExecSession } from '@memo/tools/tools/exec_runtime' +import { tool } from 'ai' +import { textResult } from '@memo/core/tools/tools/mcp' +import { writeExecSession } from '@memo/core/tools/tools/exec_runtime' const WRITE_STDIN_INPUT_SCHEMA = z .object({ @@ -12,14 +12,11 @@ const WRITE_STDIN_INPUT_SCHEMA = z }) .strict() -type WriteStdinInput = z.infer - -export const writeStdinTool = defineMcpTool({ - name: 'write_stdin', +export const writeStdinTool = tool({ description: 'Writes characters to an existing unified exec session and returns recent output.', inputSchema: WRITE_STDIN_INPUT_SCHEMA, - supportsParallelToolCalls: false, - isMutating: true, + metadata: { memo: { supportsParallelToolCalls: false, isMutating: true } }, + execute: async (input) => { try { const content = await writeExecSession({ diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index cc1f5ae..1cae1f4 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -1,60 +1,36 @@ /** @file Common type declarations shared between Core and Runtime (reused by UI/Tools). */ -import type { ApprovalRequest, ApprovalDecision } from '@memo/tools/approval' -import type { ToolActionStatus } from '@memo/tools/orchestrator' -export type { ApprovalDecision, ApprovalRequest } from '@memo/tools/approval' -export type { ToolActionStatus } from '@memo/tools/orchestrator' +import type { FinishReason, LanguageModelUsage, ModelMessage, ToolCallPart, ToolResultPart } from 'ai' +import type { ApprovalRequest, ApprovalDecision, ToolActionStatus } from '@memo/core/tools/approval' +import type { ToolExecutionContext } from '@memo/core/tools/sdk_tools' +import type { SkillIndex } from '@memo/core/skills/skills' +export type { ApprovalDecision, ApprovalRequest, ToolActionStatus } from '@memo/core/tools/approval' +export type { FinishReason, LanguageModelUsage } from 'ai' + +/** AI SDK generation result subset returned by CallLLM (all fields are AI SDK types). */ +export type LLMResult = { + /** Full generated text. */ + text: string + /** Reasoning output (DeepSeek thinking trace). */ + reasoning?: string + /** Tool calls made during generation. */ + toolCalls: ToolCallPart[] + /** Executed tool results (AI SDK executed tools with execute functions). */ + toolResults: ToolResultPart[] + /** Token usage. */ + usage: LanguageModelUsage + /** Finish reason. */ + finishReason: FinishReason +} /** * Basic type declarations for Agent layer, covering conversation messages, * parsing results, and dependency injection interfaces. - * Types are kept minimal for easy reuse in UI/tools layers. + * Types are kept minimal for easy reuse in UI/core/tools layers. */ export type Role = 'system' | 'user' | 'assistant' | 'tool' -/** Structured tool calls from Assistant (OpenAI tool_calls compatible format). */ -export type AssistantToolCall = { - id: string - type: 'function' - function: { - name: string - arguments: string - } -} - -/** Model-side messages: compatible with plain text and structured tool calls/results. */ -export type ChatMessage = - | { - /** System message. */ - role: 'system' - /** Message content. */ - content: string - } - | { - /** User message. */ - role: 'user' - /** Message content. */ - content: string - } - | { - /** Assistant text or structured tool calls. */ - role: 'assistant' - /** Assistant text; can be empty string for pure tool calls. */ - content: string - /** Optional DeepSeek thinking trace required for subsequent tool-call rounds. */ - reasoning_content?: string - /** Structured tool calls list (if any). */ - tool_calls?: AssistantToolCall[] - } - | { - /** Tool result message (corresponds to a tool_call). */ - role: 'tool' - /** Tool output text. */ - content: string - /** Corresponds to assistant.tool_calls[*].id. */ - tool_call_id: string - /** Optional tool name for debugging. */ - name?: string - } +/** Model-side messages: AI SDK ModelMessage (plain text or structured parts). */ +export type ChatMessage = ModelMessage /** Single-step debug record for replay and observability. */ export type AgentStepTrace = { @@ -66,18 +42,8 @@ export type AgentStepTrace = { parsed: ParsedAssistant /** Tool observation for this step (if any). */ observation?: string - /** Token statistics for this step. */ - tokenUsage: TokenUsage -} - -/** Token usage statistics: prompt/completion/total. */ -export type TokenUsage = { - /** Input prompt tokens. */ - prompt: number - /** Model generation tokens. */ - completion: number - /** Total tokens (prompt+completion if model doesn't return it). */ - total: number + /** Token statistics for this step (AI SDK LanguageModelUsage). */ + tokenUsage: LanguageModelUsage } export type CompactReason = 'auto' | 'manual' @@ -94,49 +60,12 @@ export type CompactResult = { errorMessage?: string } -/** Unified tokenizer counter interface compatible with different model encodings. */ +/** Unified token counter interface for prompt size estimation. */ export type TokenCounter = { - /** Actual tokenizer/encoding name used. */ - model: string /** Count tokens for plain text. */ countText: (text: string) => number /** Count tokens for message arrays. */ countMessages: (messages: ChatMessage[]) => number - /** Release underlying resources. */ - dispose: () => void -} - -/** Tool Use Block - tool call request */ -export type ToolUseBlock = { - type: 'tool_use' - /** Unique ID for the tool call */ - id: string - /** Tool name */ - name: string - /** Tool input parameters */ - input: unknown -} - -/** Text Block - text content */ -export type TextBlock = { - type: 'text' - /** Text content */ - text: string -} - -/** Content Block - can be text or tool call */ -export type ContentBlock = TextBlock | ToolUseBlock - -/** LLM response (unified structured content blocks). */ -export type LLMResponse = { - /** Structured content blocks (text + tool calls). */ - content: ContentBlock[] - /** Optional DeepSeek thinking trace for protocol-compatible follow-up requests. */ - reasoning_content?: string - /** Stop reason. */ - stop_reason: 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' - /** Token usage returned by model (optional). */ - usage?: Partial } /** Representation of parsed LLM output as action/final structure. */ @@ -149,28 +78,29 @@ export type ParsedAssistant = { thinking?: string } -/** Tool registry: keys are tool names, values are tool definitions. */ -export type ToolRegistry = Record - -/** Tool definition structure (for passing to LLM API) */ -export type ToolDefinition = { - name: string - description: string - input_schema: Record +export type ToolHookAction = NonNullable & { + toolCallId: string } +/** Tool registry: keys are tool names, values are standard AI SDK Tool definitions. */ +export type ToolRegistry = Record + /** LLM call interface: input history messages, return structured response, can stream text via onChunk. */ export type CallLLMOptions = { signal?: AbortSignal - /** Available tools list (Tool Use API mode) */ - tools?: ToolDefinition[] + /** Tool execution context (approval/gate/hooks) captured by the loop; absent disables tools. */ + toolContext?: ToolExecutionContext + /** Thinking toggle for this call; undefined falls back to the provider model profile. */ + thinking?: boolean + /** Streaming reasoning deltas for UI display. */ + onReasoningChunk?: (chunk: string) => void } export type CallLLM = ( messages: ChatMessage[], onChunk?: (chunk: string) => void, options?: CallLLMOptions, -) => Promise +) => Promise /** * Dependency injection collection required by runAgent. @@ -188,6 +118,8 @@ export type AgentDeps = { loadPrompt?: () => Promise /** Callback for each assistant output. */ onAssistantStep?: (content: string, step: number) => void + /** Callback for each streaming reasoning chunk (thinking trace). */ + onReasoningChunk?: (content: string, step: number) => void /** Hook collection: inject one-time lifecycle listeners. */ hooks?: AgentHooks /** Middleware list: can register multiple Hook implementations. */ @@ -196,6 +128,8 @@ export type AgentDeps = { dispose?: () => Promise /** Request user approval for tool calls (for dangerous operations) */ requestApproval?: (request: ApprovalRequest) => Promise + /** Deduped skill index for the session (read_skill tool reads it). */ + skillIndex?: SkillIndex } /** Session mode: currently only interactive is supported. */ @@ -212,8 +146,8 @@ export type AgentSessionOptions = { historyDir?: string /** Specify provider name to use. */ providerName?: string - /** Tokenizer encoding name, default cl100k_base. */ - tokenizerModel?: string + /** Model name override (resolved from the provider when omitted). */ + modelName?: string /** Working directory used by prompt/tool runtime for this session. */ cwd?: string /** Prompt warning threshold. */ @@ -228,6 +162,8 @@ export type AgentSessionOptions = { dangerous?: boolean /** 工具权限模式：禁用工具 / 每次审批 / 全部放行。 */ toolPermissionMode?: ToolPermissionMode + /** 思考模式初始开关（undefined 跟随模型 profile；可运行时 setThinking 切换）。 */ + thinking?: boolean } /** Session 运行需要的依赖（含扩展项）。 */ @@ -252,7 +188,7 @@ export type TurnResult = { /** 错误信息（若有）。 */ errorMessage?: string /** 本轮 token 统计。 */ - tokenUsage: TokenUsage + tokenUsage: LanguageModelUsage } export type TurnStartHookPayload = { @@ -268,13 +204,20 @@ export type ActionHookPayload = { sessionId: string turn: number step: number - action: NonNullable + action: ToolHookAction /** 并发工具调用时，包含所有工具 action（顺序与调用一致）。 */ - parallelActions?: Array> + parallelActions?: ToolHookAction[] thinking?: string history: ChatMessage[] } +export type ToolObservationResult = { + toolCallId: string + tool: string + observation: string + status: ToolActionStatus +} + export type ObservationHookPayload = { sessionId: string turn: number @@ -283,6 +226,8 @@ export type ObservationHookPayload = { observation: string resultStatus?: ToolActionStatus parallelResultStatuses?: ToolActionStatus[] + /** Structured per-call results. UI consumers should prefer this over the combined observation string. */ + results: ToolObservationResult[] history: ChatMessage[] } @@ -293,9 +238,11 @@ export type FinalHookPayload = { finalText: string status: TurnStatus errorMessage?: string - tokenUsage?: TokenUsage - turnUsage: TokenUsage + tokenUsage?: LanguageModelUsage + turnUsage: LanguageModelUsage steps: AgentStepTrace[] + /** Thinking trace of the final step (rendered on the last step cell). */ + thinking?: string } export type ContextUsagePhase = 'turn_start' | 'step_start' | 'post_compact' @@ -385,6 +332,8 @@ export type AgentSession = { listToolNames?: () => string[] /** 手动触发历史压缩。 */ compactHistory: (reason?: CompactReason) => Promise + /** 运行时切换思考模式（无需重建会话）。 */ + setThinking?: (enabled: boolean) => void /** 结束 Session，释放资源。 */ close: () => Promise } diff --git a/packages/core/src/utils/errors.test.ts b/packages/core/src/utils/errors.test.ts new file mode 100644 index 0000000..e9901ba --- /dev/null +++ b/packages/core/src/utils/errors.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from 'vitest' +import { isAbortError } from '@memo/core/utils/errors' + +describe('isAbortError', () => { + test('detects abort error by name and message', () => { + const abortError = new Error('cancelled') + abortError.name = 'AbortError' + const abortedMessageError = new Error('Request was aborted.') + expect(isAbortError(abortError)).toBe(true) + expect(isAbortError(abortedMessageError)).toBe(true) + expect(isAbortError(new Error('other'))).toBe(false) + expect(isAbortError('AbortError')).toBe(false) + }) +}) diff --git a/packages/core/src/utils/errors.ts b/packages/core/src/utils/errors.ts new file mode 100644 index 0000000..404b3f6 --- /dev/null +++ b/packages/core/src/utils/errors.ts @@ -0,0 +1,13 @@ +/** @file Error classification helpers. */ + +export function isAbortError(err: unknown): err is Error { + if (!(err instanceof Error)) return false + if (err.name === 'AbortError') return true + + const message = err.message?.toLowerCase?.() ?? '' + return ( + message.includes('request was aborted') || + message.includes('operation was aborted') || + message.includes('aborted') + ) +} diff --git a/packages/core/src/utils/serialize.test.ts b/packages/core/src/utils/serialize.test.ts new file mode 100644 index 0000000..4f000cc --- /dev/null +++ b/packages/core/src/utils/serialize.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from 'vitest' +import { stableStringify } from '@memo/core/utils/serialize' + +describe('stableStringify', () => { + test('serializes self-referencing object without throwing', () => { + const root: Record = {} + root.self = root + + const serialized = stableStringify(root) + expect(serialized).toBe('{"self":"[Circular]"}') + }) + + test('serializes indirect circular references with circular marker', () => { + const parent: Record = { name: 'parent' } + const child: Record = { name: 'child', parent } + parent.child = child + + const serialized = stableStringify(parent) + expect(serialized).toContain('"child":{"name":"child","parent":"[Circular]"}') + expect(serialized).toContain('"name":"parent"') + }) +}) diff --git a/packages/core/src/utils/serialize.ts b/packages/core/src/utils/serialize.ts new file mode 100644 index 0000000..be7fb5c --- /dev/null +++ b/packages/core/src/utils/serialize.ts @@ -0,0 +1,37 @@ +/** @file Stable JSON serialization (deterministic key order, cycle-safe). */ + +/** + * Stable serialization for duplicate action detection (ensures consistent key ordering). + * Cycles are marked with `[Circular]`; values deeper than MAX_DEPTH with `[MaxDepthExceeded]`. + */ +export function stableStringify(value: unknown): string { + return stableStringifyWithSeen(value, new WeakSet(), 0) +} + +const MAX_STABLE_STRINGIFY_DEPTH = 100 + +function stableStringifyWithSeen(value: unknown, seen: WeakSet, depth: number): string { + if (depth > MAX_STABLE_STRINGIFY_DEPTH) { + return JSON.stringify('[MaxDepthExceeded]') + } + if (typeof value === 'bigint') { + return JSON.stringify(value.toString()) + } + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' + if (seen.has(value)) { + return JSON.stringify('[Circular]') + } + + seen.add(value) + if (Array.isArray(value)) { + const result = `[${value.map((v) => stableStringifyWithSeen(v, seen, depth + 1)).join(',')}]` + seen.delete(value) + return result + } + const entries = Object.entries(value as Record).sort(([a], [b]) => a.localeCompare(b)) + const result = `{${entries + .map(([k, v]) => `${JSON.stringify(k)}:${stableStringifyWithSeen(v, seen, depth + 1)}`) + .join(',')}}` + seen.delete(value) + return result +} diff --git a/packages/core/src/utils/title.test.ts b/packages/core/src/utils/title.test.ts new file mode 100644 index 0000000..f38bbb8 --- /dev/null +++ b/packages/core/src/utils/title.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from 'vitest' +import { fallbackSessionTitleFromPrompt, normalizeSessionTitle, truncateSessionTitle } from '@memo/core/utils/title' + +describe('session title helpers', () => { + test('truncateSessionTitle appends ellipsis when exceeding max', () => { + const truncated = truncateSessionTitle('x'.repeat(80)) + expect(truncated.endsWith('...')).toBe(true) + expect(truncated.length).toBe(60) + }) + + test('normalizeSessionTitle strips quotes and whitespace', () => { + expect(normalizeSessionTitle(' " Hello\nWorld " ')).toBe('Hello World') + expect(normalizeSessionTitle(' ')).toBe('') + }) + + test('normalizeSessionTitle removes think tags and title prefixes', () => { + expect( + normalizeSessionTitle( + 'internal Title: "Build REST API migration plan" secret', + ), + ).toBe('Build REST API migration plan') + }) + + test('fallbackSessionTitleFromPrompt handles empty/cjk/word prompts', () => { + expect(fallbackSessionTitleFromPrompt(' ')).toBe('New Session') + expect(fallbackSessionTitleFromPrompt('这是一个非常非常长的中文标题用于测试截断行为')).toBe( + '这是一个非常非常长的中文标题用于测试截断...', + ) + expect(fallbackSessionTitleFromPrompt('build a rest api using express and sqlite quickly')).toBe( + 'build a rest api using express and sqlite', + ) + }) +}) diff --git a/packages/core/src/utils/title.ts b/packages/core/src/utils/title.ts new file mode 100644 index 0000000..d5b189d --- /dev/null +++ b/packages/core/src/utils/title.ts @@ -0,0 +1,37 @@ +/** @file Session title helpers. */ + +export const SESSION_TITLE_MAX_CHARS = 60 + +export function truncateSessionTitle(input: string): string { + if (input.length <= SESSION_TITLE_MAX_CHARS) return input + return `${input.slice(0, SESSION_TITLE_MAX_CHARS - 3).trimEnd()}...` +} + +export function normalizeSessionTitle(raw: string): string { + const compact = raw + .replace(/<\s*(think|thinking)\b[^>]*>[\s\S]*?<\s*\/\s*\1\s*>/gi, ' ') + .replace(/<\s*\/?\s*(think|thinking)\b[^>]*>/gi, ' ') + .replace(/\r?\n+/g, ' ') + .replace(/\s+/g, ' ') + .trim() + if (!compact) return '' + const unprefixed = compact.replace(/^(title|session title|标题)\s*[:：-]\s*/i, '').trim() + if (!unprefixed) return '' + const unquoted = unprefixed.replace(/^["'`“”‘’]+|["'`“”‘’]+$/g, '').trim() + if (!unquoted) return '' + return truncateSessionTitle(unquoted) +} + +export function fallbackSessionTitleFromPrompt(input: string): string { + const compact = input.replace(/\s+/g, ' ').trim() + if (!compact) return 'New Session' + + // Keep short CJK/non-space prompts readable. + if (!compact.includes(' ')) { + return compact.length <= 20 ? compact : `${compact.slice(0, 20).trimEnd()}...` + } + + const words = compact.split(' ').filter(Boolean) + const short = words.slice(0, 8).join(' ') + return truncateSessionTitle(short || compact) +} diff --git a/packages/core/src/utils/tokenizer.fallback.test.ts b/packages/core/src/utils/tokenizer.fallback.test.ts new file mode 100644 index 0000000..fb24e13 --- /dev/null +++ b/packages/core/src/utils/tokenizer.fallback.test.ts @@ -0,0 +1,37 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest' + +// Simulate a tiktoken load failure (e.g. corrupted bundle): the counter must +// fall back to the byte estimate and keep working. Kept in a separate file +// because the encoding singleton caches success across tests. +vi.mock('js-tiktoken/lite', () => ({ + Tiktoken: class { + constructor() { + throw new Error('mock load failure') + } + }, +})) +vi.mock('js-tiktoken/ranks/cl100k_base', () => ({ default: {} })) + +describe('createTokenCounter fallback', () => { + beforeEach(() => { + vi.resetModules() + }) + + test('countText falls back to the byte estimate when tiktoken load fails', async () => { + const { createTokenCounter: fresh } = await import('@memo/core/utils/tokenizer') + const counter = fresh() + expect(counter.countText('hello')).toBe(2) // 5 bytes → ceil(5/4) + expect(counter.countText('')).toBe(0) + }) + + test('countMessages falls back to JSON-serialized byte estimates', async () => { + const { createTokenCounter: fresh } = await import('@memo/core/utils/tokenizer') + const counter = fresh() + const messages = [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Hello there!' }, + ] + const expected = messages.reduce((sum, message) => sum + counter.countText(JSON.stringify(message)), 0) + expect(counter.countMessages(messages)).toBe(expected) + }) +}) diff --git a/packages/core/src/utils/tokenizer.test.ts b/packages/core/src/utils/tokenizer.test.ts index 575d4d7..f344e1c 100644 --- a/packages/core/src/utils/tokenizer.test.ts +++ b/packages/core/src/utils/tokenizer.test.ts @@ -1,250 +1,99 @@ -import { describe, expect, test, beforeEach, afterEach } from 'vitest' +import { describe, expect, test } from 'vitest' import { createTokenCounter } from '@memo/core/utils/tokenizer' import type { ChatMessage } from '@memo/core/types' describe('createTokenCounter', () => { - test('creates counter with default model', () => { + test('creates counter with countText/countMessages', () => { const counter = createTokenCounter() - expect(counter.model).toBe('cl100k_base') expect(typeof counter.countText).toBe('function') expect(typeof counter.countMessages).toBe('function') - expect(typeof counter.dispose).toBe('function') - counter.dispose() - }) - - test('creates counter with specified model', () => { - const counter = createTokenCounter('gpt-4') - expect(counter.model).toBe('gpt-4') - counter.dispose() - }) - - test('falls back to cl100k_base for unknown models', () => { - const counter = createTokenCounter('unknown-model-x') - expect(counter.model).toBe('cl100k_base') - counter.dispose() - }) - - test('trims whitespace in model name', () => { - const counter = createTokenCounter(' gpt-4 ') - expect(counter.model).toBe('gpt-4') - counter.dispose() }) describe('countText', () => { - let counter: ReturnType - - beforeEach(() => { - counter = createTokenCounter() - }) - - afterEach(() => { - counter.dispose() - }) + const counter = createTokenCounter() test('returns 0 for empty string', () => { expect(counter.countText('')).toBe(0) }) - test('counts tokens for simple text', () => { - const count = counter.countText('Hello world') - expect(count).toBeGreaterThan(0) + test('counts ASCII text with the cl100k encoding', () => { + expect(counter.countText('hello')).toBe(1) + expect(counter.countText('Hello world')).toBe(2) + }) + + test('counts CJK chars (~1 token each in cl100k)', () => { + expect(counter.countText('你好')).toBe(2) + expect(counter.countText('中文测试')).toBe(3) }) - test('counts tokens for longer text', () => { + test('counts longer text more than short text', () => { const short = counter.countText('Hi') const long = counter.countText('Hello, this is a longer text with more words.') expect(long).toBeGreaterThan(short) }) - test('counts tokens for special characters', () => { - const count = counter.countText('Hello\nWorld\t!\n\n') - expect(count).toBeGreaterThan(0) - }) - - test('counts tokens for unicode text', () => { - const count = counter.countText('你好世界 Hello World 🌍') - expect(count).toBeGreaterThan(0) - }) - - test('counts tokens for JSON strings', () => { - const json = JSON.stringify({ key: 'value', nested: { a: 1, b: 2 } }) - const count = counter.countText(json) - expect(count).toBeGreaterThan(0) + test('counts unicode and special characters', () => { + expect(counter.countText('你好世界 Hello World 🌍')).toBeGreaterThan(0) + expect(counter.countText('Hello\nWorld\t!\n\n')).toBeGreaterThan(0) }) }) describe('countMessages', () => { - let counter: ReturnType - - beforeEach(() => { - counter = createTokenCounter() - }) - - afterEach(() => { - counter.dispose() - }) + const counter = createTokenCounter() test('returns 0 for empty array', () => { expect(counter.countMessages([])).toBe(0) }) - test('counts system message', () => { - const messages: ChatMessage[] = [{ role: 'system', content: 'You are a helpful assistant.' }] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('counts user message', () => { - const messages: ChatMessage[] = [{ role: 'user', content: 'Hello there!' }] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('counts assistant message', () => { - const messages: ChatMessage[] = [{ role: 'assistant', content: 'Hi! How can I help?' }] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('counts tool message', () => { + test('counts encoded content plus fixed structural overhead per message', () => { const messages: ChatMessage[] = [ - { - role: 'tool', - content: 'Tool execution result', - tool_call_id: 'call-123', - name: 'test_tool', - }, + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Hello there!' }, ] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) + const expected = 6 + 3 + 3 + 3 // 'You are...' 6 tok + 'Hello there!' 3 tok + 2 × 3 structural + expect(counter.countMessages(messages)).toBe(expected) }) - test('counts multiple messages', () => { + test('counts multiple messages more than a single message', () => { const messages: ChatMessage[] = [ { role: 'system', content: 'System prompt' }, { role: 'user', content: 'User message' }, { role: 'assistant', content: 'Assistant response' }, ] - const firstMsg = messages[0] - if (firstMsg) { - const single = counter.countMessages([firstMsg]) - const multiple = counter.countMessages(messages) - expect(multiple).toBeGreaterThan(single) - } + const single = counter.countMessages([messages[0]!]) + expect(counter.countMessages(messages)).toBeGreaterThan(single) }) - test('includes assistant priming tokens', () => { - const messages: ChatMessage[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi' }, - ] - const count = counter.countMessages(messages) - const withoutAssistant = counter.countMessages([{ role: 'user', content: 'Hello' }]) - expect(count).toBeGreaterThan(withoutAssistant) - }) - - test('counts tool_calls in assistant message', () => { - const messagesWithToolCalls: ChatMessage[] = [ + test('includes structured parts (tool-call/reasoning)', () => { + const withParts: ChatMessage[] = [ { role: 'assistant', - content: 'Let me check', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { name: 'read_file', arguments: '{"path": "test.txt"}' }, - }, + content: [ + { type: 'text', text: 'Let me check' }, + { type: 'reasoning', text: 'I should inspect file A before acting.' }, + { type: 'tool-call', toolCallId: 'call-1', toolName: 'read_file', input: { path: 'test.txt' } }, ], }, ] - const messagesWithoutToolCalls: ChatMessage[] = [{ role: 'assistant', content: 'Let me check' }] - const withCalls = counter.countMessages(messagesWithToolCalls) - const withoutCalls = counter.countMessages(messagesWithoutToolCalls) - expect(withCalls).toBeGreaterThan(withoutCalls) + const withoutParts: ChatMessage[] = [{ role: 'assistant', content: 'Let me check' }] + expect(counter.countMessages(withParts)).toBeGreaterThan(counter.countMessages(withoutParts)) }) - test('counts reasoning_content in assistant message', () => { - const messagesWithReasoning: ChatMessage[] = [ - { - role: 'assistant', - content: '', - reasoning_content: 'I should inspect file A before using read_file.', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { name: 'read_file', arguments: '{"path":"README.md"}' }, - }, - ], - }, - ] - const messagesWithoutReasoning: ChatMessage[] = [ - { - role: 'assistant', - content: '', - tool_calls: [ - { - id: 'call-1', - type: 'function', - function: { name: 'read_file', arguments: '{"path":"README.md"}' }, - }, - ], - }, - ] - const withReasoning = counter.countMessages(messagesWithReasoning) - const withoutReasoning = counter.countMessages(messagesWithoutReasoning) - expect(withReasoning).toBeGreaterThan(withoutReasoning) - }) - - test('includes tool_call_id in tool message counting', () => { + test('counts tool result messages', () => { const messages: ChatMessage[] = [ { role: 'tool', - content: 'Result', - tool_call_id: 'call-abc123', - }, - ] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('includes name field in tool message counting', () => { - const messages: ChatMessage[] = [ - { - role: 'tool', - content: 'Result', - tool_call_id: 'call-1', - name: 'my_tool', - }, - ] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('handles assistant message with only tool_calls and empty content', () => { - const messages: ChatMessage[] = [ - { - role: 'assistant', - content: '', - tool_calls: [ + content: [ { - id: 'call-1', - type: 'function', - function: { name: 'test', arguments: '{}' }, + type: 'tool-result', + toolCallId: 'call-123', + toolName: 'test_tool', + output: { type: 'text', value: 'Tool execution result' }, }, ], }, ] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - - test('includes name overhead when message has name field', () => { - const msg: ChatMessage & { name?: string } = { role: 'system', content: 'Test' } - msg.name = 'custom_name' - const messages: ChatMessage[] = [msg] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) + expect(counter.countMessages(messages)).toBeGreaterThan(0) }) test('counts complex conversation', () => { @@ -253,41 +102,36 @@ describe('createTokenCounter', () => { { role: 'user', content: 'Write a function that adds two numbers.' }, { role: 'assistant', - content: 'I will create a simple add function for you.', - tool_calls: [ + content: [ + { type: 'text', text: 'I will create a simple add function for you.' }, { - id: 'call-1', - type: 'function', - function: { - name: 'write_file', - arguments: JSON.stringify({ - path: 'add.js', - content: 'function add(a, b) { return a + b; }', - }), + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'write_file', + input: { + path: 'add.js', + content: 'function add(a, b) { return a + b; }', }, }, ], }, { role: 'tool', - content: 'File written successfully', - tool_call_id: 'call-1', - name: 'write_file', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'write_file', + output: { type: 'text', value: 'File written successfully' }, + }, + ], }, { role: 'assistant', content: 'I have created the add.js file with the function.', }, ] - const count = counter.countMessages(messages) - expect(count).toBeGreaterThan(0) - }) - }) - - describe('dispose', () => { - test('disposes counter without error', () => { - const counter = createTokenCounter() - counter.dispose() + expect(counter.countMessages(messages)).toBeGreaterThan(0) }) }) }) diff --git a/packages/core/src/utils/tokenizer.ts b/packages/core/src/utils/tokenizer.ts index b28e830..1c95932 100644 --- a/packages/core/src/utils/tokenizer.ts +++ b/packages/core/src/utils/tokenizer.ts @@ -1,76 +1,93 @@ -/** @file tiktoken wrapper for token estimation. Used for compaction triggering, context overflow checks, and tool result sizing — not for usage reporting. */ -import { encoding_for_model, get_encoding, type Tiktoken } from '@dqbd/tiktoken' +/** @file Token estimation for compaction triggering and context overflow checks — not for usage reporting. */ import type { ChatMessage, TokenCounter } from '@memo/core/types' +import { Tiktoken } from 'js-tiktoken/lite' +import cl100k from 'js-tiktoken/ranks/cl100k_base' -const DEFAULT_TOKENIZER_MODEL = 'cl100k_base' +// cl100k_base covers OpenAI-compatible models (and is the closest general-purpose +// encoding for compatible providers). Loaded lazily once and cached: the vocab is +// ~1MB and encoding calls are sync. Any load failure falls back to the byte +// estimate below so the counter always works. +let encoding: Tiktoken | null = null +let encodingLoadFailed = false -type EncodingFactory = () => Tiktoken - -function safeEncodingFactory(model?: string): { model: string; factory: EncodingFactory } { - const resolvedModel = model?.trim() || DEFAULT_TOKENIZER_MODEL +function getEncodingCached(): Tiktoken | null { + if (encoding) return encoding + if (encodingLoadFailed) return null try { - // encoding_for_model requires strict model names; using type assertion for dynamic input compatibility. - const factory = () => encoding_for_model(resolvedModel as any) - factory().free() - return { model: resolvedModel, factory } + encoding = new Tiktoken(cl100k) } catch { - // Fallback to generic cl100k_base for unknown models to avoid throwing. - const fallbackModel = DEFAULT_TOKENIZER_MODEL - const factory = () => get_encoding(fallbackModel) - factory().free() - return { model: fallbackModel, factory } + encodingLoadFailed = true } + return encoding } -function messagePayloadForCounting(message: ChatMessage): string { - if (message.role === 'assistant') { - const reasoning = message.reasoning_content ? `\n${message.reasoning_content}` : '' - if (message.tool_calls?.length) { - return `${message.content}${reasoning}\n${JSON.stringify(message.tool_calls)}` - } - return `${message.content}${reasoning}` - } - if (message.role === 'tool') { - return `${message.content}\n${message.tool_call_id}\n${message.name ?? ''}` - } - return message.content +// OpenAI's common approximation: 1 token ≈ 4 bytes (UTF-8). +// CJK chars are ~3 bytes each, so the estimate stays within range for Chinese too. +const BYTES_PER_TOKEN = 4 + +const encoder = new TextEncoder() + +/** Rough token count for plain text: ceil(utf8 bytes / 4). Fallback when tiktoken is unavailable. */ +function approxTokenCount(text: string): number { + if (!text) return 0 + return Math.ceil(encoder.encode(text).length / BYTES_PER_TOKEN) } -/** Create a reusable token counter for prompt size estimation (compaction trigger, context overflow check). */ -export function createTokenCounter(model?: string): TokenCounter { - const { model: resolvedModel, factory } = safeEncodingFactory(model) - const encoding = factory() +// Fixed structural overhead per message (role wrapper + delimiters), added on top +// of the encoded content when tiktoken is available. +const STRUCTURAL_TOKENS_PER_MESSAGE = 3 - // ChatML rough estimation: each message includes role/name wrapping overhead - // Reference OpenAI's common estimates for gpt-3.5/4: about 4 tokens per message, plus 2 tokens for assistant priming. - const TOKENS_PER_MESSAGE = 4 - const TOKENS_FOR_ASSISTANT_PRIMING = 2 - const TOKENS_PER_NAME = 1 +function encodeText(enc: Tiktoken, text: string): number { + try { + return enc.encode(text).length + } catch { + return approxTokenCount(text) + } +} - const countText = (text: string) => { - if (!text) return 0 - return encoding.encode(text).length +function countMessageTokens(message: ChatMessage): number { + const enc = getEncodingCached() + if (!enc) { + return approxTokenCount(JSON.stringify(message)) } - const countMessages = (messages: ChatMessage[]) => { - if (!messages.length) return 0 - let total = 0 - for (const message of messages) { - total += TOKENS_PER_MESSAGE - total += countText(messagePayloadForCounting(message)) - // Currently not using message.name, but add overhead when name field is reserved - if ((message as any).name) { - total += TOKENS_PER_NAME - } + let count = STRUCTURAL_TOKENS_PER_MESSAGE + const content = message.content + if (typeof content === 'string') { + count += encodeText(enc, content) + return count + } + for (const part of content) { + switch (part.type) { + case 'text': + case 'reasoning': + count += encodeText(enc, part.text) + break + case 'tool-call': + count += encodeText(enc, part.toolName) + encodeText(enc, JSON.stringify(part.input)) + break + case 'tool-result': + count += encodeText(enc, part.toolName) + count += + part.output.type === 'text' + ? encodeText(enc, part.output.value) + : encodeText(enc, JSON.stringify(part.output)) + break } - total += TOKENS_FOR_ASSISTANT_PRIMING - return total } + return count +} +/** Create a token counter for prompt size estimation (compaction trigger, context overflow check). */ +export function createTokenCounter(): TokenCounter { return { - model: resolvedModel, - countText, - countMessages, - dispose: () => encoding.free(), + countText: (text: string) => { + if (!text) return 0 + const enc = getEncodingCached() + if (enc) return encodeText(enc, text) + return approxTokenCount(text) + }, + countMessages: (messages: ChatMessage[]) => + messages.reduce((sum, message) => sum + countMessageTokens(message), 0), } } diff --git a/packages/core/src/utils/usage.test.ts b/packages/core/src/utils/usage.test.ts new file mode 100644 index 0000000..78a87f9 --- /dev/null +++ b/packages/core/src/utils/usage.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'vitest' +import { accumulateUsage, emptyUsage } from '@memo/core/utils/usage' + +describe('accumulateUsage', () => { + test('uses explicit total when provided', () => { + const usage = emptyUsage() + accumulateUsage(usage, { inputTokens: 2, outputTokens: 3, totalTokens: 100 }) + expect(usage).toEqual({ ...emptyUsage(), inputTokens: 2, outputTokens: 3, totalTokens: 100 }) + }) + + test('falls back to input + output when total is absent', () => { + const usage = emptyUsage() + accumulateUsage(usage, { inputTokens: 2, outputTokens: 3 }) + expect(usage).toEqual({ ...emptyUsage(), inputTokens: 2, outputTokens: 3, totalTokens: 5 }) + }) +}) diff --git a/packages/core/src/utils/usage.ts b/packages/core/src/utils/usage.ts new file mode 100644 index 0000000..304a603 --- /dev/null +++ b/packages/core/src/utils/usage.ts @@ -0,0 +1,22 @@ +/** @file Token usage aggregation helpers for AI SDK LanguageModelUsage. */ +import type { LanguageModelUsage } from 'ai' + +export function emptyUsage(): LanguageModelUsage { + return { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + inputTokenDetails: { noCacheTokens: undefined, cacheReadTokens: undefined, cacheWriteTokens: undefined }, + outputTokenDetails: { textTokens: undefined, reasoningTokens: undefined }, + } +} + +export function accumulateUsage(target: LanguageModelUsage, delta?: Partial) { + if (!delta) return + const inputDelta = delta.inputTokens ?? 0 + const outputDelta = delta.outputTokens ?? 0 + const totalDelta = delta.totalTokens ?? inputDelta + outputDelta + target.inputTokens = (target.inputTokens ?? 0) + inputDelta + target.outputTokens = (target.outputTokens ?? 0) + outputDelta + target.totalTokens = (target.totalTokens ?? 0) + totalDelta +} diff --git a/packages/core/src/runtime/workspace.test.ts b/packages/core/src/utils/workspace.test.ts similarity index 100% rename from packages/core/src/runtime/workspace.test.ts rename to packages/core/src/utils/workspace.test.ts diff --git a/packages/core/src/runtime/workspace.ts b/packages/core/src/utils/workspace.ts similarity index 100% rename from packages/core/src/runtime/workspace.ts rename to packages/core/src/utils/workspace.ts diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 88e4ace..e1c4db1 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -11,5 +11,15 @@ export default defineConfig({ minify: false, splitting: false, bundle: true, - external: ['@dqbd/tiktoken', '@mozilla/readability', 'ipaddr.js', 'jsdom', 'robots-parser', 'turndown', 'undici'], + external: [ + '@dqbd/tiktoken', + '@mozilla/readability', + 'ipaddr.js', + 'jsdom', + 'robots-parser', + 'turndown', + 'undici', + 'ai', + '@ai-sdk/openai-compatible', + ], }) diff --git a/packages/tools/package.json b/packages/tools/package.json deleted file mode 100644 index e3baa65..0000000 --- a/packages/tools/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@memo-code/tools", - "type": "module", - "module": "src/index.ts", - "version": "0.1.0", - "private": true, - "scripts": { - "test": "vitest run" - }, - "dependencies": { - "@modelcontextprotocol/sdk": "^1.24.3", - "diff": "^8.0.3", - "ignore": "^5.3.1", - "minimatch": "^10.0.1", - "zod": "^4.3.6" - }, - "devDependencies": { - "vitest": "^2.1.8" - } -} diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts deleted file mode 100644 index f09aa74..0000000 --- a/packages/tools/src/index.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { McpTool, ToolName } from '@memo/tools/tools/types' -import { shellTool } from '@memo/tools/tools/shell' -import { shellCommandTool } from '@memo/tools/tools/shell_command' -import { execCommandTool } from '@memo/tools/tools/exec_command' -import { writeStdinTool } from '@memo/tools/tools/write_stdin' -import { applyPatchTool } from '@memo/tools/tools/apply_patch' -import { readTextFileTool } from '@memo/tools/tools/read_text_file' -import { readMediaFileTool } from '@memo/tools/tools/read_media_file' -import { readFilesTool } from '@memo/tools/tools/read_files' -import { writeFileTool } from '@memo/tools/tools/write_file' -import { editFileTool } from '@memo/tools/tools/edit_file' -import { listDirectoryTool } from '@memo/tools/tools/list_directory' -import { searchFilesTool } from '@memo/tools/tools/search_files' -import { - listMcpResourceTemplatesTool, - listMcpResourcesTool, - readMcpResourceTool, -} from '@memo/tools/tools/mcp_resources' -import { updatePlanTool } from '@memo/tools/tools/update_plan' -import { getMemoryTool } from '@memo/tools/tools/get_memory' -import { webfetchTool } from '@memo/tools/tools/webfetch' -import { closeAgentTool, resumeAgentTool, sendInputTool, spawnAgentTool, waitTool } from '@memo/tools/tools/collab' - -function buildCodexTools(): McpTool[] { - const tools: McpTool[] = [] - const shellMode = process.env.MEMO_SHELL_TOOL_TYPE?.trim() || 'unified_exec' - const collabEnabled = process.env.MEMO_ENABLE_COLLAB_TOOLS !== '0' - const memoryToolEnabled = process.env.MEMO_ENABLE_MEMORY_TOOL !== '0' - - if (shellMode === 'shell') { - tools.push(shellTool) - } else if (shellMode === 'shell_command') { - tools.push(shellCommandTool) - } else if (shellMode === 'unified_exec') { - tools.push(execCommandTool, writeStdinTool) - } else if (shellMode !== 'disabled') { - tools.push(execCommandTool, writeStdinTool) - } - - tools.push(listMcpResourcesTool, listMcpResourceTemplatesTool, readMcpResourceTool) - tools.push(updatePlanTool) - tools.push(applyPatchTool) - tools.push( - readTextFileTool, - readMediaFileTool, - readFilesTool, - writeFileTool, - editFileTool, - listDirectoryTool, - searchFilesTool, - ) - - if (memoryToolEnabled) { - tools.push(getMemoryTool) - } - - tools.push(webfetchTool) - - if (collabEnabled) { - tools.push(spawnAgentTool, sendInputTool, resumeAgentTool, waitTool, closeAgentTool) - } - - return tools -} - -function indexByName(tools: McpTool[]): Record { - const toolkit: Record = {} - for (const tool of tools) { - toolkit[tool.name] = tool - } - return toolkit -} - -/** Exposed tool collection for Agent lookup by tool name. */ -export const TOOLKIT: Record = indexByName(buildCodexTools()) - -/** Tool array form, convenient for direct registration to MCP Server etc. */ -export const TOOL_LIST: McpTool[] = Object.values(TOOLKIT) - -/** Built-in tools (already unified Tool format, no adaptation needed). */ -export const NATIVE_TOOLS = TOOL_LIST - -export type { McpTool } -export * from '@memo/tools/approval' -export * from '@memo/tools/orchestrator' -export * from '@memo/tools/router' diff --git a/packages/tools/src/orchestrator/index.test.ts b/packages/tools/src/orchestrator/index.test.ts deleted file mode 100644 index cb3a96c..0000000 --- a/packages/tools/src/orchestrator/index.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -import assert from 'node:assert' -import { describe, test } from 'vitest' -import { z } from 'zod' -import { createToolOrchestrator } from './index' - -describe('tool orchestrator', () => { - test('stops on rejection in sequential execution', async () => { - const calls: string[] = [] - const orchestrator = createToolOrchestrator({ - tools: { - shell_command: { - name: 'shell_command', - execute: async () => { - calls.push('shell_command') - return { content: [{ type: 'text', text: 'ok' }] } - }, - }, - read_file: { - name: 'read_file', - execute: async () => { - calls.push('read_file') - return { content: [{ type: 'text', text: 'read_file' }] } - }, - }, - }, - }) - - const result = await orchestrator.executeActions( - [ - { name: 'shell_command', input: { cmd: 'echo hi' } }, - { name: 'read_file', input: { file_path: '/tmp/a.txt' } }, - ], - { - requestApproval: async () => 'deny', - }, - ) - - assert.strictEqual(result.hasRejection, true) - assert.deepStrictEqual(calls, []) - assert.strictEqual(result.results.length, 1) - assert.strictEqual(result.results[0]?.tool, 'shell_command') - assert.strictEqual(result.results[0]?.status, 'approval_denied') - assert.strictEqual(result.results[0]?.errorType, 'approval_denied') - assert.strictEqual(result.results[0]?.rejected, true) - assert.ok((result.results[0]?.durationMs ?? 0) >= 0) - assert.ok(result.results[0]?.actionId.length) - }) - - test('executes tool when approval is granted', async () => { - const orchestrator = createToolOrchestrator({ - tools: { - apply_patch: { - name: 'apply_patch', - validateInput: (input) => { - const schema = z.object({ - file_path: z.string(), - old_string: z.string(), - new_string: z.string(), - }) - const parsed = schema.safeParse(input) - return parsed.success ? { ok: true, data: parsed.data } : { ok: false, error: 'invalid input' } - }, - execute: async () => ({ - content: [{ type: 'text', text: 'written' }], - }), - }, - }, - }) - - const result = await orchestrator.executeAction( - { - name: 'apply_patch', - input: { file_path: '/tmp/a.txt', old_string: 'a', new_string: 'b' }, - }, - { requestApproval: async () => 'once' }, - ) - - assert.strictEqual(result.success, true) - assert.strictEqual(result.status, 'success') - assert.strictEqual(result.observation, 'written') - assert.ok(result.durationMs >= 0) - assert.ok(result.actionId.length) - }) - - test('auto-approves subagent tools even in strict approval mode', async () => { - let askedApproval = false - const orchestrator = createToolOrchestrator({ - tools: { - spawn_agent: { - name: 'spawn_agent', - execute: async () => ({ - content: [{ type: 'text', text: 'spawned' }], - }), - }, - }, - approval: { - mode: 'strict', - }, - }) - - const result = await orchestrator.executeAction( - { name: 'spawn_agent', input: { message: 'task' } }, - { - requestApproval: async () => { - askedApproval = true - return 'deny' - }, - }, - ) - - assert.strictEqual(askedApproval, false) - assert.strictEqual(result.success, true) - assert.strictEqual(result.status, 'success') - assert.strictEqual(result.observation, 'spawned') - }) - - test('returns unknown tool error', async () => { - const orchestrator = createToolOrchestrator({ tools: {} }) - const result = await orchestrator.executeAction( - { name: 'missing', input: {} }, - { requestApproval: async () => 'once' }, - ) - assert.strictEqual(result.success, false) - assert.strictEqual(result.status, 'tool_not_found') - assert.strictEqual(result.errorType, 'tool_not_found') - assert.strictEqual(result.observation, 'Unknown tool: missing') - }) - - test('classifies sandbox-like execution failures', async () => { - const orchestrator = createToolOrchestrator({ - tools: { - exec_command: { - name: 'exec_command', - execute: async () => { - throw new Error('Permission denied by sandbox') - }, - }, - }, - }) - const result = await orchestrator.executeAction( - { name: 'exec_command', input: { cmd: 'rm -rf /' } }, - { requestApproval: async () => 'once' }, - ) - assert.strictEqual(result.success, false) - assert.strictEqual(result.status, 'sandbox_denied') - assert.strictEqual(result.errorType, 'sandbox_denied') - assert.ok(result.observation.startsWith('Tool execution failed:')) - }) - - test('replaces oversized tool output with xml system hint', async () => { - const prevLimit = process.env.MEMO_TOOL_RESULT_MAX_CHARS - process.env.MEMO_TOOL_RESULT_MAX_CHARS = '64' - try { - const orchestrator = createToolOrchestrator({ - tools: { - read_file: { - name: 'read_file', - execute: async () => ({ - content: [{ type: 'text', text: 'x'.repeat(1000) }], - }), - }, - }, - }) - - const result = await orchestrator.executeAction( - { name: 'read_file', input: { file_path: '/tmp/a' } }, - { requestApproval: async () => 'once' }, - ) - - assert.strictEqual(result.success, true) - assert.strictEqual(result.status, 'success') - assert.ok(result.observation.startsWith(' { - let executed = false - const orchestrator = createToolOrchestrator({ - tools: { - read_file: { - name: 'read_file', - execute: async () => { - executed = true - return { content: [{ type: 'text', text: 'ok' }] } - }, - }, - }, - }) - - const result = await orchestrator.executeAction({ - name: 'read_file', - input: 'x'.repeat(100_001), - }) - - assert.strictEqual(result.success, false) - assert.strictEqual(result.status, 'input_invalid') - assert.ok(result.observation.includes('input string too large')) - assert.strictEqual(executed, false) - }) - - test('rejects non-object json input payloads', async () => { - const orchestrator = createToolOrchestrator({ - tools: { - read_file: { - name: 'read_file', - execute: async () => ({ content: [{ type: 'text', text: 'ok' }] }), - }, - }, - }) - - const result = await orchestrator.executeAction({ - name: 'read_file', - input: '[]', - }) - - assert.strictEqual(result.success, false) - assert.strictEqual(result.status, 'input_invalid') - assert.ok(result.observation.includes('expected object')) - }) - - test('rejects validateInput success payload when shape is not object', async () => { - const orchestrator = createToolOrchestrator({ - tools: { - read_file: { - name: 'read_file', - validateInput: () => ({ ok: true, data: 'not-object' }), - execute: async () => ({ content: [{ type: 'text', text: 'ok' }] }), - }, - }, - }) - - const result = await orchestrator.executeAction({ - name: 'read_file', - input: { file_path: '/tmp/a' }, - }) - - assert.strictEqual(result.success, false) - assert.strictEqual(result.status, 'input_invalid') - assert.ok(result.observation.includes('expected object')) - }) -}) diff --git a/packages/tools/src/orchestrator/index.ts b/packages/tools/src/orchestrator/index.ts deleted file mode 100644 index 1d48239..0000000 --- a/packages/tools/src/orchestrator/index.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' -import { createApprovalManager } from '@memo/tools/approval' -import { getMaxToolResultChars } from '@memo/tools/runtime/tool_output_limits' -import type { - ToolAction, - ToolActionResult, - ToolActionErrorType, - ToolApprovalHooks, - ToolExecutionOptions, - ToolExecutionResult, - ToolOrchestrator, - ToolOrchestratorConfig, - OrchestratorTool, -} from './types' - -const MAX_TOOL_INPUT_STRING_CHARS = 100_000 - -function escapeXmlAttr(value: string) { - return value.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>') -} - -function estimateCallToolResultChars(result: CallToolResult) { - let total = 0 - for (const item of result.content ?? []) { - if (item.type === 'text') { - total += item.text.length - continue - } - try { - total += JSON.stringify(item).length - } catch { - total += 100 - } - } - return total -} - -function buildOversizeHintXml(toolName: string, actualChars: number, maxChars: number) { - return `Tool output too long, automatically omitted. Please narrow the scope or add limit parameters and try again.` -} - -function guardToolResultSize(toolName: string, result: CallToolResult): CallToolResult { - const maxChars = getMaxToolResultChars() - const actualChars = estimateCallToolResultChars(result) - if (actualChars <= maxChars) return result - return { - content: [ - { - type: 'text', - text: buildOversizeHintXml(toolName, actualChars, maxChars), - }, - ], - isError: false, - } -} - -function flattenCallToolResult(result: CallToolResult): string { - const texts = - result.content?.flatMap((item) => { - if (item.type === 'text') return [item.text] - return [] - }) ?? [] - return texts.join('\n') -} - -type ParseToolInputResult = { ok: true; data: Record } | { ok: false; error: string } - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function parseToolInput(tool: OrchestratorTool, rawInput: unknown): ParseToolInputResult { - let candidate: unknown = rawInput - if (typeof rawInput === 'string') { - if (rawInput.length > MAX_TOOL_INPUT_STRING_CHARS) { - return { - ok: false as const, - error: `${tool.name} invalid input: input string too large (max ${MAX_TOOL_INPUT_STRING_CHARS} chars)`, - } - } - const trimmed = rawInput.trim() - if (trimmed) { - try { - candidate = JSON.parse(trimmed) - } catch { - candidate = trimmed - } - } else { - candidate = {} - } - } - - if (!isRecord(candidate)) { - return { ok: false as const, error: `${tool.name} invalid input: expected object` } - } - - if (typeof tool.validateInput === 'function') { - const validated = tool.validateInput(candidate) - if (!validated.ok) return validated - if (!isRecord(validated.data)) { - return { ok: false as const, error: `${tool.name} invalid input: expected object` } - } - return { ok: true as const, data: validated.data } - } - - return { ok: true as const, data: candidate } -} - -function classifyExecutionError(err: unknown): ToolActionErrorType { - const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase() - if ( - message.includes('sandbox') || - message.includes('permission denied') || - message.includes('operation not permitted') || - message.includes('eacces') - ) { - return 'sandbox_denied' - } - return 'execution_failed' -} - -class ToolOrchestratorImpl implements ToolOrchestrator { - readonly approvalManager - - constructor(private config: ToolOrchestratorConfig) { - this.approvalManager = createApprovalManager(config.approval) - } - - async executeAction(action: ToolAction, options?: ToolApprovalHooks): Promise { - const startedAt = Date.now() - const actionId = action.id ?? `${action.name}:${startedAt}` - const check = this.approvalManager.check(action.name, action.input) - - if (check.needApproval) { - const request = { - toolName: check.toolName, - params: check.params, - fingerprint: check.fingerprint, - riskLevel: check.riskLevel, - reason: check.reason, - } - - await options?.onApprovalRequest?.(request) - - const decision = options?.requestApproval ? await options.requestApproval(request) : 'deny' - this.approvalManager.recordDecision(check.fingerprint, decision) - - await options?.onApprovalResponse?.({ - fingerprint: check.fingerprint, - decision, - }) - - if (decision === 'deny') { - return { - actionId, - tool: action.name, - status: 'approval_denied', - errorType: 'approval_denied', - success: false, - observation: `User denied tool execution: ${action.name}`, - durationMs: Date.now() - startedAt, - rejected: true, - } - } - } - - const tool = this.config.tools[action.name] - if (!tool) { - return { - actionId, - tool: action.name, - status: 'tool_not_found', - errorType: 'tool_not_found', - success: false, - observation: `Unknown tool: ${action.name}`, - durationMs: Date.now() - startedAt, - } - } - - try { - const parsedInput = parseToolInput(tool, action.input) - if (!parsedInput.ok) { - return { - actionId, - tool: action.name, - status: 'input_invalid', - errorType: 'input_invalid', - success: false, - observation: parsedInput.error, - durationMs: Date.now() - startedAt, - } - } - - const rawResult = await tool.execute(parsedInput.data) - const result = guardToolResultSize(action.name, rawResult) - return { - actionId, - tool: action.name, - status: 'success', - success: true, - observation: flattenCallToolResult(result) || '(no tool output)', - durationMs: Date.now() - startedAt, - } - } catch (err) { - const errorType = classifyExecutionError(err) - return { - actionId, - tool: action.name, - status: errorType, - errorType, - success: false, - observation: `Tool execution failed: ${(err as Error).message}`, - durationMs: Date.now() - startedAt, - } - } - } - - async executeActions(actions: ToolAction[], options: ToolExecutionOptions = {}): Promise { - const executionMode = options.executionMode ?? 'sequential' - const failurePolicy = options.failurePolicy ?? (options.stopOnRejection === false ? 'collect_all' : 'fail_fast') - - let results: ToolActionResult[] = [] - - if (executionMode === 'parallel') { - const parallelResults = await Promise.all(actions.map((action) => this.executeAction(action, options))) - if (failurePolicy === 'fail_fast') { - const firstRejected = parallelResults.findIndex((result) => result.rejected) - results = firstRejected >= 0 ? parallelResults.slice(0, firstRejected + 1) : parallelResults - } else { - results = parallelResults - } - } else { - for (const action of actions) { - const result = await this.executeAction(action, options) - results.push(result) - if (result.rejected && failurePolicy === 'fail_fast') { - break - } - } - } - - const hasRejection = results.some((result) => result.rejected) - const combinedObservation = results.map((result) => `[${result.tool}]: ${result.observation}`).join('\n\n') - - return { - results, - combinedObservation, - hasRejection, - executionMode, - failurePolicy, - } - } - - clearOnceApprovals(): void { - this.approvalManager.clearOnceApprovals() - } - - dispose(): void { - this.approvalManager.dispose() - } -} - -export function createToolOrchestrator(config: ToolOrchestratorConfig): ToolOrchestrator { - return new ToolOrchestratorImpl(config) -} - -export type { - ToolAction, - ToolActionResult, - ToolActionErrorType, - ToolActionStatus, - ToolApprovalHooks, - ToolExecutionOptions, - ToolExecutionResult, - ToolOrchestrator, - ToolOrchestratorConfig, - OrchestratorTool, - OrchestratorToolRegistry, -} from './types' diff --git a/packages/tools/src/orchestrator/types.ts b/packages/tools/src/orchestrator/types.ts deleted file mode 100644 index bcb65eb..0000000 --- a/packages/tools/src/orchestrator/types.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' -import type { ApprovalDecision, ApprovalManager, ApprovalManagerConfig, ApprovalRequest } from '@memo/tools/approval' - -export type ToolValidateResult = { ok: true; data: unknown } | { ok: false; error: string } - -export type OrchestratorTool = { - name: string - supportsParallelToolCalls?: boolean - isMutating?: boolean - validateInput?: (input: unknown) => ToolValidateResult - execute: (input: unknown) => Promise -} - -export type OrchestratorToolRegistry = Record - -export type ToolAction = { - id?: string - name: string - input: unknown -} - -export type ToolActionErrorType = - | 'approval_denied' - | 'policy_denied' - | 'sandbox_denied' - | 'tool_not_found' - | 'input_invalid' - | 'execution_failed' - -export type ToolActionStatus = 'success' | ToolActionErrorType -export type ToolExecutionMode = 'sequential' | 'parallel' -export type ToolFailurePolicy = 'fail_fast' | 'collect_all' - -export type ToolActionResult = { - actionId: string - tool: string - status: ToolActionStatus - errorType?: ToolActionErrorType - success: boolean - observation: string - durationMs: number - rejected?: boolean -} - -export type ToolExecutionResult = { - results: ToolActionResult[] - combinedObservation: string - hasRejection: boolean - executionMode: ToolExecutionMode - failurePolicy: ToolFailurePolicy -} - -export type ToolApprovalHooks = { - onApprovalRequest?: (request: ApprovalRequest) => Promise | void - onApprovalResponse?: (payload: { fingerprint: string; decision: ApprovalDecision }) => Promise | void - requestApproval?: (request: ApprovalRequest) => Promise -} - -export type ToolExecutionOptions = ToolApprovalHooks & { - executionMode?: ToolExecutionMode - failurePolicy?: ToolFailurePolicy - /** @deprecated use failurePolicy */ - stopOnRejection?: boolean -} - -export interface ToolOrchestrator { - readonly approvalManager: ApprovalManager - executeAction(action: ToolAction, options?: ToolApprovalHooks): Promise - executeActions(actions: ToolAction[], options?: ToolExecutionOptions): Promise - clearOnceApprovals(): void - dispose(): void -} - -export type ToolOrchestratorConfig = { - tools: OrchestratorToolRegistry - approval?: ApprovalManagerConfig -} diff --git a/packages/tools/src/router/index.test.ts b/packages/tools/src/router/index.test.ts deleted file mode 100644 index ff871d9..0000000 --- a/packages/tools/src/router/index.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import assert from 'node:assert' -import { afterEach, describe, expect, test, vi } from 'vitest' -import { McpToolRegistry } from './mcp' -import { ToolRouter, createToolRouter, type MCPServerConfig } from './index' - -function serverConfig(): Record { - return { - remote: { - type: 'streamable_http', - url: 'https://example.com/mcp', - }, - } -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -describe('tool router mcp oauth wiring', () => { - test('loadMcpServers forwards oauth settings to mcp registry', async () => { - const loadSpy = vi.spyOn(McpToolRegistry.prototype, 'loadServersWithOptions').mockResolvedValue(1) - const router = new ToolRouter() - const settings = { memoHome: '/tmp/memo', storeMode: 'file' as const, callbackPort: 4567 } - - const loaded = await router.loadMcpServers(serverConfig(), settings) - - assert.strictEqual(loaded, 1) - expect(loadSpy).toHaveBeenCalledWith(serverConfig(), settings) - }) - - test('createToolRouter passes mcpOAuthSettings when loading servers', async () => { - const loadSpy = vi.spyOn(McpToolRegistry.prototype, 'loadServersWithOptions').mockResolvedValue(1) - const settings = { - memoHome: '/tmp/memo-home', - storeMode: 'auto' as const, - callbackPort: 33333, - } - - await createToolRouter({ - mcpServers: serverConfig(), - mcpOAuthSettings: settings, - }) - - expect(loadSpy).toHaveBeenCalledWith(serverConfig(), settings) - }) - - test('createToolRouter skips load when no mcp servers are configured', async () => { - const loadSpy = vi.spyOn(McpToolRegistry.prototype, 'loadServersWithOptions').mockResolvedValue(0) - - await createToolRouter({}) - - expect(loadSpy).not.toHaveBeenCalled() - }) -}) - -describe('ToolRouter', () => { - test('registerNativeTool adds tool to registry', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'test_tool', - description: 'Test tool', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - expect(router.hasTool('test_tool')).toBe(true) - expect(router.getTool('test_tool')).toBeDefined() - }) - - test('registerNativeTools registers multiple tools', () => { - const router = new ToolRouter() - router.registerNativeTools([ - { - name: 'tool1', - description: 'Tool 1', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }, - { - name: 'tool2', - description: 'Tool 2', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }, - ]) - - expect(router.hasTool('tool1')).toBe(true) - expect(router.hasTool('tool2')).toBe(true) - }) - - test('getAllTools returns combined native and mcp tools', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'native_tool', - description: 'Native', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const tools = router.getAllTools() - expect(tools.length).toBeGreaterThan(0) - expect(tools.some((t) => t.name === 'native_tool')).toBe(true) - }) - - test('toRegistry returns merged registry', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const registry = router.toRegistry() - expect(registry.test).toBeDefined() - }) - - test('getToolCount returns counts', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const counts = router.getToolCount() - expect(counts.native).toBeGreaterThan(0) - expect(counts.total).toBe(counts.native + counts.mcp) - }) - - test('execute throws when tool not found', async () => { - const router = new ToolRouter() - await expect(router.execute('nonexistent', {})).rejects.toThrow("Tool 'nonexistent' not found") - }) - - test('execute runs tool successfully', async () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'echo', - description: 'Echo input', - source: 'native', - inputSchema: { type: 'object' }, - execute: async (input) => ({ - content: [{ type: 'text', text: JSON.stringify(input) }], - }), - }) - - const result = await router.execute('echo', { test: true }) - expect(result.content[0]).toEqual({ type: 'text', text: '{"test":true}' }) - }) - - test('generateToolDefinitions returns all tools', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'test_tool', - description: 'Test tool', - source: 'native', - inputSchema: { type: 'object', properties: { foo: { type: 'string' } } }, - execute: async () => ({ content: [] }), - }) - - const defs = router.generateToolDefinitions() - expect(defs.length).toBeGreaterThan(0) - const testDef = defs.find((d) => d.name === 'test_tool') - expect(testDef).toBeDefined() - expect(testDef?.description).toBe('Test tool') - }) - - test('generateToolDescriptions returns empty for no tools', () => { - const router = new ToolRouter() - expect(router.generateToolDescriptions()).toBe('') - }) - - test('generateToolDescriptions includes native tools', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'my_tool', - description: 'My tool description', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const desc = router.generateToolDescriptions() - expect(desc).toContain('## Available Tools') - expect(desc).toContain('### Built-in Tools') - expect(desc).toContain('my_tool') - expect(desc).toContain('My tool description') - }) - - test('getToolDescriptions returns structured data', () => { - const router = new ToolRouter() - router.registerNativeTool({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const descs = router.getToolDescriptions() - expect(descs.length).toBeGreaterThan(0) - expect(descs[0].name).toBe('test') - expect(descs[0].source).toBe('native') - }) - - test('createToolRouter registers native tools', async () => { - const router = await createToolRouter({ - nativeTools: [ - { - name: 'custom_tool', - description: 'Custom', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }, - ], - }) - - expect(router.hasTool('custom_tool')).toBe(true) - }) -}) diff --git a/packages/tools/src/router/index.ts b/packages/tools/src/router/index.ts deleted file mode 100644 index 4768bcc..0000000 --- a/packages/tools/src/router/index.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** @file ToolRouter - 统一工具路由管理 - * - * 职责： - * 1. 管理内置工具（NativeToolRegistry） - * 2. 管理外部 MCP 工具（McpToolRegistry） - * 3. 提供统一的工具查询和执行接口 - * 4. 生成工具描述（用于 Prompt） - */ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' -import type { Tool, ToolRegistry, MCPServerConfig, ToolDescription } from './types' -import { NativeToolRegistry } from './native' -import { McpToolRegistry } from './mcp' -import type { McpOAuthSettings } from './mcp/oauth' - -export type { Tool, ToolRegistry, MCPServerConfig, ToolDescription, NativeTool, McpTool } from './types' -export { NativeToolRegistry, McpToolRegistry } -export type { McpOAuthSettings } from './mcp/oauth' - -/** 工具路由管理器 */ -export class ToolRouter { - private nativeRegistry: NativeToolRegistry - private mcpRegistry: McpToolRegistry - - constructor() { - this.nativeRegistry = new NativeToolRegistry() - this.mcpRegistry = new McpToolRegistry() - } - - // ==================== 注册方法 ==================== - - /** 注册内置工具 */ - registerNativeTool(tool: Tool): void { - this.nativeRegistry.register(tool as import('./types').NativeTool) - } - - /** 批量注册内置工具 */ - registerNativeTools(tools: Tool[]): void { - for (const tool of tools) { - this.registerNativeTool(tool) - } - } - - /** 连接并加载 MCP Servers */ - async loadMcpServers( - servers: Record | undefined, - oauthSettings?: McpOAuthSettings, - ): Promise { - return this.mcpRegistry.loadServersWithOptions(servers, oauthSettings) - } - - // ==================== 查询方法 ==================== - - /** 获取指定工具（优先 native，然后 mcp） */ - getTool(name: string): Tool | undefined { - return this.nativeRegistry.get(name) ?? this.mcpRegistry.get(name) - } - - /** 获取所有工具 */ - getAllTools(): Tool[] { - return [...this.nativeRegistry.getAll(), ...this.mcpRegistry.getAll()] - } - - /** 获取工具注册表格式 */ - toRegistry(): ToolRegistry { - return { - ...this.nativeRegistry.toRegistry(), - ...this.mcpRegistry.toRegistry(), - } - } - - /** 检查工具是否存在 */ - hasTool(name: string): boolean { - return this.nativeRegistry.has(name) || this.mcpRegistry.has(name) - } - - /** 获取工具总数 */ - getToolCount(): { native: number; mcp: number; total: number } { - const native = this.nativeRegistry.size - const mcp = this.mcpRegistry.size - return { native, mcp, total: native + mcp } - } - - // ==================== 执行方法 ==================== - - /** - * 执行指定工具 - * @param name - 工具名称 - * @param input - 工具输入参数 - * @returns 工具执行结果 - * @throws 如果工具不存在 - */ - async execute(name: string, input: unknown): Promise { - const tool = this.getTool(name) - if (!tool) { - throw new Error(`Tool '${name}' not found`) - } - return tool.execute(input) - } - - // ==================== Prompt 生成 ==================== - - /** - * 生成 Tool Use API 格式的工具定义列表 - * @returns 工具定义数组，用于传递给 LLM API - */ - generateToolDefinitions(): Array<{ - name: string - description: string - input_schema: Record - }> { - return this.getAllTools().map((tool) => ({ - name: tool.name, - description: tool.description, - input_schema: tool.inputSchema || { type: 'object', properties: {} }, - })) - } - - /** - * 生成工具描述文本，用于注入到系统 Prompt - * @returns 格式化的工具描述 - */ - generateToolDescriptions(): string { - const tools = this.getAllTools() - if (tools.length === 0) { - return '' - } - - const lines: string[] = [] - lines.push('## Available Tools') - lines.push('') - - // 分组：内置工具和 MCP 工具 - const nativeTools = tools.filter((t) => t.source === 'native') - const mcpTools = tools.filter((t) => t.source === 'mcp') - - // 内置工具 - if (nativeTools.length > 0) { - lines.push('### Built-in Tools') - lines.push('') - for (const tool of nativeTools) { - lines.push(this.formatToolDescription(tool)) - } - lines.push('') - } - - // MCP tools - if (mcpTools.length > 0) { - lines.push('### External MCP Tools') - lines.push('') - - // Group by server - const grouped = this.groupByServer(mcpTools) - for (const [serverName, serverTools] of Object.entries(grouped)) { - lines.push(`**Server: ${serverName}**`) - lines.push('') - for (const tool of serverTools) { - lines.push(this.formatToolDescription(tool)) - } - lines.push('') - } - } - - return lines.join('\n') - } - - /** Format single tool description */ - private formatToolDescription(tool: Tool): string { - const lines: string[] = [] - lines.push(`#### ${tool.name}`) - lines.push(`- **Description**: ${tool.description}`) - - if (tool.inputSchema && Object.keys(tool.inputSchema).length > 0) { - lines.push(`- **Input Schema**: ${JSON.stringify(tool.inputSchema)}`) - } - - return lines.join('\n') - } - - /** Group MCP tools by server */ - private groupByServer(tools: Tool[]): Record { - const grouped: Record = {} - for (const tool of tools) { - if (tool.source === 'mcp') { - const serverName = (tool as import('./types').McpTool).serverName - if (!grouped[serverName]) { - grouped[serverName] = [] - } - grouped[serverName].push(tool) - } - } - return grouped - } - - /** - * 获取工具描述列表（结构化数据） - */ - getToolDescriptions(): ToolDescription[] { - return this.getAllTools().map((tool) => ({ - name: tool.name, - description: tool.description, - source: tool.source, - serverName: tool.source === 'mcp' ? (tool as import('./types').McpTool).serverName : undefined, - inputSchema: tool.inputSchema, - })) - } - - // ==================== 生命周期 ==================== - - /** 清理资源（关闭 MCP 连接等） */ - async dispose(): Promise { - await this.mcpRegistry.dispose() - } -} - -/** 创建并初始化 ToolRouter（便捷函数） */ -export async function createToolRouter(options: { - nativeTools?: Tool[] - mcpServers?: Record - mcpOAuthSettings?: McpOAuthSettings -}): Promise { - const router = new ToolRouter() - - // 注册内置工具 - if (options.nativeTools && options.nativeTools.length > 0) { - router.registerNativeTools(options.nativeTools) - } - - // 加载 MCP Servers - if (options.mcpServers && Object.keys(options.mcpServers).length > 0) { - await router.loadMcpServers(options.mcpServers, options.mcpOAuthSettings) - } - - return router -} diff --git a/packages/tools/src/router/native/index.test.ts b/packages/tools/src/router/native/index.test.ts deleted file mode 100644 index df35a3d..0000000 --- a/packages/tools/src/router/native/index.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { NativeToolRegistry } from './index' - -describe('NativeToolRegistry', () => { - test('register adds tool', () => { - const registry = new NativeToolRegistry() - registry.register({ - name: 'test_tool', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - expect(registry.has('test_tool')).toBe(true) - expect(registry.get('test_tool')).toBeDefined() - }) - - test('registerMany adds multiple tools', () => { - const registry = new NativeToolRegistry() - registry.registerMany([ - { - name: 'tool1', - description: '1', - source: 'native', - inputSchema: {}, - execute: async () => ({ content: [] }), - }, - { - name: 'tool2', - description: '2', - source: 'native', - inputSchema: {}, - execute: async () => ({ content: [] }), - }, - ]) - - expect(registry.has('tool1')).toBe(true) - expect(registry.has('tool2')).toBe(true) - }) - - test('getAll returns all tools', () => { - const registry = new NativeToolRegistry() - registry.register({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - expect(registry.getAll().length).toBe(1) - }) - - test('toRegistry returns correct format', () => { - const registry = new NativeToolRegistry() - registry.register({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - const reg = registry.toRegistry() - expect(reg.test).toBeDefined() - }) - - test('size returns correct count', () => { - const registry = new NativeToolRegistry() - expect(registry.size).toBe(0) - - registry.register({ - name: 'test', - description: 'Test', - source: 'native', - inputSchema: { type: 'object' }, - execute: async () => ({ content: [] }), - }) - - expect(registry.size).toBe(1) - }) -}) diff --git a/packages/tools/src/router/native/index.ts b/packages/tools/src/router/native/index.ts deleted file mode 100644 index dfec1ee..0000000 --- a/packages/tools/src/router/native/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** @file Built-in tool registry */ -import type { NativeTool, ToolRegistry } from '../types' - -/** Built-in tool registry */ -export class NativeToolRegistry { - private tools: Map = new Map() - - /** Register a single tool */ - register(tool: NativeTool): void { - this.tools.set(tool.name, tool) - } - - /** Register multiple tools in batch */ - registerMany(tools: NativeTool[]): void { - for (const tool of tools) { - this.register(tool) - } - } - - /** Get tool */ - get(name: string): NativeTool | undefined { - return this.tools.get(name) - } - - /** Get all tools */ - getAll(): NativeTool[] { - return Array.from(this.tools.values()) - } - - /** Convert to ToolRegistry format */ - toRegistry(): ToolRegistry { - const registry: ToolRegistry = {} - for (const [name, tool] of this.tools) { - registry[name] = tool - } - return registry - } - - /** Check if tool exists */ - has(name: string): boolean { - return this.tools.has(name) - } - - /** Get tool count */ - get size(): number { - return this.tools.size - } -} diff --git a/packages/tools/src/router/types.ts b/packages/tools/src/router/types.ts deleted file mode 100644 index 95e4a08..0000000 --- a/packages/tools/src/router/types.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** @file ToolRouter unified type definitions */ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' - -/** Tool source type */ -export type ToolSource = 'native' | 'mcp' - -/** JSON Schema basic type */ -export interface JSONSchema { - type?: string - properties?: Record - required?: string[] - description?: string - [key: string]: unknown -} - -/** Unified tool interface */ -export interface Tool { - /** Unique tool name (MCP tools have serverName_ prefix) */ - name: string - /** Tool description */ - description: string - /** Tool source */ - source: ToolSource - /** JSON Schema for input parameters */ - inputSchema: JSONSchema - /** Whether parallel calls are supported (default false, conservative serial). */ - supportsParallelToolCalls?: boolean - /** Whether it modifies external state (files, processes, network writes, etc.). */ - isMutating?: boolean - /** Optional input validator (usually provided by native/zod adapter layer) */ - validateInput?: (input: unknown) => { ok: true; data: unknown } | { ok: false; error: string } - /** Execute tool */ - execute: (input: unknown) => Promise -} - -/** Built-in tool */ -export interface NativeTool extends Tool { - source: 'native' -} - -/** MCP external tool */ -export interface McpTool extends Tool { - source: 'mcp' - /** Source server name */ - serverName: string - /** Original tool name on server side */ - originalName: string -} - -/** Tool registry */ -export type ToolRegistry = Record - -/** Tool description (for Prompt generation) */ -export interface ToolDescription { - name: string - description: string - source: ToolSource - serverName?: string - inputSchema: JSONSchema -} - -/** MCP Server configuration (reuses definition from config.ts) */ -export type MCPServerConfig = - | { - type?: 'stdio' - command: string - args?: string[] - env?: Record - /** Subprocess stderr behavior (silent in TTY by default). */ - stderr?: 'inherit' | 'pipe' | 'ignore' - } - | { - type?: 'streamable_http' - url: string - headers?: Record - http_headers?: Record - bearer_token_env_var?: string - } - -/** MCP Client connection info */ -export interface McpClientConnection { - name: string - client: import('@modelcontextprotocol/sdk/client/index.js').Client - transport: - | import('@modelcontextprotocol/sdk/client/stdio.js').StdioClientTransport - | import('@modelcontextprotocol/sdk/client/streamableHttp.js').StreamableHTTPClientTransport - tools: McpTool[] -} diff --git a/packages/tools/src/tools/mcp.test.ts b/packages/tools/src/tools/mcp.test.ts deleted file mode 100644 index be6d36a..0000000 --- a/packages/tools/src/tools/mcp.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import assert from 'node:assert' -import { describe, test } from 'vitest' -import { textResult, flattenText } from './mcp' - -describe('mcp helpers', () => { - describe('textResult', () => { - test('creates successful text result', () => { - const result = textResult('hello world') - assert.deepStrictEqual(result.content, [{ type: 'text', text: 'hello world' }]) - assert.strictEqual(result.isError, false) - }) - - test('creates error text result', () => { - const result = textResult('error message', true) - assert.deepStrictEqual(result.content, [{ type: 'text', text: 'error message' }]) - assert.strictEqual(result.isError, true) - }) - - test('handles empty string', () => { - const result = textResult('') - assert.deepStrictEqual(result.content, [{ type: 'text', text: '' }]) - assert.strictEqual(result.isError, false) - }) - - test('handles unicode content', () => { - const result = textResult('你好世界 🌍 Привет') - assert.strictEqual(result.content[0].text, '你好世界 🌍 Привет') - }) - - test('handles multi-line content', () => { - const result = textResult('line1\nline2\nline3') - assert.strictEqual(result.content[0].text, 'line1\nline2\nline3') - }) - - test('handles special characters', () => { - const result = textResult('\n') - assert.strictEqual(result.content[0].text, '\n') - }) - - test('handles very long content', () => { - const longContent = 'x'.repeat(100000) - const result = textResult(longContent) - assert.strictEqual(result.content[0].text.length, 100000) - }) - - test('handles JSON-like content', () => { - const result = textResult('{"key": "value", "nested": {"a": 1}}') - assert.ok(result.content[0].text.includes('"key"')) - }) - }) - - describe('flattenText', () => { - test('extracts text from single content item', () => { - const result = textResult('single line') - assert.strictEqual(flattenText(result), 'single line') - }) - - test('joins multiple text content items', () => { - const result: Parameters[0] = { - content: [ - { type: 'text', text: 'line1' }, - { type: 'text', text: 'line2' }, - ], - isError: false, - } - assert.strictEqual(flattenText(result), 'line1\nline2') - }) - - test('ignores non-text content', () => { - const result: Parameters[0] = { - content: [ - { type: 'text', text: 'visible' }, - { type: 'image', data: 'base64data' }, - { type: 'text', text: 'also visible' }, - ], - isError: false, - } - assert.strictEqual(flattenText(result), 'visible\nalso visible') - }) - - test('handles empty result', () => { - const result: Parameters[0] = { content: [], isError: false } - assert.strictEqual(flattenText(result), '') - }) - - test('handles undefined content', () => { - const result: Parameters[0] = { content: undefined, isError: false } - assert.strictEqual(flattenText(result), '') - }) - - test('handles content with only non-text items', () => { - const result: Parameters[0] = { - content: [ - { type: 'image', data: 'base64' }, - { type: 'resource', resource: { uri: 'file:///test' } }, - ], - isError: false, - } - assert.strictEqual(flattenText(result), '') - }) - - test('handles mixed empty and non-empty text', () => { - const result: Parameters[0] = { - content: [ - { type: 'text', text: 'first' }, - { type: 'text', text: '' }, - { type: 'text', text: 'last' }, - ], - isError: false, - } - assert.strictEqual(flattenText(result), 'first\n\nlast') - }) - - test('preserves exact text including whitespace', () => { - const result: Parameters[0] = { - content: [ - { type: 'text', text: ' leading spaces' }, - { type: 'text', text: 'trailing spaces ' }, - { type: 'text', text: '\ttab\t' }, - ], - isError: false, - } - const output = flattenText(result) - assert.ok(output.includes(' leading spaces')) - assert.ok(output.includes('trailing spaces ')) - assert.ok(output.includes('\ttab\t')) - }) - - test('handles isError flag correctly', () => { - const errorResult = textResult('error message', true) - assert.strictEqual(errorResult.isError, true) - - const successResult = textResult('success message', false) - assert.strictEqual(successResult.isError, false) - }) - - test('handles many content items', () => { - const content = Array(100) - .fill(null) - .map((_, i) => ({ type: 'text' as const, text: `line${i}` })) - const result: Parameters[0] = { content, isError: false } - const output = flattenText(result) - assert.ok(output.includes('line0')) - assert.ok(output.includes('line99')) - assert.strictEqual(output.split('\n').length, 100) - }) - }) -}) diff --git a/packages/tools/src/tools/mcp.ts b/packages/tools/src/tools/mcp.ts deleted file mode 100644 index 1e816a5..0000000 --- a/packages/tools/src/tools/mcp.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' - -/** Quick constructor for text-based CallToolResult. */ -export function textResult(text: string, isError = false): CallToolResult { - return { content: [{ type: 'text', text }], isError } -} - -/** Flatten CallToolResult text content to string for observation. */ -export function flattenText(result: CallToolResult): string { - const texts = - result.content?.flatMap((item) => { - if (item.type === 'text') return [item.text] - return [] - }) ?? [] - return texts.join('\n') -} diff --git a/packages/tools/src/tools/types.ts b/packages/tools/src/tools/types.ts deleted file mode 100644 index d7e9ed4..0000000 --- a/packages/tools/src/tools/types.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { CallToolResult } from '@modelcontextprotocol/sdk/types' -import type { NativeTool } from '@memo/tools/router/types' -import type { ZodTypeAny } from 'zod' - -// Tool-related type declarations - -/** Tool name enum, used as tool field in Agent action. */ -/** Tool name (string), supports built-in and dynamically extended tools. */ -export type ToolName = string - -/** Unified tool definition (consistent with router layer Tool format). */ -export type McpTool = NativeTool - -/** Define tool based on zod schema, outputs unified MCP/Router tool format. */ -export function defineMcpTool(tool: { - name: ToolName - description: string - inputSchema: ZodTypeAny - supportsParallelToolCalls?: boolean - isMutating?: boolean - execute: (input: Input) => Promise -}): McpTool { - const { inputSchema, execute, ...rest } = tool - const jsonSchema = (inputSchema as any).toJSONSchema?.() - const { $schema: _$schema, ...inputSchemaJson } = - (jsonSchema as Record & { $schema?: string }) ?? {} - - return { - ...rest, - source: 'native', - inputSchema: inputSchemaJson, - validateInput: (input: unknown) => { - const parsed = inputSchema.safeParse(input) - if (!parsed.success) { - const detail = parsed.error.issues[0]?.message ?? 'invalid input' - return { ok: false, error: `${tool.name} invalid input: ${detail}` } - } - return { ok: true, data: parsed.data } - }, - execute: execute as (input: unknown) => Promise, - } -} diff --git a/packages/tui/package.json b/packages/tui/package.json index 9f68c8e..f37d27f 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -10,7 +10,6 @@ "dependencies": { "@inkjs/ui": "^2.0.0", "@memo-code/core": "workspace:*", - "@memo-code/tools": "workspace:*", "ignore": "^7.0.5", "ink": "^6.7.0", "marked": "^17.0.1", diff --git a/packages/tui/src/app/App.tsx b/packages/tui/src/app/App.tsx index 02721ac..fc95425 100644 --- a/packages/tui/src/app/App.tsx +++ b/packages/tui/src/app/App.tsx @@ -8,6 +8,7 @@ import { resolveContextWindowForProvider, selectProvider, writeMemoConfig, + type ApprovalDecision, type AgentSession, type AgentSessionDeps, type AgentSessionOptions, @@ -16,25 +17,30 @@ import { type ModelProfileOverride, type ProviderConfig, } from '@memo/core' -import { useApproval } from './hooks/useApproval' +import { ApprovalQueue } from './approvalQueue' +import { + createInitialRuntimeState, + pendingRuntimeApproval, + runtimeReducer, + runtimeStatus, + type TurnRequest, +} from './runtimeState' +import { VisibleUpdateQueue, type VisibleUpdate } from './visibleUpdateQueue' import { ChatWidget } from '../features/timeline/ChatWidget' import { Composer } from '../features/composer/Composer' import { Footer } from '../shared/ui/Footer' import { ApprovalOverlay } from '../features/approval/ApprovalOverlay' import { McpActivationOverlay } from '../features/mcp/McpActivationOverlay' import { notifyApprovalRequested } from '../features/approval/approvalNotification' +import { PlanPanel } from '../features/plan/PlanPanel' +import { planStateReducer } from '../features/plan/planState' import { SetupWizard } from '../features/setup/SetupWizard' import { parseHistoryLog } from '../features/session/historyParser' -import { - chatTimelineReducer, - createInitialTimelineState, - type ChatTimelineAction, -} from '../features/timeline/chatTimeline' +import { chatTimelineReducer, createInitialTimelineState } from '../features/timeline/chatTimeline' import { calculateContextPercent, inferParallelToolStatuses, inferToolStatus } from '../shared/lib/utils' import { checkForUpdate, findLocalPackageInfoSync } from '../shared/lib/version' import type { SessionHistoryEntry } from '../features/session/sessionHistory' import { loadTaskPrompt } from '../shared/lib/taskPrompt' -import { resolveReviewBackend } from '../features/review/backend' import { formatSlashCommand, PLAIN_EXIT_COMMAND, @@ -114,12 +120,19 @@ export function App({ sessionOptions.toolPermissionMode ?? (dangerous ? TOOL_PERMISSION_MODES.FULL : TOOL_PERMISSION_MODES.ONCE) const [timeline, dispatchTimeline] = useReducer(chatTimelineReducer, undefined, createInitialTimelineState) + const [runtime, dispatchRuntime] = useReducer(runtimeReducer, undefined, createInitialRuntimeState) + const [activePlan, dispatchPlan] = useReducer(planStateReducer, null) const [currentProvider, setCurrentProvider] = useState(providerName) const [currentModel, setCurrentModel] = useState(model) const [providersState, setProvidersState] = useState(providers) const [modelProfilesState, setModelProfilesState] = useState(modelProfiles) const [toolPermissionMode, setToolPermissionMode] = useState(defaultToolPermissionMode) + const [thinkingOn, setThinkingOn] = useState(() => { + const override = modelProfilesState?.[model] ?? modelProfilesState?.[`${providerName}:${model}`] + return override?.supports_reasoning_content ?? true + }) + const thinkingOnRef = useRef(thinkingOn) const resolveContextLimit = useCallback( (providerConfig: Pick) => @@ -130,55 +143,139 @@ export function App({ const [sessionOptionsState, setSessionOptionsState] = useState({ ...sessionOptions, providerName, + modelName: model, contextWindow: resolveContextLimit({ name: providerName, model }), dangerous: defaultToolPermissionMode === TOOL_PERMISSION_MODES.FULL, toolPermissionMode: defaultToolPermissionMode, + thinking: thinkingOn, }) const [inputHistory, setInputHistory] = useState([]) const [contextLimit, setContextLimit] = useState(resolveContextLimit({ name: providerName, model })) const [currentContextTokens, setCurrentContextTokens] = useState(0) + const [followOutput, setFollowOutput] = useState(true) const [setupPending, setSetupPending] = useState(needsSetup) const [mcpSelectionPending, setMcpSelectionPending] = useState(!needsSetup && availableMcpServerNames.length > 0) const [activeMcpServerNames, setActiveMcpServerNames] = useState(initialActiveMcpServers) const [exitMessage, setExitMessage] = useState(null) - const [busy, setBusy] = useState(false) const [sessionLogPath, setSessionLogPath] = useState(null) const [pendingHistoryMessages, setPendingHistoryMessages] = useState(null) const [session, setSession] = useState(null) const sessionRef = useRef(null) const currentTurnRef = useRef(null) const nextUserInputOverrideRef = useRef(null) + const startedOperationRef = useRef(null) + const followOutputRef = useRef(true) + + const applyVisibleUpdate = useCallback((update: VisibleUpdate) => { + if (update.kind === 'timeline') { + dispatchTimeline(update.action) + } else if (update.kind === 'plan') { + dispatchPlan(update.action) + } else { + setCurrentContextTokens(update.promptTokens) + } + }, []) + const visibleUpdateQueueRef = useRef(null) + if (!visibleUpdateQueueRef.current) { + visibleUpdateQueueRef.current = new VisibleUpdateQueue(applyVisibleUpdate) + } + const visibleUpdateQueue = visibleUpdateQueueRef.current + + const setOutputFollowing = useCallback( + (following: boolean) => { + followOutputRef.current = following + visibleUpdateQueue.setFollowing(following) + setFollowOutput(following) + }, + [visibleUpdateQueue], + ) + + const resetVisibleOutput = useCallback(() => { + visibleUpdateQueue.clear() + followOutputRef.current = true + visibleUpdateQueue.setFollowing(true) + setFollowOutput(true) + }, [visibleUpdateQueue]) + + const operationStatus = runtimeStatus(runtime) + const pendingApproval = pendingRuntimeApproval(runtime) + const approvalQueueRef = useRef(null) + if (!approvalQueueRef.current) { + approvalQueueRef.current = new ApprovalQueue((request) => { + dispatchRuntime(request ? { type: 'approval_requested', request } : { type: 'approval_resolved' }) + }) + } + const approvalQueue = approvalQueueRef.current + const handleApprovalDecision = useCallback( + (decision: ApprovalDecision) => { + approvalQueue.decide(decision) + }, + [approvalQueue], + ) + + const handleToggleThinking = useCallback(() => { + setThinkingOn((prev) => { + const next = !prev + thinkingOnRef.current = next + sessionRef.current?.setThinking?.(next) + return next + }) + }, []) - const { pendingApproval, setPendingApproval, approvalResolverRef, handleApprovalDecision } = useApproval() + const handleToggleFollowOutput = useCallback(() => { + if (followOutputRef.current && runtime.active?.kind !== 'turn') return + setOutputFollowing(!followOutputRef.current) + }, [runtime.active, setOutputFollowing]) const localPackageInfo = useMemo(() => findLocalPackageInfoSync(), []) - const dispatch = useCallback((action: ChatTimelineAction) => { - dispatchTimeline(action) + const restoreSessionUiState = useCallback((parsed: ParsedHistoryLog) => { + const restoredToolPermissionMode = + parsed.toolPermissionMode === 'none' || + parsed.toolPermissionMode === 'once' || + parsed.toolPermissionMode === 'full' + ? parsed.toolPermissionMode + : undefined + if (parsed.providerName) setCurrentProvider(parsed.providerName) + if (parsed.modelName) setCurrentModel(parsed.modelName) + if (typeof parsed.thinking === 'boolean') { + thinkingOnRef.current = parsed.thinking + setThinkingOn(parsed.thinking) + } + if (parsed.contextWindow) setContextLimit(parsed.contextWindow) + if (restoredToolPermissionMode) setToolPermissionMode(restoredToolPermissionMode) + setInputHistory(parsed.turns.map((turn) => turn.userInput.trim()).filter(Boolean)) + setSessionOptionsState((prev) => ({ + ...prev, + providerName: parsed.providerName ?? prev.providerName, + modelName: parsed.modelName ?? prev.modelName, + contextWindow: parsed.contextWindow ?? prev.contextWindow, + toolPermissionMode: restoredToolPermissionMode ?? prev.toolPermissionMode, + dangerous: + restoredToolPermissionMode === undefined + ? prev.dangerous + : restoredToolPermissionMode === TOOL_PERMISSION_MODES.FULL, + thinking: typeof parsed.thinking === 'boolean' ? parsed.thinking : prev.thinking, + })) }, []) useEffect(() => { if (!initialHistory) return - dispatch({ type: 'clear_current_timeline' }) - dispatch({ + resetVisibleOutput() + dispatchTimeline({ type: 'clear_current_timeline' }) + dispatchTimeline({ type: 'replace_history', turns: initialHistory.turns, maxSequence: initialHistory.maxSequence, }) + dispatchPlan({ type: 'restore_history', turns: initialHistory.turns }) setPendingHistoryMessages(initialHistory.messages) - if (initialHistory.summary.trim()) { - dispatch({ - type: 'append_system_message', - title: 'History', - content: initialHistory.summary, - tone: 'info', - }) - } - }, [dispatch, initialHistory]) + restoreSessionUiState(initialHistory) + }, [dispatchTimeline, initialHistory, resetVisibleOutput, restoreSessionUiState]) useEffect(() => { if (setupPending) return @@ -188,9 +285,9 @@ export function App({ const appendSystemMessage = useCallback( (title: string, content: string, tone: 'info' | 'warning' | 'error' = 'info') => { - dispatch({ type: 'append_system_message', title, content, tone }) + dispatchTimeline({ type: 'append_system_message', title, content, tone }) }, - [dispatch], + [dispatchTimeline], ) const deps = useMemo( @@ -198,17 +295,26 @@ export function App({ onAssistantStep: (chunk: string, step: number) => { const turn = currentTurnRef.current if (!turn) return - dispatch({ type: 'assistant_chunk', turn, step, chunk }) + visibleUpdateQueue.enqueue({ + kind: 'timeline', + action: { type: 'assistant_chunk', turn, step, chunk }, + }) + }, + onReasoningChunk: (chunk: string, step: number) => { + const turn = currentTurnRef.current + if (!turn) return + visibleUpdateQueue.enqueue({ + kind: 'timeline', + action: { type: 'reasoning_chunk', turn, step, chunk }, + }) }, requestApproval: toolPermissionMode === TOOL_PERMISSION_MODES.FULL || toolPermissionMode === TOOL_PERMISSION_MODES.NONE ? undefined - : (request) => - new Promise((resolve) => { - void notifyApprovalRequested(request) - setPendingApproval(request) - approvalResolverRef.current = resolve - }), + : (request) => { + void notifyApprovalRequested(request) + return approvalQueue.request(request) + }, hooks: { onTurnStart: ({ turn, input, promptTokens }) => { currentTurnRef.current = turn @@ -218,88 +324,116 @@ export function App({ } const displayInput = override ?? input + const updates: VisibleUpdate[] = [] if (promptTokens && promptTokens > 0) { - setCurrentContextTokens(promptTokens) + updates.push({ kind: 'context', promptTokens }) } - - dispatch({ - type: 'turn_start', - turn, - input: displayInput, - promptTokens, + updates.push({ + kind: 'timeline', + action: { + type: 'turn_start', + turn, + input: displayInput, + promptTokens, + }, }) + visibleUpdateQueue.enqueueMany(updates) }, onContextUsage: ({ turn, step, promptTokens, phase }) => { - setCurrentContextTokens(promptTokens) - dispatch({ - type: 'context_usage', - turn, - step, - promptTokens, - phase, - }) + visibleUpdateQueue.enqueueMany([ + { kind: 'context', promptTokens }, + { + kind: 'timeline', + action: { + type: 'context_usage', + turn, + step, + promptTokens, + phase, + }, + }, + ]) }, onContextCompacted: ({ reason, status, beforeTokens, afterTokens, reductionPercent, errorMessage }) => { - if (status === 'success') { - setCurrentContextTokens(afterTokens) - } const compactedBy = reason === 'manual' ? 'manual command' : 'auto trigger' + let content: string + let tone: 'info' | 'warning' = 'info' if (status === 'success') { - appendSystemMessage( - 'Context compacted', - `Compacted by ${compactedBy}: ${beforeTokens} -> ${afterTokens} tokens (${reductionPercent.toFixed(2)}% reduced).`, - ) - return - } - if (status === 'skipped') { - appendSystemMessage( - 'Context compacted', - `Skipped (${compactedBy}): nothing to compact.`, - 'warning', - ) - return + content = `Compacted by ${compactedBy}: ${beforeTokens} -> ${afterTokens} tokens (${reductionPercent.toFixed(2)}% reduced).` + } else if (status === 'skipped') { + content = `Skipped (${compactedBy}): nothing to compact.` + tone = 'warning' + } else { + content = `Failed (${compactedBy}): ${errorMessage ?? 'unknown error'}` + tone = 'warning' } - appendSystemMessage( - 'Context compacted', - `Failed (${compactedBy}): ${errorMessage ?? 'unknown error'}`, - 'warning', - ) + const updates: VisibleUpdate[] = [] + if (status === 'success') updates.push({ kind: 'context', promptTokens: afterTokens }) + updates.push({ + kind: 'timeline', + action: { type: 'append_system_message', title: 'Context compacted', content, tone }, + }) + visibleUpdateQueue.enqueueMany(updates) }, onAction: ({ turn, step, action, thinking, parallelActions }) => { - dispatch({ - type: 'tool_action', - turn, - step, - action, - thinking, - parallelActions, + visibleUpdateQueue.enqueue({ + kind: 'timeline', + action: { + type: 'tool_action', + turn, + step, + action, + thinking, + parallelActions, + }, }) }, - onObservation: ({ turn, step, observation, resultStatus, parallelResultStatuses }) => { - dispatch({ - type: 'tool_observation', - turn, - step, - observation, - toolStatus: inferToolStatus(resultStatus), - parallelToolStatuses: inferParallelToolStatuses(parallelResultStatuses), - }) + onObservation: ({ turn, step, observation, resultStatus, parallelResultStatuses, results }) => { + const toolResults = results.map((result) => ({ + toolCallId: result.toolCallId, + tool: result.tool, + observation: result.observation, + status: inferToolStatus(result.status), + })) + visibleUpdateQueue.enqueueMany([ + { + kind: 'timeline', + action: { + type: 'tool_observation', + turn, + step, + observation, + toolStatus: inferToolStatus(resultStatus), + parallelToolStatuses: inferParallelToolStatuses(parallelResultStatuses), + toolResults, + }, + }, + ...toolResults.map( + (result): VisibleUpdate => ({ + kind: 'plan', + action: { type: 'tool_result', result }, + }), + ), + ]) }, - onFinal: ({ turn, finalText, status, errorMessage, turnUsage, tokenUsage }) => { - dispatch({ - type: 'turn_final', - turn, - finalText, - status, - errorMessage, - turnUsage, - tokenUsage, + onFinal: ({ turn, finalText, status, errorMessage, turnUsage, tokenUsage, thinking }) => { + visibleUpdateQueue.enqueue({ + kind: 'timeline', + action: { + type: 'turn_final', + turn, + finalText, + status, + errorMessage, + turnUsage, + tokenUsage, + thinking, + }, }) - setBusy(false) }, }, }), - [appendSystemMessage, dispatch, toolPermissionMode], + [approvalQueue, toolPermissionMode, visibleUpdateQueue], ) useEffect(() => { @@ -313,7 +447,10 @@ export function App({ await previous.close() } - const created = await createAgentSession(deps, sessionOptionsState) + const created = await createAgentSession(deps, { + ...sessionOptionsState, + thinking: thinkingOnRef.current, + }) if (cancelled) { await created.close() return @@ -327,7 +464,8 @@ export function App({ sessionRef.current = null setSession(null) setSessionLogPath(null) - setBusy(false) + resetVisibleOutput() + dispatchRuntime({ type: 'reset' }) appendSystemMessage('Session', `Failed to create session: ${(err as Error).message}`, 'error') } })() @@ -335,8 +473,9 @@ export function App({ return () => { cancelled = true } - }, [appendSystemMessage, deps, mcpSelectionPending, sessionOptionsState, setupPending]) + }, [appendSystemMessage, deps, mcpSelectionPending, resetVisibleOutput, sessionOptionsState, setupPending]) + useEffect(() => () => visibleUpdateQueue.dispose(), [visibleUpdateQueue]) useEffect(() => { return () => { if (sessionRef.current) { @@ -362,50 +501,55 @@ export function App({ }, [appendSystemMessage]) const handleExit = useCallback(async () => { - const resolver = approvalResolverRef.current - if (resolver) { - resolver('deny') - approvalResolverRef.current = null - } - if (pendingApproval) { - setPendingApproval(null) + approvalQueue.denyAll() + if (runtime.active?.kind === 'turn') { + dispatchRuntime({ type: 'cancel_requested' }) } if (sessionRef.current) { await sessionRef.current.close() } setExitMessage('Bye!') - setTimeout(() => exit(), 250) - }, [exit, pendingApproval]) + }, [approvalQueue, runtime.active]) - const handleClear = useCallback(() => { - if (busy) { - appendSystemMessage('Clear', 'Cancel current run before clearing timeline.', 'warning') - return - } - if (pendingApproval) { - appendSystemMessage('Clear', 'Resolve current approval request before clearing timeline.', 'warning') - return + // Render the farewell message first, then unmount. + useEffect(() => { + if (exitMessage) { + exit() } - dispatch({ type: 'clear_current_timeline' }) + }, [exit, exitMessage]) + + const guardActiveOperation = useCallback( + (action: string): boolean => { + if (!runtime.active) return false + const message = + operationStatus === 'awaiting_approval' + ? 'Resolve the current approval request before proceeding.' + : operationStatus === 'compacting' + ? 'Wait for context compaction to finish before proceeding.' + : operationStatus === 'cancelling' + ? 'Wait for cancellation to finish before proceeding.' + : 'Cancel the current run before proceeding.' + appendSystemMessage(action, message, 'warning') + return true + }, + [appendSystemMessage, operationStatus, runtime.active], + ) + + const handleClear = useCallback(() => { + if (guardActiveOperation('Clear')) return + resetVisibleOutput() + dispatchTimeline({ type: 'clear_current_timeline' }) setPendingHistoryMessages(null) setCurrentContextTokens(0) clearTerminalScreen() - }, [appendSystemMessage, busy, dispatch, pendingApproval]) + }, [dispatchTimeline, guardActiveOperation, resetVisibleOutput]) const handleNewSession = useCallback(() => { - if (busy) { - appendSystemMessage('New Session', 'Cancel current run before starting a new session.', 'warning') - return - } - if (pendingApproval) { - appendSystemMessage( - 'New Session', - 'Resolve current approval request before starting a new session.', - 'warning', - ) - return - } - dispatch({ type: 'reset_all' }) + if (guardActiveOperation('New Session')) return + resetVisibleOutput() + dispatchTimeline({ type: 'reset_all' }) + dispatchRuntime({ type: 'reset' }) + dispatchPlan({ type: 'clear' }) setPendingHistoryMessages(null) setCurrentContextTokens(0) currentTurnRef.current = null @@ -414,7 +558,7 @@ export function App({ sessionId: randomUUID(), })) appendSystemMessage('New Session', 'Started a fresh session.') - }, [appendSystemMessage, busy, dispatch, pendingApproval]) + }, [appendSystemMessage, dispatchTimeline, guardActiveOperation, resetVisibleOutput]) const persistCurrentProvider = useCallback( async (name: string) => { @@ -433,17 +577,17 @@ export function App({ const handleModelSelect = useCallback( async (provider: ProviderConfig) => { - if (busy) { - appendSystemMessage('Model switch', 'Cancel current run before switching models.', 'warning') - return - } + if (guardActiveOperation('Model switch')) return if (provider.name === currentProvider && provider.model === currentModel) { appendSystemMessage('Model switch', `Already using ${provider.name} (${provider.model}).`) return } - dispatch({ type: 'reset_all' }) + resetVisibleOutput() + dispatchTimeline({ type: 'reset_all' }) + dispatchRuntime({ type: 'reset' }) + dispatchPlan({ type: 'clear' }) setCurrentContextTokens(0) currentTurnRef.current = null @@ -455,6 +599,7 @@ export function App({ ...prev, sessionId: randomUUID(), providerName: provider.name, + modelName: provider.model, contextWindow: nextContextLimit, })) @@ -463,11 +608,12 @@ export function App({ }, [ appendSystemMessage, - busy, currentModel, currentProvider, - dispatch, + dispatchTimeline, + guardActiveOperation, persistCurrentProvider, + resetVisibleOutput, resolveContextLimit, ], ) @@ -480,35 +626,39 @@ export function App({ const handleSetToolPermission = useCallback( (mode: ToolPermissionMode) => { - if (busy) { - appendSystemMessage('Tools', 'Cancel current run before changing tool permission mode.', 'warning') - return - } - - if (pendingApproval) { - appendSystemMessage( - 'Tools', - 'Resolve current approval request before changing tool permission mode.', - 'warning', - ) - return - } + if (guardActiveOperation('Tools')) return if (mode === toolPermissionMode) { appendSystemMessage('Tools', `Already using ${toolPermissionLabel(mode)}.`) return } + // Tool permission is baked into the session at creation time, so + // switching modes recreates the session; reset the visible timeline + // to match the fresh session's (empty) history. setToolPermissionMode(mode) + resetVisibleOutput() + dispatchTimeline({ type: 'reset_all' }) + dispatchRuntime({ type: 'reset' }) + dispatchPlan({ type: 'clear' }) + setCurrentContextTokens(0) + currentTurnRef.current = null setSessionOptionsState((prev) => ({ ...prev, sessionId: randomUUID(), dangerous: mode === TOOL_PERMISSION_MODES.FULL, toolPermissionMode: mode, })) - appendSystemMessage('Tools', `Tool permission set to ${toolPermissionLabel(mode)}.`) + appendSystemMessage('Tools', `Tool permission set to ${toolPermissionLabel(mode)}. Conversation reset.`) }, - [appendSystemMessage, busy, pendingApproval, toolPermissionLabel, toolPermissionMode], + [ + appendSystemMessage, + dispatchTimeline, + guardActiveOperation, + resetVisibleOutput, + toolPermissionLabel, + toolPermissionMode, + ], ) const persistActiveMcpServers = useCallback( @@ -546,35 +696,26 @@ export function App({ const handleHistorySelect = useCallback( async (entry: SessionHistoryEntry) => { - if (busy) { - appendSystemMessage('History', 'Cancel current run before loading session history.', 'warning') - return - } - if (pendingApproval) { - appendSystemMessage( - 'History', - 'Resolve current approval request before loading session history.', - 'warning', - ) - return - } + if (guardActiveOperation('History')) return try { const raw = await readFile(entry.sessionFile, 'utf8') const parsed = parseHistoryLog(raw) - dispatch({ type: 'clear_current_timeline' }) - dispatch({ + resetVisibleOutput() + dispatchTimeline({ type: 'clear_current_timeline' }) + dispatchTimeline({ type: 'replace_history', turns: parsed.turns, maxSequence: parsed.maxSequence, }) + dispatchPlan({ type: 'restore_history', turns: parsed.turns }) setPendingHistoryMessages(parsed.messages) - setBusy(false) + dispatchRuntime({ type: 'reset' }) setSession(null) setSessionLogPath(null) setCurrentContextTokens(0) currentTurnRef.current = null + restoreSessionUiState(parsed) setSessionOptionsState((prev) => ({ ...prev, sessionId: randomUUID() })) - appendSystemMessage('History', parsed.summary || entry.input) } catch (err) { appendSystemMessage( 'History', @@ -583,99 +724,57 @@ export function App({ ) } }, - [appendSystemMessage, busy, dispatch, pendingApproval], + [appendSystemMessage, dispatchTimeline, guardActiveOperation, resetVisibleOutput, restoreSessionUiState], ) const handleCancelRun = useCallback(() => { - if (!busy) return + if (runtime.active?.kind !== 'turn') return + approvalQueue.denyAll() + dispatchRuntime({ type: 'cancel_requested' }) session?.cancelCurrentTurn?.() - }, [busy, session]) + }, [approvalQueue, runtime.active, session]) - const runCompactCommand = useCallback(async () => { - if (busy) { - appendSystemMessage('Compact', 'Cancel current run before compacting context.', 'warning') - return - } - if (pendingApproval) { - appendSystemMessage('Compact', 'Resolve current approval request before compacting context.', 'warning') - return - } + const runCompactCommand = useCallback(() => { if (!session) return - - try { - const result = await session.compactHistory('manual') - setCurrentContextTokens(result.afterTokens) - } catch (err) { - appendSystemMessage('Compact', `Failed to compact context: ${(err as Error).message}`, 'error') + if (runtime.active) { + const message = + runtime.active.kind === 'compact' + ? 'Context compaction is already running.' + : 'Compact is unavailable while a turn is running.' + appendSystemMessage('Compact', message, 'warning') + return } - }, [appendSystemMessage, busy, pendingApproval, session]) + dispatchRuntime({ type: 'start_compact' }) + }, [appendSystemMessage, runtime.active, session]) const runInitCommand = useCallback(async () => { - if (!session || busy) return + if (!session) return + if (runtime.active) { + appendSystemMessage('Init', 'Init is unavailable while a turn is running.', 'warning') + return + } const initCommand = formatSlashCommand(SLASH_COMMANDS.INIT) + const targetSessionId = session.id try { const prompt = await loadTaskPrompt('init_agents') + if (sessionRef.current?.id !== targetSessionId) { + appendSystemMessage('Init', 'Session changed before the init task could start.', 'warning') + return + } setInputHistory((prev) => [...prev, initCommand]) - setBusy(true) - nextUserInputOverrideRef.current = initCommand - await session.runTurn(prompt) + dispatchRuntime({ + type: 'submit_turn', + request: { + id: randomUUID(), + input: prompt, + displayInput: initCommand, + }, + }) } catch (err) { - setBusy(false) appendSystemMessage('Init', `Failed to run init task: ${(err as Error).message}`, 'error') } - }, [appendSystemMessage, busy, session]) - - const runReviewPullRequestCommand = useCallback( - async (prNumber: number) => { - if (!session || busy) return - - if (toolPermissionMode === TOOL_PERMISSION_MODES.NONE) { - appendSystemMessage( - 'Review', - 'Tool permission mode is "none". Set `/tools once` or `/tools full` before running `/review`.', - 'warning', - ) - return - } - - const reviewCommand = `${formatSlashCommand(SLASH_COMMANDS.REVIEW)} ${prNumber}` - try { - const backend = await resolveReviewBackend({ - cwd, - mcpServers, - activeMcpServerNames, - availableToolNames: session.listToolNames?.() ?? [], - }) - - if (backend.kind === 'unavailable') { - appendSystemMessage('Review', backend.reason, 'error') - return - } - - const prompt = await loadTaskPrompt('review_pull_request', { - pr_number: String(prNumber), - backend_strategy: backend.strategy, - backend_details: backend.details, - mcp_server_prefix: backend.kind === 'github_mcp' ? backend.mcpServerPrefix : 'github', - }) - - setInputHistory((prev) => [...prev, reviewCommand]) - appendSystemMessage('Review', backend.details) - setBusy(true) - nextUserInputOverrideRef.current = reviewCommand - await session.runTurn(prompt) - } catch (err) { - setBusy(false) - appendSystemMessage( - 'Review', - `Failed to run review task for PR #${prNumber}: ${(err as Error).message}`, - 'error', - ) - } - }, - [activeMcpServerNames, appendSystemMessage, busy, cwd, mcpServers, session, toolPermissionMode], - ) + }, [appendSystemMessage, runtime.active, session]) const handleSubmit = useCallback( async (value: string) => { @@ -687,25 +786,57 @@ export function App({ return } + if (!followOutputRef.current) { + setOutputFollowing(true) + } + if (trimmed === formatSlashCommand(SLASH_COMMANDS.INIT)) { await runInitCommand() return } - if (!session || busy) return + if (!session) return setInputHistory((prev) => [...prev, trimmed]) - setBusy(true) - try { - await session.runTurn(trimmed) - } catch (err) { - setBusy(false) - appendSystemMessage('Run', `Turn failed: ${(err as Error).message}`, 'error') + const request: TurnRequest = { + id: randomUUID(), + input: trimmed, + displayInput: trimmed, } + dispatchRuntime({ type: 'submit_turn', request }) }, - [appendSystemMessage, busy, handleExit, runInitCommand, session], + [handleExit, runInitCommand, session, setOutputFollowing], ) + useEffect(() => { + const active = runtime.active + if (!active || !session || startedOperationRef.current === active.id) return + startedOperationRef.current = active.id + + void (async () => { + try { + if (active.kind === 'compact') { + const result = await session.compactHistory('manual') + setCurrentContextTokens(result.afterTokens) + return + } + + const override = + active.request.displayInput !== active.request.input ? active.request.displayInput : null + nextUserInputOverrideRef.current = override + await session.runTurn(active.request.input) + } catch (err) { + const title = active.kind === 'compact' ? 'Compact' : 'Run' + appendSystemMessage(title, `${title} failed: ${(err as Error).message}`, 'error') + } finally { + if (active.kind === 'turn' && nextUserInputOverrideRef.current === active.request.displayInput) { + nextUserInputOverrideRef.current = null + } + dispatchRuntime({ type: 'operation_finished', operationId: active.id }) + } + })() + }, [appendSystemMessage, runtime.active, session]) + const handleSetupComplete = useCallback(async () => { try { const loaded = await loadMemoConfig() @@ -720,6 +851,7 @@ export function App({ ...prev, sessionId: randomUUID(), providerName: provider.name, + modelName: provider.model, contextWindow: nextContextLimit, autoCompactThresholdPercent: loaded.config.auto_compact_threshold_percent, })) @@ -767,7 +899,7 @@ export function App({ } if (setupPending) { - return + return } if (mcpSelectionPending) { @@ -776,9 +908,6 @@ export function App({ serverNames={availableMcpServerNames} defaultSelected={initialActiveMcpServers} onConfirm={handleConfirmMcpActivation} - onExit={() => { - void handleExit() - }} /> ) } @@ -792,9 +921,11 @@ export function App({ historicalTurns={timeline.historicalTurns} /> + {activePlan ? : null} + { void runCompactCommand() }} + onToggleThinking={handleToggleThinking} + onToggleFollowOutput={handleToggleFollowOutput} onHistorySelect={(entry) => { void handleHistorySelect(entry) }} @@ -824,15 +957,19 @@ export function App({ void handleModelSelect(provider) }} onSetToolPermission={handleSetToolPermission} - onReviewPullRequest={(prNumber) => { - void runReviewPullRequestCommand(prNumber) - }} + thinkingOn={thinkingOn} onSystemMessage={appendSystemMessage} /> {pendingApproval ? : null} -