From 541ac2b4cee9e124142c58b281b3093cebaa6b37 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Sun, 26 Jul 2026 16:41:33 -0700 Subject: [PATCH] =?UTF-8?q?refactor(core):=20tail=20migrations=20=E2=80=94?= =?UTF-8?q?=20codebuff,=20openclaw=20(phase=208)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/src/providers/codebuff.ts | 365 ++---------------- packages/cli/src/providers/openclaw.ts | 226 ++--------- .../chat-messages.json | 98 +++++ .../agents/anon/sessions/anon-session.jsonl | 4 + .../agents/myagent/sessions/test-sess-1.jsonl | 6 + .../tests/providers/codebuff-bridge.test.ts | 139 +++++++ .../tests/providers/openclaw-bridge.test.ts | 152 ++++++++ packages/core/package.json | 8 + .../core/src/providers/codebuff/decode.ts | 246 ++++++++++++ packages/core/src/providers/codebuff/index.ts | 30 ++ .../src/providers/codebuff/observations.ts | 86 +++++ packages/core/src/providers/codebuff/types.ts | 98 +++++ .../core/src/providers/openclaw/decode.ts | 182 +++++++++ packages/core/src/providers/openclaw/index.ts | 27 ++ .../src/providers/openclaw/observations.ts | 86 +++++ packages/core/src/providers/openclaw/types.ts | 62 +++ packages/core/tests/architecture-gate.test.ts | 4 + packages/core/tests/content-smuggling.test.ts | 110 ++++++ .../tests/providers/codebuff-decode.test.ts | 141 +++++++ .../tests/providers/openclaw-decode.test.ts | 98 +++++ packages/core/tsup.config.ts | 2 + 21 files changed, 1659 insertions(+), 511 deletions(-) create mode 100644 packages/cli/tests/fixtures/codebuff-parity/manicode/projects/parity-proj/chats/2026-04-14T10-00-00.000Z/chat-messages.json create mode 100644 packages/cli/tests/fixtures/openclaw-parity-noid/.openclaw/agents/anon/sessions/anon-session.jsonl create mode 100644 packages/cli/tests/fixtures/openclaw-parity/.openclaw/agents/myagent/sessions/test-sess-1.jsonl create mode 100644 packages/cli/tests/providers/codebuff-bridge.test.ts create mode 100644 packages/cli/tests/providers/openclaw-bridge.test.ts create mode 100644 packages/core/src/providers/codebuff/decode.ts create mode 100644 packages/core/src/providers/codebuff/index.ts create mode 100644 packages/core/src/providers/codebuff/observations.ts create mode 100644 packages/core/src/providers/codebuff/types.ts create mode 100644 packages/core/src/providers/openclaw/decode.ts create mode 100644 packages/core/src/providers/openclaw/index.ts create mode 100644 packages/core/src/providers/openclaw/observations.ts create mode 100644 packages/core/src/providers/openclaw/types.ts create mode 100644 packages/core/tests/providers/codebuff-decode.test.ts create mode 100644 packages/core/tests/providers/openclaw-decode.test.ts diff --git a/packages/cli/src/providers/codebuff.ts b/packages/cli/src/providers/codebuff.ts index d75a9dee..c1b4c2c0 100644 --- a/packages/cli/src/providers/codebuff.ts +++ b/packages/cli/src/providers/codebuff.ts @@ -1,9 +1,12 @@ import { readdir, readFile, stat } from 'fs/promises' -import { basename, dirname, join } from 'path' +import { basename, join } from 'path' import { homedir } from 'os' +import { decodeCodebuff, codebuffToolNameMap } from '@codeburn/core/providers/codebuff' +import type { CodebuffDecodedCall } from '@codeburn/core/providers/codebuff' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' // Codebuff (formerly Manicode) uses a credit-based billing system. The local // chat-messages.json doesn't record per-call token counts the way Claude Code @@ -27,101 +30,6 @@ const modelDisplayNames: Record = { 'codebuff-max': 'Codebuff Max', } -// Codebuff's native tool names map to codeburn's canonical tool set so -// classifier heuristics (edit/read/bash/etc.) behave consistently with the -// other providers. -const toolNameMap: Record = { - read_files: 'Read', - read_file: 'Read', - code_search: 'Grep', - glob: 'Glob', - find_files: 'Glob', - str_replace: 'Edit', - edit_file: 'Edit', - write_file: 'Write', - run_terminal_command: 'Bash', - terminal: 'Bash', - spawn_agents: 'Agent', - spawn_agent: 'Agent', - write_todos: 'TodoWrite', - create_plan: 'TodoWrite', - browser_logs: 'WebFetch', - web_search: 'WebSearch', - fetch_url: 'WebFetch', -} - -// Tool names we ignore for classification -- they're not useful signals for -// distinguishing "coding" vs "exploration" vs "planning" work. -const IGNORED_TOOLS = new Set(['suggest_followups', 'end_turn']) - -type CodebuffUsage = { - inputTokens?: number - input_tokens?: number - promptTokens?: number - prompt_tokens?: number - outputTokens?: number - output_tokens?: number - completionTokens?: number - completion_tokens?: number - cacheCreationInputTokens?: number - cache_creation_input_tokens?: number - cacheReadInputTokens?: number - cache_read_input_tokens?: number - promptTokensDetails?: { cachedTokens?: number } - prompt_tokens_details?: { cached_tokens?: number } -} - -type CodebuffBlock = { - type?: string - content?: string - toolName?: string - input?: Record - output?: string - agentName?: string - agentType?: string - status?: string - blocks?: CodebuffBlock[] -} - -type CodebuffHistoryMessage = { - role?: string - providerOptions?: { - codebuff?: { model?: string; usage?: CodebuffUsage } - usage?: CodebuffUsage - } -} - -type CodebuffMetadata = { - model?: string - modelId?: string - timestamp?: string | number - usage?: CodebuffUsage - codebuff?: { model?: string; usage?: CodebuffUsage } - runState?: { - cwd?: string - sessionState?: { - cwd?: string - projectContext?: { cwd?: string } - fileContext?: { cwd?: string } - mainAgentState?: { - agentType?: string - messageHistory?: CodebuffHistoryMessage[] - } - } - } -} - -type CodebuffChatMessage = { - id?: string - variant?: string - role?: string - content?: string - timestamp?: string | number - credits?: number - blocks?: CodebuffBlock[] - metadata?: CodebuffMetadata -} - function getCodebuffBaseDir(override?: string): string { if (override && override.trim()) return override const envPath = process.env['CODEBUFF_DATA_DIR'] @@ -129,116 +37,6 @@ function getCodebuffBaseDir(override?: string): string { return join(homedir(), '.config', 'manicode') } -function pickNumber(...vals: Array): number | undefined { - for (const v of vals) { - if (typeof v === 'number' && Number.isFinite(v)) return v - } - return undefined -} - -function normalizeUsage(u: CodebuffUsage | undefined): { - input: number - output: number - cacheRead: number - cacheWrite: number -} { - if (!u) return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } - return { - input: pickNumber(u.inputTokens, u.input_tokens, u.promptTokens, u.prompt_tokens) ?? 0, - output: pickNumber(u.outputTokens, u.output_tokens, u.completionTokens, u.completion_tokens) ?? 0, - cacheRead: - pickNumber( - u.cacheReadInputTokens, - u.cache_read_input_tokens, - u.promptTokensDetails?.cachedTokens, - u.prompt_tokens_details?.cached_tokens, - ) ?? 0, - cacheWrite: pickNumber(u.cacheCreationInputTokens, u.cache_creation_input_tokens) ?? 0, - } -} - -function coerceTimestamp(value: string | number | undefined): string { - if (value == null) return '' - if (typeof value === 'number') { - return Number.isFinite(value) ? new Date(value).toISOString() : '' - } - const parsed = Date.parse(value) - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : value -} - -function parseChatIdToIso(chatId: string): string { - const iso = chatId.replace(/(\d{4}-\d{2}-\d{2}T\d{2})-(\d{2})-(\d{2})/, '$1:$2:$3') - const parsed = Date.parse(iso) - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : '' -} - -function extractCwd(meta: CodebuffMetadata | undefined): string | null { - const rs = meta?.runState - if (!rs) return null - return ( - rs.sessionState?.projectContext?.cwd ?? - rs.sessionState?.fileContext?.cwd ?? - rs.sessionState?.cwd ?? - rs.cwd ?? - null - ) -} - -function extractAgentType(meta: CodebuffMetadata | undefined): string | null { - return meta?.runState?.sessionState?.mainAgentState?.agentType ?? null -} - -function collectBlockTools(blocks: CodebuffBlock[] | undefined, acc: { tools: string[]; bash: string[] }): void { - if (!Array.isArray(blocks)) return - for (const block of blocks) { - if (!block || typeof block !== 'object') continue - if (block.type === 'tool' && typeof block.toolName === 'string') { - const raw = block.toolName - if (!IGNORED_TOOLS.has(raw)) { - acc.tools.push(toolNameMap[raw] ?? raw) - } - if ((raw === 'run_terminal_command' || raw === 'terminal') && block.input) { - const cmd = block.input['command'] - if (typeof cmd === 'string') { - acc.bash.push(...extractBashCommands(cmd)) - } - } - } - if (block.type === 'agent' && Array.isArray(block.blocks)) { - collectBlockTools(block.blocks, acc) - } - } -} - -function resolveModel(meta: CodebuffMetadata | undefined, stashedModel: string | null): string { - const direct = meta?.model ?? meta?.modelId ?? meta?.codebuff?.model - if (direct) return direct - if (stashedModel) return stashedModel - const agentType = extractAgentType(meta) - if (agentType) return `codebuff-${agentType}` - return 'codebuff' -} - -function usageFromHistory(meta: CodebuffMetadata | undefined): { - model: string | null - input: number - output: number - cacheRead: number - cacheWrite: number -} { - const hist = meta?.runState?.sessionState?.mainAgentState?.messageHistory - if (!Array.isArray(hist)) return { model: null, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } - for (let i = hist.length - 1; i >= 0; i--) { - const entry = hist[i] - if (!entry || entry.role !== 'assistant' || !entry.providerOptions) continue - const u = normalizeUsage(entry.providerOptions.usage ?? entry.providerOptions.codebuff?.usage) - if (u.input > 0 || u.output > 0 || u.cacheRead > 0 || u.cacheWrite > 0) { - return { model: entry.providerOptions.codebuff?.model ?? null, ...u } - } - } - return { model: null, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } -} - async function readJson(filePath: string): Promise { try { const raw = await readFile(filePath, 'utf-8') @@ -280,10 +78,15 @@ async function discoverChannel(root: string): Promise { // Resolve the real cwd from run-state.json so sessions group by the // originating project directory instead of the sanitized chat folder // name (which is often the same for many users). - const runState = await readJson( + const runState = await readJson<{ cwd?: string; sessionState?: { cwd?: string; projectContext?: { cwd?: string }; fileContext?: { cwd?: string } } }>( join(chatDir, 'run-state.json'), ) - const cwd = extractCwd({ runState: runState ?? undefined }) + const cwd = + runState?.sessionState?.projectContext?.cwd ?? + runState?.sessionState?.fileContext?.cwd ?? + runState?.sessionState?.cwd ?? + runState?.cwd ?? + null const project = cwd ? basename(cwd) : projectName sources.push({ path: chatDir, project, provider: 'codebuff' }) @@ -315,124 +118,33 @@ async function discoverSessionsInBase(baseDir: string): Promise return results } -// Downstream aggregation groups sessions by `(provider, sessionId, project)` -// (see src/parser.ts). Codebuff chat folders are ISO timestamps, which means -// the same `chatId` can legitimately appear under each channel root -// (`manicode`, `manicode-dev`, `manicode-staging`) and even resolve to the -// same project cwd. To keep those sessions distinct we include the channel -// identity in the sessionId. The channel is derived from the fixed path -// structure Codebuff writes on disk: `/projects//chats/`. -// Returns null when the path doesn't match that shape so the caller can fall -// back to a plain chatId. -// -// We use '/' as the channel/chatId separator rather than ':' because -// src/parser.ts builds its session key as `${provider}:${sessionId}:${project}` -// and reconstructs the sessionId with `key.split(':')[1]` -- any colon inside -// sessionId would get truncated to just the channel name downstream. -function extractChannelFromChatDir(chatDir: string): string | null { - const chatsDir = dirname(chatDir) - if (basename(chatsDir) !== 'chats') return null - const projectDir = dirname(chatsDir) - const projectsDir = dirname(projectDir) - if (basename(projectsDir) !== 'projects') return null - const channel = basename(dirname(projectsDir)) - return channel ? channel : null -} - -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +function toProviderCall(rich: CodebuffDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - const chatDir = source.path - const chatId = basename(chatDir) - const channel = extractChannelFromChatDir(chatDir) - const sessionId = channel ? `${channel}/${chatId}` : chatId - const fallbackTs = parseChatIdToIso(chatId) - - const messages = await readJson( - join(chatDir, 'chat-messages.json'), - ) - if (!Array.isArray(messages)) return - - let pendingUserMessage = '' - - for (const [idx, msg] of messages.entries()) { - if (!msg || typeof msg !== 'object') continue - - const variant = msg.variant ?? msg.role - if (variant === 'user') { - if (typeof msg.content === 'string' && msg.content.length > 0) { - pendingUserMessage = msg.content - } - continue - } - - if (variant !== 'ai' && variant !== 'agent' && variant !== 'assistant') continue - - const credits = typeof msg.credits === 'number' && Number.isFinite(msg.credits) ? msg.credits : 0 - const directUsage = normalizeUsage(msg.metadata?.usage ?? msg.metadata?.codebuff?.usage) - const stashedUsage = usageFromHistory(msg.metadata) - - const hasDirect = - directUsage.input > 0 || - directUsage.output > 0 || - directUsage.cacheRead > 0 || - directUsage.cacheWrite > 0 - const usage = hasDirect ? directUsage : stashedUsage - const stashedModel = stashedUsage.model - - // Skip messages with neither credits nor tokens -- they're typically - // in-progress mode dividers or empty framing blocks. - if (credits === 0 && usage.input === 0 && usage.output === 0 && usage.cacheRead === 0 && usage.cacheWrite === 0) { - continue - } - - const model = resolveModel(msg.metadata, stashedModel) - const timestamp = coerceTimestamp(msg.timestamp ?? msg.metadata?.timestamp) || fallbackTs - - const dedupId = msg.id ?? String(idx) - const dedupKey = `codebuff:${chatDir}:${dedupId}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const acc = { tools: [] as string[], bash: [] as string[] } - collectBlockTools(msg.blocks, acc) - - // Prefer calculated cost from tokens when available (multi-provider - // models routed through Codebuff still show up in LiteLLM); otherwise - // fall back to the credit-based approximation. The pricing pass applies - // fallbackCostUSD only when the table price is 0 (unknown model), - // reproducing that precedence. - yield { - provider: 'codebuff', - model, - inputTokens: usage.input, - outputTokens: usage.output, - cacheCreationInputTokens: usage.cacheWrite, - cacheReadInputTokens: usage.cacheRead, - cachedInputTokens: usage.cacheRead, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - ...(credits > 0 ? { fallbackCostUSD: credits * USD_PER_CREDIT } : {}), - tools: acc.tools, - bashCommands: acc.bash, - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: pendingUserMessage, - sessionId, - } - - pendingUserMessage = '' - } - }, + provider: 'codebuff', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + ...(rich.credits > 0 ? { fallbackCostUSD: rich.credits * USD_PER_CREDIT } : {}), + tools: rich.tools, + bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)), + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } export function createCodebuffProvider(baseDir?: string): Provider { const dir = getCodebuffBaseDir(baseDir) - return { + return createBridgedProvider({ name: 'codebuff', displayName: 'Codebuff', @@ -441,17 +153,22 @@ export function createCodebuffProvider(baseDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return codebuffToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { return discoverSessionsInBase(dir) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + async readRecords(source: SessionSource): Promise { + const messages = await readJson(join(source.path, 'chat-messages.json')) + if (!Array.isArray(messages)) return null + return messages }, - } + + decode: decodeCodebuff, + toProviderCall, + }) } export const codebuff = createCodebuffProvider() diff --git a/packages/cli/src/providers/openclaw.ts b/packages/cli/src/providers/openclaw.ts index 4ec954b4..b0fc6e7e 100644 --- a/packages/cli/src/providers/openclaw.ts +++ b/packages/cli/src/providers/openclaw.ts @@ -1,57 +1,13 @@ import { readdir, readFile } from 'fs/promises' -import { basename, join } from 'path' +import { join } from 'path' import { homedir } from 'os' +import { decodeOpenClaw, openclawToolNameMap } from '@codeburn/core/providers/openclaw' +import type { OpenClawDecodedCall } from '@codeburn/core/providers/openclaw' import { readSessionFile } from '../fs-utils.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' - -const toolNameMap: Record = { - bash: 'Bash', - exec: 'Bash', - read: 'Read', - edit: 'Edit', - write: 'Write', - glob: 'Glob', - grep: 'Grep', - task: 'Agent', - dispatch_agent: 'Agent', - fetch: 'WebFetch', - search: 'WebSearch', - todo: 'TodoWrite', - patch: 'Patch', -} - -type OpenClawUsage = { - input: number - output: number - cacheRead: number - cacheWrite: number - totalTokens?: number - cost?: { - total?: number - } -} - -type OpenClawEntry = { - type: string - customType?: string - id?: string - timestamp?: string - provider?: string - modelId?: string - data?: { - provider?: string - modelId?: string - } - message?: { - role?: string - content?: Array<{ type?: string; text?: string; name?: string; arguments?: Record }> - model?: string - provider?: string - usage?: OpenClawUsage - } -} +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' type SessionIndex = Record }> | undefined): { tools: string[]; bashCommands: string[] } { - const tools: string[] = [] - const bashCommands: string[] = [] - if (!content) return { tools, bashCommands } - - for (const block of content) { - if ((block.type === 'tool_use' || block.type === 'toolCall') && block.name) { - const mapped = toolNameMap[block.name] ?? block.name - tools.push(mapped) - if (mapped === 'Bash' && block.arguments && typeof block.arguments.command === 'string') { - bashCommands.push(...extractBashCommands(block.arguments.command)) - } - } - } - return { tools, bashCommands } -} - -function createParser(source: SessionSource, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { - const raw = await readSessionFile(source.path) - if (raw === null) return - - const lines = raw.split('\n').filter(l => l.trim()) - let sessionId = '' - let sessionTimestamp = '' - let currentModel = '' - - const calls: { - model: string - usage: OpenClawUsage - tools: string[] - bashCommands: string[] - timestamp: string - userMessage: string - dedupId: string - }[] = [] - - let pendingUserMessage = '' - - for (const line of lines) { - let entry: OpenClawEntry - try { - entry = JSON.parse(line) - } catch { - continue - } - - if (entry.type === 'session') { - sessionId = entry.id ?? basename(source.path, '.jsonl') - sessionTimestamp = entry.timestamp ?? '' - continue - } - - if (entry.type === 'model_change') { - currentModel = entry.modelId ?? currentModel - continue - } - - if (entry.type === 'custom' && entry.customType === 'model-snapshot') { - currentModel = entry.data?.modelId ?? currentModel - continue - } - - if (entry.type !== 'message' || !entry.message) continue - - const msg = entry.message - if (msg.role === 'user') { - if (!pendingUserMessage && Array.isArray(msg.content)) { - const textBlock = msg.content.find(c => c.type === 'text' && c.text) - pendingUserMessage = (textBlock?.text ?? '').slice(0, 500) - } - continue - } - - if (msg.role !== 'assistant') continue - - const model = msg.model ?? currentModel - if (msg.usage) { - const { tools, bashCommands } = extractTools(msg.content) - calls.push({ - model, - usage: msg.usage, - tools, - bashCommands, - timestamp: entry.timestamp ?? sessionTimestamp, - userMessage: pendingUserMessage, - dedupId: entry.id ?? '', - }) - pendingUserMessage = '' - } - } - - if (!sessionId) sessionId = basename(source.path, '.jsonl') - - for (let i = 0; i < calls.length; i++) { - const call = calls[i] - const dedupKey = `openclaw:${sessionId}:${call.dedupId || i}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const u = call.usage - const costFromProvider = u.cost?.total ?? 0 - - const ts = new Date(call.timestamp) - if (isNaN(ts.getTime()) || ts.getTime() < 1_000_000_000_000) continue - - yield { - provider: 'openclaw', - model: call.model || 'openclaw-auto', - inputTokens: u.input, - outputTokens: u.output, - cacheCreationInputTokens: u.cacheWrite, - cacheReadInputTokens: u.cacheRead, - cachedInputTokens: u.cacheRead, - reasoningTokens: 0, - webSearchRequests: 0, - ...(costFromProvider > 0 - ? { costUSD: costFromProvider, costBasis: 'measured' as const } - : { costBasis: 'estimated' as const }), - tools: [...new Set(call.tools)], - bashCommands: [...new Set(call.bashCommands)], - timestamp: ts.toISOString(), - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: call.userMessage, - sessionId, - } - } - }, - } -} - async function discoverInDir(agentsDir: string): Promise { const sources: SessionSource[] = [] @@ -248,8 +71,32 @@ async function discoverInDir(agentsDir: string): Promise { return sources } -export function createOpenClawProvider(overrideDir?: string): Provider { +function toProviderCall(rich: OpenClawDecodedCall): ParsedProviderCall { return { + provider: 'openclaw', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + ...(rich.costBasis === 'measured' + ? { costUSD: rich.costUSD, costBasis: 'measured' as const } + : { costBasis: 'estimated' as const }), + tools: [...new Set(rich.tools)], + bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))], + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + } +} + +export function createOpenClawProvider(overrideDir?: string): Provider { + return createBridgedProvider({ name: 'openclaw', displayName: 'OpenClaw', @@ -258,7 +105,7 @@ export function createOpenClawProvider(overrideDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return openclawToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { @@ -271,10 +118,15 @@ export function createOpenClawProvider(overrideDir?: string): Provider { return all }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + async readRecords(source: SessionSource): Promise { + const raw = await readSessionFile(source.path) + if (raw === null) return null + return raw.split('\n').filter(l => l.trim()) }, - } + + decode: decodeOpenClaw, + toProviderCall, + }) } export const openclaw = createOpenClawProvider() diff --git a/packages/cli/tests/fixtures/codebuff-parity/manicode/projects/parity-proj/chats/2026-04-14T10-00-00.000Z/chat-messages.json b/packages/cli/tests/fixtures/codebuff-parity/manicode/projects/parity-proj/chats/2026-04-14T10-00-00.000Z/chat-messages.json new file mode 100644 index 00000000..34b2c71a --- /dev/null +++ b/packages/cli/tests/fixtures/codebuff-parity/manicode/projects/parity-proj/chats/2026-04-14T10-00-00.000Z/chat-messages.json @@ -0,0 +1,98 @@ +[ + { + "id": "msg-user-1", + "variant": "user", + "content": "implement the feature", + "timestamp": "2026-04-14T10:00:10.000Z" + }, + { + "id": "a1", + "variant": "ai", + "content": "", + "timestamp": "2026-04-14T10:00:30.000Z", + "credits": 42, + "metadata": { + "runState": { + "sessionState": { + "mainAgentState": { + "agentType": "base2" + } + } + } + }, + "blocks": [ + { "type": "tool", "toolName": "read_files", "input": {} }, + { "type": "tool", "toolName": "str_replace", "input": {} }, + { "type": "tool", "toolName": "run_terminal_command", "input": { "command": "npm test" } }, + { "type": "tool", "toolName": "suggest_followups", "input": {} }, + { "type": "tool", "toolName": "read_files", "input": {} }, + { "type": "tool", "toolName": "run_terminal_command", "input": { "command": "npm run build" } } + ] + }, + { + "id": "msg-user-2", + "variant": "user", + "content": "fix the bug", + "timestamp": "2026-04-14T10:01:00.000Z" + }, + { + "id": "a2", + "variant": "ai", + "content": "", + "timestamp": "2026-04-14T10:01:30.000Z", + "credits": 10, + "metadata": { + "model": "claude-haiku-4-5-20251001", + "usage": { + "inputTokens": 5000, + "outputTokens": 2000, + "cacheCreationInputTokens": 1000, + "cacheReadInputTokens": 500 + } + } + }, + { + "id": "a3", + "variant": "ai", + "content": "", + "timestamp": "2026-04-14T10:02:00.000Z", + "credits": 7, + "metadata": { + "runState": { + "sessionState": { + "mainAgentState": { + "messageHistory": [ + { "role": "user" }, + { + "role": "assistant", + "providerOptions": { + "codebuff": { + "model": "openai/gpt-4o", + "usage": { + "prompt_tokens": 2000, + "completion_tokens": 800, + "prompt_tokens_details": { "cached_tokens": 400 } + } + } + } + } + ] + } + } + } + } + }, + { + "id": "a1", + "variant": "ai", + "content": "", + "timestamp": "2026-04-14T10:02:30.000Z", + "credits": 5 + }, + { + "id": "a4", + "variant": "ai", + "content": "mode-divider", + "timestamp": "2026-04-14T10:03:00.000Z" + } +] diff --git a/packages/cli/tests/fixtures/openclaw-parity-noid/.openclaw/agents/anon/sessions/anon-session.jsonl b/packages/cli/tests/fixtures/openclaw-parity-noid/.openclaw/agents/anon/sessions/anon-session.jsonl new file mode 100644 index 00000000..fbf5498e --- /dev/null +++ b/packages/cli/tests/fixtures/openclaw-parity-noid/.openclaw/agents/anon/sessions/anon-session.jsonl @@ -0,0 +1,4 @@ +{"type":"session","timestamp":"2026-04-20T10:00:00.000Z"} +{"type":"model_change","modelId":"claude-x","timestamp":"2026-04-20T10:00:01.000Z"} +{"type":"message","id":"u1","timestamp":"2026-04-20T10:00:02.000Z","message":{"role":"user","content":[{"type":"text","text":"do work"}]}} +{"type":"message","id":"a1","timestamp":"2026-04-20T10:00:03.000Z","message":{"role":"assistant","model":"claude-x","content":[{"type":"toolCall","name":"exec","arguments":{"command":"echo hi"}}],"usage":{"input":5,"output":5,"cacheRead":0,"cacheWrite":0}}} diff --git a/packages/cli/tests/fixtures/openclaw-parity/.openclaw/agents/myagent/sessions/test-sess-1.jsonl b/packages/cli/tests/fixtures/openclaw-parity/.openclaw/agents/myagent/sessions/test-sess-1.jsonl new file mode 100644 index 00000000..25bd568a --- /dev/null +++ b/packages/cli/tests/fixtures/openclaw-parity/.openclaw/agents/myagent/sessions/test-sess-1.jsonl @@ -0,0 +1,6 @@ +{"type":"session","version":3,"id":"test-sess-1","timestamp":"2026-04-20T10:00:00.000Z","cwd":"/tmp"} +{"type":"model_change","id":"mc1","timestamp":"2026-04-20T10:00:01.000Z","provider":"anthropic","modelId":"claude-sonnet-4-6"} +{"type":"message","id":"u1","timestamp":"2026-04-20T10:00:02.000Z","message":{"role":"user","content":[{"type":"text","text":"hello world"}]}} +{"type":"message","id":"a1","timestamp":"2026-04-20T10:00:03.000Z","message":{"role":"assistant","model":"claude-sonnet-4-6","content":[{"type":"text","text":"Hi!"}],"usage":{"input":500,"output":100,"cacheRead":200,"cacheWrite":50,"totalTokens":850}}} +{"type":"message","id":"a2","timestamp":"2026-04-20T10:00:05.000Z","message":{"role":"assistant","model":"claude-sonnet-4-6","content":[{"type":"text","text":"Running command"},{"type":"toolCall","name":"exec","arguments":{"command":"ls -la"}},{"type":"tool_use","name":"write","arguments":{"path":"/tmp/y"}},{"type":"toolCall","name":"exec","arguments":{"command":"ls -la"}}],"usage":{"input":600,"output":200,"cacheRead":100,"cacheWrite":0,"totalTokens":900,"cost":{"total":0.05}}}} +{"type":"message","id":"a1","timestamp":"2026-04-20T10:00:06.000Z","message":{"role":"assistant","model":"claude-sonnet-4-6","content":[{"type":"text","text":"Duplicate"}],"usage":{"input":10,"output":10,"cacheRead":0,"cacheWrite":0,"totalTokens":20}}} diff --git a/packages/cli/tests/providers/codebuff-bridge.test.ts b/packages/cli/tests/providers/codebuff-bridge.test.ts new file mode 100644 index 00000000..bdfafa91 --- /dev/null +++ b/packages/cli/tests/providers/codebuff-bridge.test.ts @@ -0,0 +1,139 @@ +import { dirname, join, resolve } from 'path' +import { fileURLToPath } from 'url' + +import { describe, it, expect } from 'vitest' + +import { createCodebuffProvider } from '../../src/providers/codebuff.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the codebuff bridge migration. The GOLDEN below +// was captured from the legacy in-CLI decode before the migration; the bridged +// provider (discovery + I/O CLI-side, pure decode in @codeburn/core/providers/codebuff) +// must reproduce it exactly. Dedup keys contain absolute source paths, so they are +// computed from the discovered source at runtime rather than hard-coded. + +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/codebuff-parity/manicode') + +function expectedGolden(sourcePath: string): ParsedProviderCall[] { + const chatDir = sourcePath + return [ + { + provider: 'codebuff', + model: 'codebuff-base2', + inputTokens: 0, + outputTokens: 0, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + fallbackCostUSD: 0.42, + tools: ['Read', 'Edit', 'Bash', 'Read', 'Bash'], + bashCommands: ['npm', 'npm'], + timestamp: '2026-04-14T10:00:30.000Z', + speed: 'standard', + deduplicationKey: `codebuff:${chatDir}:a1`, + userMessage: 'implement the feature', + sessionId: 'manicode/2026-04-14T10-00-00.000Z', + }, + { + provider: 'codebuff', + model: 'claude-haiku-4-5-20251001', + inputTokens: 5000, + outputTokens: 2000, + cacheCreationInputTokens: 1000, + cacheReadInputTokens: 500, + cachedInputTokens: 500, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + fallbackCostUSD: 0.1, + tools: [], + bashCommands: [], + timestamp: '2026-04-14T10:01:30.000Z', + speed: 'standard', + deduplicationKey: `codebuff:${chatDir}:a2`, + userMessage: 'fix the bug', + sessionId: 'manicode/2026-04-14T10-00-00.000Z', + }, + { + provider: 'codebuff', + model: 'openai/gpt-4o', + inputTokens: 2000, + outputTokens: 800, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 400, + cachedInputTokens: 400, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + fallbackCostUSD: 0.07, + tools: [], + bashCommands: [], + timestamp: '2026-04-14T10:02:00.000Z', + speed: 'standard', + deduplicationKey: `codebuff:${chatDir}:a3`, + userMessage: '', + sessionId: 'manicode/2026-04-14T10-00-00.000Z', + }, + ] +} + +async function collect(): Promise<{ calls: ParsedProviderCall[]; sourcePath: string }> { + const provider = createCodebuffProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + expect(sources).toHaveLength(1) + const sourcePath = sources[0]!.path + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(sources[0]!, seen).parse()) { + calls.push(call) + } + return { calls, sourcePath } +} + +describe('codebuff bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const { calls, sourcePath } = await collect() + expect(calls).toEqual(expectedGolden(sourcePath)) + }) + + it('the priced output survives the pricing pass; credit fallback is used when table price is zero', async () => { + const { calls } = await collect() + const priced = calls.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + const raw = calls[i]! + expect(call.costBasis).toBe(raw.costBasis) + // Every field except the added costUSD must stay byte-identical. + const { costUSD, ...rest } = call + expect(rest).toEqual(raw) + }) + // The token-less codebuff-base2 call has no table price, so the credit + // fallback becomes the final cost. + expect(priced[0]!.costUSD).toBeCloseTo(0.42, 6) + // The Claude/GPT calls have known table prices that beat their credit fallback. + expect(priced[1]!.costUSD).not.toBeCloseTo(0.1, 6) + expect(priced[2]!.costUSD).not.toBeCloseTo(0.07, 6) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const provider = createCodebuffProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(3) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/openclaw-bridge.test.ts b/packages/cli/tests/providers/openclaw-bridge.test.ts new file mode 100644 index 00000000..75c2fcb4 --- /dev/null +++ b/packages/cli/tests/providers/openclaw-bridge.test.ts @@ -0,0 +1,152 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' + +import { describe, it, expect } from 'vitest' + +import { createOpenClawProvider } from '../../src/providers/openclaw.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the openclaw bridge migration. The GOLDEN below +// was captured from the legacy in-CLI decode before the migration; the bridged +// provider (discovery + I/O CLI-side, pure decode in @codeburn/core/providers/openclaw) +// must reproduce it exactly. The fixture exercises model_change fallbacks, the +// provider-reported measured-cost path, tool mapping, bash extraction, and dedup. + +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/openclaw-parity/.openclaw/agents') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'openclaw', + model: 'claude-sonnet-4-6', + inputTokens: 500, + outputTokens: 100, + cacheCreationInputTokens: 50, + cacheReadInputTokens: 200, + cachedInputTokens: 200, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-04-20T10:00:03.000Z', + speed: 'standard', + deduplicationKey: 'openclaw:test-sess-1:a1', + userMessage: 'hello world', + sessionId: 'test-sess-1', + }, + { + provider: 'openclaw', + model: 'claude-sonnet-4-6', + inputTokens: 600, + outputTokens: 200, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 100, + cachedInputTokens: 100, + reasoningTokens: 0, + webSearchRequests: 0, + costUSD: 0.05, + costBasis: 'measured', + tools: ['Bash', 'Write'], + bashCommands: ['ls'], + timestamp: '2026-04-20T10:00:05.000Z', + speed: 'standard', + deduplicationKey: 'openclaw:test-sess-1:a2', + userMessage: '', + sessionId: 'test-sess-1', + }, +] + +async function collect(): Promise { + const provider = createOpenClawProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe('openclaw bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output preserves the measured cost and leaves everything else unchanged', async () => { + const raw = await collect() + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe(raw[i]!.costBasis) + if (raw[i]!.costBasis === 'measured') { + // For measured calls the parse already carries the provider-reported + // cost; the pricing pass must leave the call byte-identical. + expect(call).toEqual(raw[i]) + } else { + // For estimated calls the pass adds exactly costUSD. + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + } + }) + // The provider-reported cost survives unchanged through the pricing pass. + expect(priced[1]!.costUSD).toBe(0.05) + expect(priced[1]!.costBasis).toBe('measured') + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const provider = createOpenClawProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(2) + expect(second).toEqual([]) + }) + + // The session-init entry may omit its id (or be absent). The legacy decode + // fell back to `basename(path, '.jsonl')`, which strips the extension; the + // bridge must match, since sessionId flows into the dedup key. + it('falls back to the extension-stripped filename when the session id is missing', async () => { + const NOID_DIR = resolve(here, '../fixtures/openclaw-parity-noid/.openclaw/agents') + const provider = createOpenClawProvider(NOID_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + } + expect(calls).toEqual([ + { + provider: 'openclaw', + model: 'claude-x', + inputTokens: 5, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Bash'], + bashCommands: ['echo'], + timestamp: '2026-04-20T10:00:03.000Z', + speed: 'standard', + deduplicationKey: 'openclaw:anon-session:a1', + userMessage: 'do work', + sessionId: 'anon-session', + }, + ]) + }) +}) diff --git a/packages/core/package.json b/packages/core/package.json index 37fd5ebc..aaf15d30 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -39,6 +39,10 @@ "types": "./dist/providers/claude/index.d.ts", "import": "./dist/providers/claude/index.js" }, + "./providers/codebuff": { + "types": "./dist/providers/codebuff/index.d.ts", + "import": "./dist/providers/codebuff/index.js" + }, "./providers/codewhale": { "types": "./dist/providers/codewhale/index.d.ts", "import": "./dist/providers/codewhale/index.js" @@ -71,6 +75,10 @@ "types": "./dist/providers/mux/index.d.ts", "import": "./dist/providers/mux/index.js" }, + "./providers/openclaw": { + "types": "./dist/providers/openclaw/index.d.ts", + "import": "./dist/providers/openclaw/index.js" + }, "./providers/open-design": { "types": "./dist/providers/open-design/index.d.ts", "import": "./dist/providers/open-design/index.js" diff --git a/packages/core/src/providers/codebuff/decode.ts b/packages/core/src/providers/codebuff/decode.ts new file mode 100644 index 00000000..53c6e830 --- /dev/null +++ b/packages/core/src/providers/codebuff/decode.ts @@ -0,0 +1,246 @@ +// @codeburn/core Codebuff decoder: pure decode over a host-supplied chat-messages +// record array. No fs / env / clock — the host reads the JSON file and hands the +// parsed array straight through. The rich output carries token buckets but NO +// pricing (cost leaves the decoder; the host prices via its estimated-cost seam) +// and NO bash base-name extraction (that stays host-side with strip-ansi). + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { + CodebuffBlock, + CodebuffChatMessage, + CodebuffDecodedCall, + CodebuffMetadata, + CodebuffUsage, +} from './types.js' + +// Codebuff's native tool names mapped to codeburn's canonical vocabulary. +export const codebuffToolNameMap: Record = { + read_files: 'Read', + read_file: 'Read', + code_search: 'Grep', + glob: 'Glob', + find_files: 'Glob', + str_replace: 'Edit', + edit_file: 'Edit', + write_file: 'Write', + run_terminal_command: 'Bash', + terminal: 'Bash', + spawn_agents: 'Agent', + spawn_agent: 'Agent', + write_todos: 'TodoWrite', + create_plan: 'TodoWrite', + browser_logs: 'WebFetch', + web_search: 'WebSearch', + fetch_url: 'WebFetch', +} + +// Tool names ignored for classification. +const IGNORED_TOOLS = new Set(['suggest_followups', 'end_turn']) + +function pickNumber(...vals: Array): number | undefined { + for (const v of vals) { + if (typeof v === 'number' && Number.isFinite(v)) return v + } + return undefined +} + +function normalizeUsage(u: CodebuffUsage | undefined): { + input: number + output: number + cacheRead: number + cacheWrite: number +} { + if (!u) return { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } + return { + input: pickNumber(u.inputTokens, u.input_tokens, u.promptTokens, u.prompt_tokens) ?? 0, + output: pickNumber(u.outputTokens, u.output_tokens, u.completionTokens, u.completion_tokens) ?? 0, + cacheRead: + pickNumber( + u.cacheReadInputTokens, + u.cache_read_input_tokens, + u.promptTokensDetails?.cachedTokens, + u.prompt_tokens_details?.cached_tokens, + ) ?? 0, + cacheWrite: pickNumber(u.cacheCreationInputTokens, u.cache_creation_input_tokens) ?? 0, + } +} + +function coerceTimestamp(value: string | number | undefined): string { + if (value == null) return '' + if (typeof value === 'number') { + return Number.isFinite(value) ? new Date(value).toISOString() : '' + } + const parsed = Date.parse(value) + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : value +} + +function parseChatIdToIso(chatId: string): string { + const iso = chatId.replace(/(\d{4}-\d{2}-\d{2}T\d{2})-(\d{2})-(\d{2})/, '$1:$2:$3') + const parsed = Date.parse(iso) + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : '' +} + +function extractAgentType(meta: CodebuffMetadata | undefined): string | null { + return meta?.runState?.sessionState?.mainAgentState?.agentType ?? null +} + +function collectBlockTools( + blocks: CodebuffBlock[] | undefined, + acc: { tools: string[]; rawBashCommands: string[] }, +): void { + if (!Array.isArray(blocks)) return + for (const block of blocks) { + if (!block || typeof block !== 'object') continue + if (block.type === 'tool' && typeof block.toolName === 'string') { + const raw = block.toolName + if (!IGNORED_TOOLS.has(raw)) { + acc.tools.push(codebuffToolNameMap[raw] ?? raw) + } + if ((raw === 'run_terminal_command' || raw === 'terminal') && block.input) { + const cmd = block.input['command'] + if (typeof cmd === 'string') { + acc.rawBashCommands.push(cmd) + } + } + } + if (block.type === 'agent' && Array.isArray(block.blocks)) { + collectBlockTools(block.blocks, acc) + } + } +} + +function resolveModel(meta: CodebuffMetadata | undefined, stashedModel: string | null): string { + const direct = meta?.model ?? meta?.modelId ?? meta?.codebuff?.model + if (direct) return direct + if (stashedModel) return stashedModel + const agentType = extractAgentType(meta) + if (agentType) return `codebuff-${agentType}` + return 'codebuff' +} + +function usageFromHistory(meta: CodebuffMetadata | undefined): { + model: string | null + input: number + output: number + cacheRead: number + cacheWrite: number +} { + const hist = meta?.runState?.sessionState?.mainAgentState?.messageHistory + if (!Array.isArray(hist)) return { model: null, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } + for (let i = hist.length - 1; i >= 0; i--) { + const entry = hist[i] + if (!entry || entry.role !== 'assistant' || !entry.providerOptions) continue + const u = normalizeUsage(entry.providerOptions.usage ?? entry.providerOptions.codebuff?.usage) + if (u.input > 0 || u.output > 0 || u.cacheRead > 0 || u.cacheWrite > 0) { + return { model: entry.providerOptions.codebuff?.model ?? null, ...u } + } + } + return { model: null, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } +} + +function extractChannelFromChatDir(chatDir: string): string | null { + const parts = chatDir.split(/[/\\]/) + if (parts.length < 5) return null + if (parts[parts.length - 2] !== 'chats') return null + if (parts[parts.length - 4] !== 'projects') return null + return parts[parts.length - 5] ?? null +} + +export type CodebuffDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type CodebuffDecodeResult = { + calls: CodebuffDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode a Codebuff chat-messages.json array into rich, cost-free calls. A single + * pass: user messages set the pending prompt for the next assistant call; assistant + * messages that carry credits or token usage flush into a call. Dedup is keyed on + * `codebuff::` against the live `seenKeys` set (host-owned). + */ +export function decodeCodebuff({ records, context, seenKeys: liveSeen }: CodebuffDecodeInput): CodebuffDecodeResult { + const seen = liveSeen ?? new Set() + const calls: CodebuffDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + const chatDir = context.sourceRef + const chatId = chatDir.split(/[/\\]/).pop() ?? '' + const fallbackTs = parseChatIdToIso(chatId) + + let pendingUserMessage = '' + + for (const [idx, rawMsg] of records.entries()) { + const msg = rawMsg as CodebuffChatMessage | null + if (!msg || typeof msg !== 'object') continue + + const variant = msg.variant ?? msg.role + if (variant === 'user') { + if (typeof msg.content === 'string' && msg.content.length > 0) { + pendingUserMessage = msg.content + } + continue + } + + if (variant !== 'ai' && variant !== 'agent' && variant !== 'assistant') continue + + const credits = typeof msg.credits === 'number' && Number.isFinite(msg.credits) ? msg.credits : 0 + const directUsage = normalizeUsage(msg.metadata?.usage ?? msg.metadata?.codebuff?.usage) + const stashedUsage = usageFromHistory(msg.metadata) + + const hasDirect = + directUsage.input > 0 || + directUsage.output > 0 || + directUsage.cacheRead > 0 || + directUsage.cacheWrite > 0 + const usage = hasDirect ? directUsage : stashedUsage + const stashedModel = stashedUsage.model + + if (credits === 0 && usage.input === 0 && usage.output === 0 && usage.cacheRead === 0 && usage.cacheWrite === 0) { + continue + } + + const model = resolveModel(msg.metadata, stashedModel) + const timestamp = coerceTimestamp(msg.timestamp ?? msg.metadata?.timestamp) || fallbackTs + + const dedupId = msg.id ?? String(idx) + const dedupKey = `codebuff:${chatDir}:${dedupId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const acc = { tools: [] as string[], rawBashCommands: [] as string[] } + collectBlockTools(msg.blocks, acc) + + const channel = extractChannelFromChatDir(chatDir) + const sessionId = channel ? `${channel}/${chatId}` : chatId + + calls.push({ + provider: 'codebuff', + model, + inputTokens: usage.input, + outputTokens: usage.output, + cacheCreationInputTokens: usage.cacheWrite, + cacheReadInputTokens: usage.cacheRead, + cachedInputTokens: usage.cacheRead, + reasoningTokens: 0, + webSearchRequests: 0, + tools: acc.tools, + rawBashCommands: acc.rawBashCommands, + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: pendingUserMessage, + sessionId, + credits, + }) + + pendingUserMessage = '' + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/codebuff/index.ts b/packages/core/src/providers/codebuff/index.ts new file mode 100644 index 00000000..198192c5 --- /dev/null +++ b/packages/core/src/providers/codebuff/index.ts @@ -0,0 +1,30 @@ +// @codeburn/core Codebuff provider. +// +// Two layers: +// - Rich pure decode (`decodeCodebuff`): host-facing, NOT part of the stable +// minimized surface. Pure over supplied records; carries content in-memory +// but no pricing and no bash base-name extraction. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeCodebuff, + codebuffToolNameMap, + type CodebuffDecodeInput, + type CodebuffDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichCodebuffSessionDecode, + type CodebuffToObservationsContext, +} from './observations.js' + +export type { + CodebuffDecodedCall, + CodebuffChatMessage, + CodebuffMetadata, + CodebuffUsage, + CodebuffBlock, + CodebuffHistoryMessage, +} from './types.js' diff --git a/packages/core/src/providers/codebuff/observations.ts b/packages/core/src/providers/codebuff/observations.ts new file mode 100644 index 00000000..71ad60d0 --- /dev/null +++ b/packages/core/src/providers/codebuff/observations.ts @@ -0,0 +1,86 @@ +// Minimizing transform: rich Codebuff decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the user message, project path, or shell +// command. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { CodebuffDecodedCall } from './types.js' + +/** One Codebuff session's rich decode, as the host holds it before minimization. */ +export interface RichCodebuffSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (as decodeCodebuff emits them). */ + calls: CodebuffDecodedCall[] +} + +export interface CodebuffToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +// Canonical tool-name charset, mirroring core's CanonicalToolName schema. +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: CodebuffDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichCodebuffSessionDecode, ctx: CodebuffToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'codebuff' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Codebuff decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + * + * Content-smuggling guarantee: no free text (user message, project path, + * command) is ever copied into the result. Only fingerprints, enums, numbers, + * timestamps, dedup keys, and canonical tool names cross the boundary. + */ +export function toObservations( + decode: RichCodebuffSessionDecode | RichCodebuffSessionDecode[], + ctx: CodebuffToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/codebuff/types.ts b/packages/core/src/providers/codebuff/types.ts new file mode 100644 index 00000000..f19926d3 --- /dev/null +++ b/packages/core/src/providers/codebuff/types.ts @@ -0,0 +1,98 @@ +// Raw record + rich-decode types for the Codebuff provider. +// +// The record types describe the shape of Codebuff's chat-messages.json. The +// Decoded* types are the rich decode layer's output: pure over supplied records, +// carrying content in-memory but NO pricing (the host prices them) and NO bash +// base-name extraction (the host runs that). + +export type CodebuffUsage = { + inputTokens?: number + input_tokens?: number + promptTokens?: number + prompt_tokens?: number + outputTokens?: number + output_tokens?: number + completionTokens?: number + completion_tokens?: number + cacheCreationInputTokens?: number + cache_creation_input_tokens?: number + cacheReadInputTokens?: number + cache_read_input_tokens?: number + promptTokensDetails?: { cachedTokens?: number } + prompt_tokens_details?: { cached_tokens?: number } +} + +export type CodebuffBlock = { + type?: string + content?: string + toolName?: string + input?: Record + output?: string + agentName?: string + agentType?: string + status?: string + blocks?: CodebuffBlock[] +} + +export type CodebuffHistoryMessage = { + role?: string + providerOptions?: { + codebuff?: { model?: string; usage?: CodebuffUsage } + usage?: CodebuffUsage + } +} + +export type CodebuffMetadata = { + model?: string + modelId?: string + timestamp?: string | number + usage?: CodebuffUsage + codebuff?: { model?: string; usage?: CodebuffUsage } + runState?: { + cwd?: string + sessionState?: { + cwd?: string + projectContext?: { cwd?: string } + fileContext?: { cwd?: string } + mainAgentState?: { + agentType?: string + messageHistory?: CodebuffHistoryMessage[] + } + } + } +} + +export type CodebuffChatMessage = { + id?: string + variant?: string + role?: string + content?: string + timestamp?: string | number + credits?: number + blocks?: CodebuffBlock[] + metadata?: CodebuffMetadata +} + +// The rich decode of one Codebuff call (one assistant message with usage/credits), +// pre-pricing. Mirrors the host's ParsedProviderCall minus cost fields. +export type CodebuffDecodedCall = { + provider: 'codebuff' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string + /** Credits attached to the assistant message; used host-side to compute + * fallbackCostUSD when the pricing table cannot price the model. */ + credits: number +} diff --git a/packages/core/src/providers/openclaw/decode.ts b/packages/core/src/providers/openclaw/decode.ts new file mode 100644 index 00000000..857bc366 --- /dev/null +++ b/packages/core/src/providers/openclaw/decode.ts @@ -0,0 +1,182 @@ +// @codeburn/core OpenClaw decoder: pure decode over host-supplied JSONL records. +// No fs / env / clock — the host reads the file and hands the lines straight +// through. The rich output carries token buckets and any provider-reported cost +// but NO host-computed pricing. Bash base-name extraction stays host-side. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { OpenClawDecodedCall, OpenClawEntry, OpenClawUsage } from './types.js' + +export const openclawToolNameMap: Record = { + bash: 'Bash', + exec: 'Bash', + read: 'Read', + edit: 'Edit', + write: 'Write', + glob: 'Glob', + grep: 'Grep', + task: 'Agent', + dispatch_agent: 'Agent', + fetch: 'WebFetch', + search: 'WebSearch', + todo: 'TodoWrite', + patch: 'Patch', +} + +function extractTools( + content: Array<{ type?: string; name?: string; arguments?: Record }> | undefined, +): { tools: string[]; rawBashCommands: string[] } { + const tools: string[] = [] + const rawBashCommands: string[] = [] + if (!content) return { tools, rawBashCommands } + + for (const block of content) { + if ((block.type === 'tool_use' || block.type === 'toolCall') && block.name) { + const mapped = openclawToolNameMap[block.name] ?? block.name + tools.push(mapped) + if (mapped === 'Bash' && block.arguments && typeof block.arguments.command === 'string') { + rawBashCommands.push(block.arguments.command) + } + } + } + return { tools, rawBashCommands } +} + +function parseOpenClawLine(line: string): OpenClawEntry | null { + if (!line.trim()) return null + try { + return JSON.parse(line) as OpenClawEntry + } catch { + return null + } +} + +export type OpenClawDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type OpenClawDecodeResult = { + calls: OpenClawDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode an OpenClaw session JSONL file's records into rich, cost-free calls. + * The host owns file I/O and the live cross-file dedup set; this function is + * pure over the supplied records. Provider-reported cost is preserved as + * `costUSD` with `costBasis: 'measured'`; otherwise the call carries + * `costBasis: 'estimated'` for the host's pricing seam. + */ +export function decodeOpenClaw({ records, context, seenKeys: liveSeen }: OpenClawDecodeInput): OpenClawDecodeResult { + const seen = liveSeen ?? new Set() + const calls: OpenClawDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + let sessionId = '' + let sessionTimestamp = '' + let currentModel = '' + let pendingUserMessage = '' + + const pendingCalls: { + model: string + usage: OpenClawUsage + tools: string[] + rawBashCommands: string[] + timestamp: string + userMessage: string + dedupId: string + }[] = [] + + for (const rawLine of records) { + const entry = typeof rawLine === 'string' ? parseOpenClawLine(rawLine) : (rawLine as OpenClawEntry | null) + if (!entry) continue + + if (entry.type === 'session') { + sessionId = entry.id ?? '' + sessionTimestamp = entry.timestamp ?? '' + continue + } + + if (entry.type === 'model_change') { + currentModel = entry.modelId ?? currentModel + continue + } + + if (entry.type === 'custom' && entry.customType === 'model-snapshot') { + currentModel = entry.data?.modelId ?? currentModel + continue + } + + if (entry.type !== 'message' || !entry.message) continue + + const msg = entry.message + if (msg.role === 'user') { + if (!pendingUserMessage && Array.isArray(msg.content)) { + const textBlock = msg.content.find(c => c.type === 'text' && c.text) + pendingUserMessage = (textBlock?.text ?? '').slice(0, 500) + } + continue + } + + if (msg.role !== 'assistant') continue + + const model = msg.model ?? currentModel + if (msg.usage) { + const { tools, rawBashCommands } = extractTools(msg.content) + pendingCalls.push({ + model, + usage: msg.usage, + tools, + rawBashCommands, + timestamp: entry.timestamp ?? sessionTimestamp, + userMessage: pendingUserMessage, + dedupId: entry.id ?? '', + }) + pendingUserMessage = '' + } + } + + // Fallback mirrors the host's `basename(sourceRef, '.jsonl')`: take the last + // path segment and strip the .jsonl extension (the split-only pop kept it). + if (!sessionId) sessionId = (context.sourceRef.split(/[/\\]/).pop() ?? '').replace(/\.jsonl$/, '') + + for (let i = 0; i < pendingCalls.length; i++) { + const call = pendingCalls[i]! + const dedupKey = `openclaw:${sessionId}:${call.dedupId || i}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const u = call.usage + const costFromProvider = u.cost?.total ?? 0 + + const ts = new Date(call.timestamp) + if (isNaN(ts.getTime()) || ts.getTime() < 1_000_000_000_000) continue + + const measured = costFromProvider > 0 + const decodedCall: OpenClawDecodedCall = { + provider: 'openclaw', + model: call.model || 'openclaw-auto', + inputTokens: u.input, + outputTokens: u.output, + cacheCreationInputTokens: u.cacheWrite, + cacheReadInputTokens: u.cacheRead, + cachedInputTokens: u.cacheRead, + reasoningTokens: 0, + webSearchRequests: 0, + ...(measured ? { costUSD: costFromProvider, costBasis: 'measured' as const } : { costBasis: 'estimated' as const }), + tools: call.tools, + rawBashCommands: call.rawBashCommands, + timestamp: ts.toISOString(), + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: call.userMessage, + sessionId, + } + + calls.push(decodedCall) + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/openclaw/index.ts b/packages/core/src/providers/openclaw/index.ts new file mode 100644 index 00000000..47af3ec1 --- /dev/null +++ b/packages/core/src/providers/openclaw/index.ts @@ -0,0 +1,27 @@ +// @codeburn/core OpenClaw provider. +// +// Two layers: +// - Rich pure decode (`decodeOpenClaw`): host-facing, NOT part of the stable +// minimized surface. Pure over supplied records; carries content in-memory +// but no host-computed pricing and no bash base-name extraction. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeOpenClaw, + openclawToolNameMap, + type OpenClawDecodeInput, + type OpenClawDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichOpenClawSessionDecode, + type OpenClawToObservationsContext, +} from './observations.js' + +export type { + OpenClawDecodedCall, + OpenClawEntry, + OpenClawUsage, +} from './types.js' diff --git a/packages/core/src/providers/openclaw/observations.ts b/packages/core/src/providers/openclaw/observations.ts new file mode 100644 index 00000000..d9e88b7a --- /dev/null +++ b/packages/core/src/providers/openclaw/observations.ts @@ -0,0 +1,86 @@ +// Minimizing transform: rich OpenClaw decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the user message, project path, or shell +// command. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { OpenClawDecodedCall } from './types.js' + +/** One OpenClaw session's rich decode, as the host holds it before minimization. */ +export interface RichOpenClawSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (as decodeOpenClaw emits them). */ + calls: OpenClawDecodedCall[] +} + +export interface OpenClawToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +// Canonical tool-name charset, mirroring core's CanonicalToolName schema. +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: OpenClawDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: call.costBasis, + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichOpenClawSessionDecode, ctx: OpenClawToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'openclaw' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich OpenClaw decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + * + * Content-smuggling guarantee: no free text (user message, project path, + * command) is ever copied into the result. Only fingerprints, enums, numbers, + * timestamps, dedup keys, and canonical tool names cross the boundary. + */ +export function toObservations( + decode: RichOpenClawSessionDecode | RichOpenClawSessionDecode[], + ctx: OpenClawToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/openclaw/types.ts b/packages/core/src/providers/openclaw/types.ts new file mode 100644 index 00000000..c07c4382 --- /dev/null +++ b/packages/core/src/providers/openclaw/types.ts @@ -0,0 +1,62 @@ +// Raw record + rich-decode types for the OpenClaw provider. +// +// OpenClaw sessions are JSONL files. The host reads the file and hands the lines +// straight through. The Decoded* types are the rich decode layer's output: pure +// over supplied records, carrying content in-memory but NO computed pricing (the +// host may add its own estimated-cost seam). Bash base-name extraction stays +// host-side. + +export type OpenClawUsage = { + input: number + output: number + cacheRead: number + cacheWrite: number + totalTokens?: number + cost?: { + total?: number + } +} + +export type OpenClawEntry = { + type: string + customType?: string + id?: string + timestamp?: string + provider?: string + modelId?: string + data?: { + provider?: string + modelId?: string + } + message?: { + role?: string + content?: Array<{ type?: string; text?: string; name?: string; arguments?: Record }> + model?: string + provider?: string + usage?: OpenClawUsage + } +} + +// The rich decode of one OpenClaw call (one assistant message with usage), +// pre-pricing. Mirrors the host's ParsedProviderCall minus the host-side +// estimated-cost seam. Provider-reported cost is preserved when present. +export type OpenClawDecodedCall = { + provider: 'openclaw' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + costUSD?: number + costBasis: 'estimated' | 'measured' + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index aacfe072..a444dbcd 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -118,6 +118,8 @@ const CORRECTION_PHRASES = [ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/claude/decode.ts', 'src/providers/claude/types.ts', + 'src/providers/codebuff/decode.ts', + 'src/providers/codebuff/types.ts', 'src/providers/codewhale/decode.ts', 'src/providers/codewhale/types.ts', 'src/providers/codex/decode.ts', @@ -134,6 +136,8 @@ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/droid/types.ts', 'src/providers/mux/decode.ts', 'src/providers/mux/types.ts', + 'src/providers/openclaw/decode.ts', + 'src/providers/openclaw/types.ts', 'src/providers/open-design/decode.ts', 'src/providers/open-design/types.ts', 'src/providers/lingtai-tui/decode.ts', diff --git a/packages/core/tests/content-smuggling.test.ts b/packages/core/tests/content-smuggling.test.ts index 973d24da..b1c23123 100644 --- a/packages/core/tests/content-smuggling.test.ts +++ b/packages/core/tests/content-smuggling.test.ts @@ -23,6 +23,8 @@ import { decodeQwen, toObservations as toQwenObservations } from '../src/provide import { decodeGrok, toObservations as toGrokObservations } from '../src/providers/grok/index.js' import { decodeKimi, toObservations as toKimiObservations } from '../src/providers/kimi/index.js' import { decodeCodeWhale, toObservations as toCodeWhaleObservations } from '../src/providers/codewhale/index.js' +import { decodeCodebuff, toObservations as toCodebuffObservations } from '../src/providers/codebuff/index.js' +import { decodeOpenClaw, toObservations as toOpenClawObservations } from '../src/providers/openclaw/index.js' import type { DecodeContext } from '../src/contracts.js' const here = dirname(fileURLToPath(import.meta.url)) @@ -577,6 +579,114 @@ describe('content-smuggling guardrail: real codewhale decode -> toObservations i }) }) +describe('content-smuggling guardrail: real codebuff decode -> toObservations is secret-free', () => { + const codebuffContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'codebuff', sourceRef: '/data/manicode/projects/hostile/chats/2026-07-17T10-00-00.000Z' } + + function decodeAndMinimize() { + const records = [ + { + id: 'u1', + variant: 'user', + content: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}`, + timestamp: '2026-07-17T10:00:00.000Z', + }, + { + id: 'a1', + variant: 'ai', + timestamp: '2026-07-17T10:00:05.000Z', + credits: 1, + blocks: [ + { type: 'tool', toolName: 'run_terminal_command', input: { command: SECRETS.commandLine } }, + // A hostile tool NAME carrying a command line: fails canonical charset. + { type: 'tool', toolName: SECRETS.commandLine, input: {} }, + ], + }, + ] + const { calls } = decodeCodebuff({ records, context: codebuffContext }) + const { sessions } = toCodebuffObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'codebuff' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile chat', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + +describe('content-smuggling guardrail: real openclaw decode -> toObservations is secret-free', () => { + const openclawContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'openclaw', sourceRef: '/data/agents/hostile/sessions/sess-hostile.jsonl' } + + function decodeAndMinimize() { + const records = [ + JSON.stringify({ type: 'session', id: 'sess-hostile', timestamp: '2026-07-17T10:00:00.000Z' }), + JSON.stringify({ + type: 'message', id: 'u1', timestamp: '2026-07-17T10:00:01.000Z', + message: { role: 'user', content: [{ type: 'text', text: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` }] }, + }), + JSON.stringify({ + type: 'message', id: 'a1', timestamp: '2026-07-17T10:00:05.000Z', + message: { + role: 'assistant', + content: [ + { type: 'toolCall', name: 'exec', arguments: { command: SECRETS.commandLine } }, + // A hostile tool NAME carrying a command line: fails canonical charset. + { type: 'toolCall', name: SECRETS.commandLine, arguments: {} }, + ], + usage: { input: 500, output: 200, cacheRead: 0, cacheWrite: 0 }, + }, + }), + ] + const { calls } = decodeOpenClaw({ records, context: openclawContext }) + const { sessions } = toOpenClawObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'openclaw' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile session', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + describe('content-smuggling guardrail: diagnostic detail rejects paths', () => { it('rejects an absolute path', () => { expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false) diff --git a/packages/core/tests/providers/codebuff-decode.test.ts b/packages/core/tests/providers/codebuff-decode.test.ts new file mode 100644 index 00000000..66082790 --- /dev/null +++ b/packages/core/tests/providers/codebuff-decode.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest' + +import { decodeCodebuff, toObservations, type CodebuffChatMessage } from '../../src/providers/codebuff/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'codebuff', sourceRef: '/data/manicode/projects/alpha/chats/2026-04-14T10-00-00.000Z' } + +const MESSAGES: CodebuffChatMessage[] = [ + { id: 'u1', variant: 'user', content: 'implement the feature', timestamp: '2026-04-14T10:00:10.000Z' }, + { + id: 'a1', + variant: 'ai', + timestamp: '2026-04-14T10:00:30.000Z', + credits: 42, + metadata: { + runState: { sessionState: { mainAgentState: { agentType: 'base2' } } }, + }, + blocks: [ + { type: 'tool', toolName: 'read_files', input: {} }, + { type: 'tool', toolName: 'str_replace', input: {} }, + { type: 'tool', toolName: 'run_terminal_command', input: { command: 'npm test' } }, + { type: 'tool', toolName: 'suggest_followups', input: {} }, + ], + }, + { id: 'u2', variant: 'user', content: 'fix the bug', timestamp: '2026-04-14T10:01:00.000Z' }, + { + id: 'a2', + variant: 'ai', + timestamp: '2026-04-14T10:01:30.000Z', + credits: 10, + metadata: { + model: 'claude-haiku-4-5-20251001', + usage: { + inputTokens: 5000, + outputTokens: 2000, + cacheCreationInputTokens: 1000, + cacheReadInputTokens: 500, + }, + }, + }, + { + id: 'a3', + variant: 'ai', + timestamp: '2026-04-14T10:02:00.000Z', + credits: 7, + metadata: { + runState: { + sessionState: { + mainAgentState: { + messageHistory: [ + { role: 'user' }, + { + role: 'assistant', + providerOptions: { + codebuff: { + model: 'openai/gpt-4o', + usage: { + prompt_tokens: 2000, + completion_tokens: 800, + prompt_tokens_details: { cached_tokens: 400 }, + }, + }, + }, + }, + ], + }, + }, + }, + }, + }, + // Duplicate id -> must drop. + { id: 'a1', variant: 'ai', timestamp: '2026-04-14T10:02:30.000Z', credits: 5 }, + // No credits, no tokens -> must skip. + { id: 'a4', variant: 'ai', content: 'mode-divider', timestamp: '2026-04-14T10:03:00.000Z' }, +] + +describe('codebuff rich decode (moved to @codeburn/core)', () => { + it('decodes assistant calls into cost-free rich calls, preserving credits for host-side fallback', () => { + const { calls } = decodeCodebuff({ records: MESSAGES, context }) + expect(calls).toHaveLength(3) + + const [first, second, third] = calls + // No host-computed pricing crosses into the decode layer. + expect(first).not.toHaveProperty('costUSD') + expect(first).not.toHaveProperty('costBasis') + + expect(first!.model).toBe('codebuff-base2') + expect(first!.inputTokens).toBe(0) + expect(first!.outputTokens).toBe(0) + expect(first!.tools).toEqual(['Read', 'Edit', 'Bash']) + expect(first!.rawBashCommands).toEqual(['npm test']) + expect(first!.credits).toBe(42) + expect(first!.userMessage).toBe('implement the feature') + expect(first!.deduplicationKey).toBe(`codebuff:${context.sourceRef}:a1`) + + expect(second!.model).toBe('claude-haiku-4-5-20251001') + expect(second!.inputTokens).toBe(5000) + expect(second!.outputTokens).toBe(2000) + expect(second!.cacheCreationInputTokens).toBe(1000) + expect(second!.cacheReadInputTokens).toBe(500) + expect(second!.cachedInputTokens).toBe(500) + expect(second!.credits).toBe(10) + + // History fallback usage. + expect(third!.model).toBe('openai/gpt-4o') + expect(third!.inputTokens).toBe(2000) + expect(third!.outputTokens).toBe(800) + expect(third!.cacheReadInputTokens).toBe(400) + expect(third!.credits).toBe(7) + // The previous message was assistant, so no pending user message. + expect(third!.userMessage).toBe('') + }) + + it('threads a live seenKeys set so a repeated id across passes drops', () => { + const seen = new Set() + const first = decodeCodebuff({ records: MESSAGES.slice(0, 2), context, seenKeys: seen }).calls + expect(first).toHaveLength(1) + const again = decodeCodebuff({ records: MESSAGES.slice(0, 2), context, seenKeys: seen }).calls + expect(again).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const { calls } = decodeCodebuff({ records: MESSAGES, context }) + const { sessions } = toObservations( + { sessionId: 'sess-a', projectPath: '/Users/t/alpha', calls }, + { privacyKey: 'test-privacy-key', provider: 'codebuff' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + const toolNames = sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(toolNames).toContain('Read') + expect(toolNames).toContain('Edit') + expect(toolNames).toContain('Bash') + }) +}) diff --git a/packages/core/tests/providers/openclaw-decode.test.ts b/packages/core/tests/providers/openclaw-decode.test.ts new file mode 100644 index 00000000..47953306 --- /dev/null +++ b/packages/core/tests/providers/openclaw-decode.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' + +import { decodeOpenClaw, toObservations } from '../../src/providers/openclaw/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'openclaw', sourceRef: '/data/agents/myagent/sessions/test-sess-1.jsonl' } + +const RECORDS: string[] = [ + JSON.stringify({ type: 'session', version: 3, id: 'test-sess-1', timestamp: '2026-04-20T10:00:00.000Z', cwd: '/tmp' }), + JSON.stringify({ type: 'model_change', id: 'mc1', timestamp: '2026-04-20T10:00:01.000Z', provider: 'anthropic', modelId: 'claude-sonnet-4-6' }), + JSON.stringify({ + type: 'message', id: 'u1', timestamp: '2026-04-20T10:00:02.000Z', + message: { role: 'user', content: [{ type: 'text', text: 'hello world' }] }, + }), + JSON.stringify({ + type: 'message', id: 'a1', timestamp: '2026-04-20T10:00:03.000Z', + message: { + role: 'assistant', model: 'claude-sonnet-4-6', + content: [{ type: 'text', text: 'Hi!' }], + usage: { input: 500, output: 100, cacheRead: 200, cacheWrite: 50, totalTokens: 850 }, + }, + }), + JSON.stringify({ + type: 'message', id: 'a2', timestamp: '2026-04-20T10:00:05.000Z', + message: { + role: 'assistant', model: 'claude-sonnet-4-6', + content: [ + { type: 'text', text: 'Running command' }, + { type: 'toolCall', name: 'exec', arguments: { command: 'ls -la' } }, + { type: 'tool_use', name: 'write', arguments: { path: '/tmp/y' } }, + ], + usage: { input: 600, output: 200, cacheRead: 100, cacheWrite: 0, totalTokens: 900, cost: { total: 0.05 } }, + }, + }), + // Duplicate id -> must drop. + JSON.stringify({ + type: 'message', id: 'a1', timestamp: '2026-04-20T10:00:06.000Z', + message: { + role: 'assistant', model: 'claude-sonnet-4-6', + content: [{ type: 'text', text: 'Duplicate' }], + usage: { input: 10, output: 10, cacheRead: 0, cacheWrite: 0, totalTokens: 20 }, + }, + }), +] + +describe('openclaw rich decode (moved to @codeburn/core)', () => { + it('decodes assistant calls into cost-free rich calls, preserving provider-reported cost', () => { + const { calls } = decodeOpenClaw({ records: RECORDS, context }) + expect(calls).toHaveLength(2) + + const [first, second] = calls + expect(first!.model).toBe('claude-sonnet-4-6') + expect(first!.inputTokens).toBe(500) + expect(first!.outputTokens).toBe(100) + expect(first!.cacheReadInputTokens).toBe(200) + expect(first!.cacheCreationInputTokens).toBe(50) + expect(first!.costBasis).toBe('estimated') + expect(first).not.toHaveProperty('costUSD') + expect(first!.userMessage).toBe('hello world') + expect(first!.deduplicationKey).toBe('openclaw:test-sess-1:a1') + + expect(second!.model).toBe('claude-sonnet-4-6') + expect(second!.inputTokens).toBe(600) + expect(second!.outputTokens).toBe(200) + expect(second!.costBasis).toBe('measured') + expect(second!.costUSD).toBe(0.05) + expect(second!.tools).toEqual(['Bash', 'Write']) + expect(second!.rawBashCommands).toEqual(['ls -la']) + expect(second!.userMessage).toBe('') + }) + + it('threads a live seenKeys set so a repeated id across passes drops', () => { + const seen = new Set() + const first = decodeOpenClaw({ records: RECORDS.slice(0, 4), context, seenKeys: seen }).calls + expect(first).toHaveLength(1) + const again = decodeOpenClaw({ records: RECORDS.slice(0, 4), context, seenKeys: seen }).calls + expect(again).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const { calls } = decodeOpenClaw({ records: RECORDS, context }) + const { sessions } = toObservations( + { sessionId: 'test-sess-1', projectPath: '/tmp', calls }, + { privacyKey: 'test-privacy-key', provider: 'openclaw' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + const toolNames = sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(toolNames).toContain('Bash') + expect(toolNames).toContain('Write') + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index a1886dd4..b168404b 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ 'src/contracts.ts', 'src/detectors/index.ts', 'src/providers/claude/index.ts', + 'src/providers/codebuff/index.ts', 'src/providers/codewhale/index.ts', 'src/providers/codex/index.ts', 'src/providers/grok/index.ts', @@ -20,6 +21,7 @@ export default defineConfig({ 'src/providers/zerostack/index.ts', 'src/providers/droid/index.ts', 'src/providers/mux/index.ts', + 'src/providers/openclaw/index.ts', 'src/providers/open-design/index.ts', 'src/providers/lingtai-tui/index.ts', ],