From afaf9ffc8ae078b33012ba6e40f84dfd413f28d2 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Sun, 26 Jul 2026 18:27:44 -0700 Subject: [PATCH] =?UTF-8?q?refactor(core):=20tail=20migrations=20=E2=80=94?= =?UTF-8?q?=20crush,=20zcode,=20zed,=20forge,=20goose=20(phase=208,=20sqli?= =?UTF-8?q?te=20batch=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category B (sqlite) variant of the bridge migration: the sqlite driver and every SQL query stay CLI-side. Each provider's `readRecords` opens the database, runs the same queries as before, and hands the resulting rows (blob and all) to a pure core decoder; `toProviderCall` maps the rich, cost-free decode back onto ParsedProviderCall, where cost re-enters via the parser.ts pricing pass. Per provider: - crush: session row + dominant-model query -> one combined record. Crush stores cost in dollars, so a row with cost > 0 carries `measuredCostUSD` (costBasis 'measured'); a zero-cost row falls back to token estimation, arm order intact. - zcode: model_usage + tool_usage row sets -> one composite record. Each turn's tools still attach to the first non-skipped usage row of that turn only. - zed: threads rows handed over compressed; zstd decompression, JSON parsing and per-request/cumulative-remainder accounting are pure. The Node >= 22.15 zstd capability check stays host-side. - forge: conversation row handed over with `context` still serialized; JSON parsing and per-message decode are pure. Bash base-name extraction (and its strip-ansi dependency) stays CLI-side over the decoder's raw command strings. - goose: session + assistant tool-message + first-user-message rows, BLOB columns pre-converted to text host-side, bundled into one composite record. Validator fixes (original behavior is the authority): - forge: the draft replaced the pre-migration `mapToolName` switch with an object-literal lookup. Tool names come straight from conversation JSON, so names colliding with Object.prototype members ("constructor", "toString", "__proto__", "hasOwnProperty") resolved to inherited Functions / the prototype object and were pushed into `tools` as non-strings instead of falling through to the identity default. Restored the switch and pinned the arm in the fixture. - zed: the draft routed the "skipped N unreadable Zed threads" notice into record diagnostics, which the bridge discards, silently dropping a warning the pre-migration decode printed. Re-emitted host-side from the diagnostics count and pinned with a stderr assertion. - Fixture coverage extended for the arms that were regression-blind: forge's prototype-named tool calls, zed's aggregate stderr line, and goose's single-turn `toolSequence` omission plus the unparseable-timestamp fallback. Parity was verified independently of the bridge tests with a git-show harness that runs the same fixtures through the pre-migration provider files and asserts field-for-field equality, including the extra arms above. --- packages/cli/src/providers/crush.ts | 164 +++++----- packages/cli/src/providers/forge.ts | 264 ++++------------ packages/cli/src/providers/goose.ts | 299 +++++++----------- packages/cli/src/providers/zcode.ts | 190 ++++------- packages/cli/src/providers/zed.ts | 235 ++++---------- .../cli/tests/providers/crush-bridge.test.ts | 212 +++++++++++++ .../cli/tests/providers/forge-bridge.test.ts | 219 +++++++++++++ .../cli/tests/providers/goose-bridge.test.ts | 178 +++++++++++ .../cli/tests/providers/zcode-bridge.test.ts | 172 ++++++++++ .../cli/tests/providers/zed-bridge.test.ts | 231 ++++++++++++++ packages/core/package.json | 20 ++ packages/core/src/providers/crush/decode.ts | 73 +++++ packages/core/src/providers/crush/index.ts | 26 ++ .../core/src/providers/crush/observations.ts | 83 +++++ packages/core/src/providers/crush/types.ts | 45 +++ packages/core/src/providers/forge/decode.ts | 188 +++++++++++ packages/core/src/providers/forge/index.ts | 27 ++ .../core/src/providers/forge/observations.ts | 83 +++++ packages/core/src/providers/forge/types.ts | 50 +++ packages/core/src/providers/goose/decode.ts | 171 ++++++++++ packages/core/src/providers/goose/index.ts | 31 ++ .../core/src/providers/goose/observations.ts | 83 +++++ packages/core/src/providers/goose/types.ts | 78 +++++ packages/core/src/providers/zcode/decode.ts | 103 ++++++ packages/core/src/providers/zcode/index.ts | 27 ++ .../core/src/providers/zcode/observations.ts | 81 +++++ packages/core/src/providers/zcode/types.ts | 53 ++++ packages/core/src/providers/zed/decode.ts | 143 +++++++++ packages/core/src/providers/zed/index.ts | 28 ++ .../core/src/providers/zed/observations.ts | 82 +++++ packages/core/src/providers/zed/types.ts | 50 +++ packages/core/tests/architecture-gate.test.ts | 6 + packages/core/tests/content-smuggling.test.ts | 190 +++++++++++ .../core/tests/providers/crush-decode.test.ts | 78 +++++ .../core/tests/providers/forge-decode.test.ts | 104 ++++++ .../core/tests/providers/goose-decode.test.ts | 107 +++++++ .../core/tests/providers/zcode-decode.test.ts | 76 +++++ .../core/tests/providers/zed-decode.test.ts | 106 +++++++ packages/core/tsup.config.ts | 5 + 39 files changed, 3606 insertions(+), 755 deletions(-) create mode 100644 packages/cli/tests/providers/crush-bridge.test.ts create mode 100644 packages/cli/tests/providers/forge-bridge.test.ts create mode 100644 packages/cli/tests/providers/goose-bridge.test.ts create mode 100644 packages/cli/tests/providers/zcode-bridge.test.ts create mode 100644 packages/cli/tests/providers/zed-bridge.test.ts create mode 100644 packages/core/src/providers/crush/decode.ts create mode 100644 packages/core/src/providers/crush/index.ts create mode 100644 packages/core/src/providers/crush/observations.ts create mode 100644 packages/core/src/providers/crush/types.ts create mode 100644 packages/core/src/providers/forge/decode.ts create mode 100644 packages/core/src/providers/forge/index.ts create mode 100644 packages/core/src/providers/forge/observations.ts create mode 100644 packages/core/src/providers/forge/types.ts create mode 100644 packages/core/src/providers/goose/decode.ts create mode 100644 packages/core/src/providers/goose/index.ts create mode 100644 packages/core/src/providers/goose/observations.ts create mode 100644 packages/core/src/providers/goose/types.ts create mode 100644 packages/core/src/providers/zcode/decode.ts create mode 100644 packages/core/src/providers/zcode/index.ts create mode 100644 packages/core/src/providers/zcode/observations.ts create mode 100644 packages/core/src/providers/zcode/types.ts create mode 100644 packages/core/src/providers/zed/decode.ts create mode 100644 packages/core/src/providers/zed/index.ts create mode 100644 packages/core/src/providers/zed/observations.ts create mode 100644 packages/core/src/providers/zed/types.ts create mode 100644 packages/core/tests/providers/crush-decode.test.ts create mode 100644 packages/core/tests/providers/forge-decode.test.ts create mode 100644 packages/core/tests/providers/goose-decode.test.ts create mode 100644 packages/core/tests/providers/zcode-decode.test.ts create mode 100644 packages/core/tests/providers/zed-decode.test.ts diff --git a/packages/cli/src/providers/crush.ts b/packages/cli/src/providers/crush.ts index b56f4885..2d0fa1e6 100644 --- a/packages/cli/src/providers/crush.ts +++ b/packages/cli/src/providers/crush.ts @@ -2,8 +2,12 @@ import { readFile } from 'fs/promises' import { join, resolve } from 'path' import { homedir, platform } from 'os' +import { decodeCrush } from '@codeburn/core/providers/crush' +import type { CrushDecodedCall, CrushRawRecord } from '@codeburn/core/providers/crush' + import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' /// Crush stores per-project SQLite databases discovered through a JSON registry. /// We only read both. Schema source: charmbracelet/crush @@ -93,13 +97,6 @@ function validateSchema(db: SqliteDatabase): boolean { } } -function epochSecondsToIso(epochSeconds: number | null): string { - if (epochSeconds === null || !Number.isFinite(epochSeconds)) { - return new Date(0).toISOString() - } - return new Date(epochSeconds * 1000).toISOString() -} - function dominantModel(db: SqliteDatabase, sessionId: string): string { try { const rows = db.query<{ model: string | null }>( @@ -117,80 +114,34 @@ function dominantModel(db: SqliteDatabase, sessionId: string): string { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +// Map one rich, cost-free-or-measured decoder call into the host's +// ParsedProviderCall. Crush already stores cost in dollars, so a row with +// `measuredCostUSD` maps to `costBasis: 'measured'` (the pricing pass leaves it +// untouched); otherwise the row falls back to token-based estimation. Crush +// never captures a user message, so it is hardcoded empty here rather than +// carried through the rich decode. +function toProviderCall(rich: CrushDecodedCall): ParsedProviderCall { + const measured = rich.measuredCostUSD !== undefined return { - async *parse(): AsyncGenerator { - if (!isSqliteAvailable()) { - process.stderr.write(getSqliteLoadError() + '\n') - return - } - - // Source paths are encoded as `:`. Split from the - // right because dbPath may contain a colon on Windows (drive letter). - const segments = source.path.split(':') - const sessionId = segments[segments.length - 1]! - const dbPath = segments.slice(0, -1).join(':') - - let db: SqliteDatabase - try { - db = openDatabase(dbPath) - } catch (err) { - process.stderr.write( - `codeburn: cannot open Crush database: ${err instanceof Error ? err.message : err}\n`, - ) - return - } - - try { - if (!validateSchema(db)) return - - const rows = db.query( - `SELECT id, prompt_tokens, completion_tokens, cost, created_at, updated_at, message_count - FROM sessions - WHERE id = ? AND parent_session_id IS NULL`, - [sessionId], - ) - if (rows.length === 0) return - const session = rows[0]! - - const inputTokens = session.prompt_tokens ?? 0 - const outputTokens = session.completion_tokens ?? 0 - const cost = session.cost ?? 0 - if (inputTokens === 0 && outputTokens === 0 && cost === 0) return - - const dedupKey = `crush:${sessionId}` - if (seenKeys.has(dedupKey)) return - seenKeys.add(dedupKey) - - const model = dominantModel(db, sessionId) - // Crush already records cost in dollars; trust it. Fall back to - // host-side pricing-table calculation only when the row is missing a cost. - - yield { - provider: 'crush', - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - ...(cost > 0 - ? { costUSD: cost, costBasis: 'measured' as const } - : { costBasis: 'estimated' as const }), - tools: [], - bashCommands: [], - timestamp: epochSecondsToIso(session.updated_at ?? session.created_at), - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: '', - sessionId, - } - } finally { - db.close() - } - }, + provider: 'crush', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + ...(measured + ? { costUSD: rich.measuredCostUSD, costBasis: 'measured' as const } + : { costBasis: 'estimated' as const }), + tools: rich.tools, + bashCommands: [], + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: '', + sessionId: rich.sessionId, } } @@ -222,7 +173,7 @@ async function discoverFromDb(dbPath: string, project: string): Promise({ name: 'crush', displayName: 'Crush', @@ -247,10 +198,53 @@ export function createCrushProvider(): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: open the db, run the session-row query and the dominant-model + // query (both sqlite-side), and hand the core decoder one combined record. + async readRecords(source: SessionSource): Promise { + if (!isSqliteAvailable()) { + process.stderr.write(getSqliteLoadError() + '\n') + return null + } + + // Source paths are encoded as `:`. Split from the + // right because dbPath may contain a colon on Windows (drive letter). + const segments = source.path.split(':') + const sessionId = segments[segments.length - 1]! + const dbPath = segments.slice(0, -1).join(':') + + let db: SqliteDatabase + try { + db = openDatabase(dbPath) + } catch (err) { + process.stderr.write( + `codeburn: cannot open Crush database: ${err instanceof Error ? err.message : err}\n`, + ) + return null + } + + try { + if (!validateSchema(db)) return null + + const rows = db.query( + `SELECT id, prompt_tokens, completion_tokens, cost, created_at, updated_at, message_count + FROM sessions + WHERE id = ? AND parent_session_id IS NULL`, + [sessionId], + ) + if (rows.length === 0) return null + const session = rows[0]! + + const model = dominantModel(db, sessionId) + const record: CrushRawRecord = { ...session, model } + return [record] + } finally { + db.close() + } }, - } + + decode: decodeCrush, + toProviderCall, + }) } export const crush = createCrushProvider() diff --git a/packages/cli/src/providers/forge.ts b/packages/cli/src/providers/forge.ts index 3ab17d4b..bd3beb29 100644 --- a/packages/cli/src/providers/forge.ts +++ b/packages/cli/src/providers/forge.ts @@ -2,18 +2,13 @@ import { existsSync } from 'fs' import { homedir } from 'os' import { join } from 'path' +import { decodeForge } from '@codeburn/core/providers/forge' +import type { ForgeConversationRow, ForgeDecodedCall } from '@codeburn/core/providers/forge' + import { extractBashCommands } from '../bash-utils.js' import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js' -import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' - -type ConversationRow = { - conversation_id: string - title: string | null - workspace_id: number | string - context: string | null - created_at: string | null - updated_at: string | null -} +import { createBridgedProvider } from './bridge.js' +import type { ParsedProviderCall, Provider, SessionSource } from './types.js' type DiscoveryRow = { conversation_id: string @@ -21,18 +16,6 @@ type DiscoveryRow = { workspace_id: string } -type ContextMessage = { - message?: { - text?: { - role?: unknown - content?: unknown - model?: unknown - tool_calls?: unknown - } - } - usage?: unknown -} - const DEFAULT_DB_PATH = join(homedir(), '.forge', '.forge.db') function validateSchema(db: SqliteDatabase): boolean { @@ -44,186 +27,36 @@ function validateSchema(db: SqliteDatabase): boolean { } } -function sqliteTimestampToIso(value: string | null | undefined): string { - if (!value) return new Date(0).toISOString() - - const match = value.match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?$/) - if (match) { - const ms = (match[3] ?? '').padEnd(3, '0').slice(0, 3) - const parsed = new Date(`${match[1]}T${match[2]}.${ms}Z`) - if (!Number.isNaN(parsed.getTime())) return parsed.toISOString() - } - - const parsed = new Date(value) - return Number.isNaN(parsed.getTime()) ? new Date(0).toISOString() : parsed.toISOString() -} - -function actual(value: unknown): number { - if (!value || typeof value !== 'object') return 0 - const raw = (value as Record)['actual'] - return typeof raw === 'number' && Number.isFinite(raw) ? raw : 0 -} - -function usageActual(usage: unknown, key: string): number { - if (!usage || typeof usage !== 'object') return 0 - return actual((usage as Record)[key]) -} - -function mapToolName(name: string): string { - switch (name) { - case 'shell': - case 'bash': - return 'Bash' - case 'read': - case 'Read': - return 'Read' - case 'write': - case 'Write': - return 'Write' - case 'patch': - case 'Edit': - case 'edit': - return 'Edit' - case 'fs_search': - case 'grep': - return 'Grep' - case 'task': - case 'dispatch_agent': - return 'Agent' - default: - return name - } -} - -function pushUnique(values: string[], value: string): void { - if (!values.includes(value)) values.push(value) -} - -function toolCalls(value: unknown): Record[] { - return Array.isArray(value) ? value.filter(v => v && typeof v === 'object') as Record[] : [] -} - -function extractToolsAndCommands(calls: Record[]): { tools: string[]; bashCommands: string[]; firstCallId?: string } { - const tools: string[] = [] - const bashCommands: string[] = [] - let firstCallId: string | undefined - - for (const call of calls) { - const rawName = call['name'] - if (typeof rawName !== 'string') continue - if (!firstCallId && typeof call['call_id'] === 'string') firstCallId = call['call_id'] - - const tool = mapToolName(rawName) - pushUnique(tools, tool) - - if (tool === 'Bash') { - const args = call['arguments'] - if (args && typeof args === 'object') { - const command = (args as Record)['command'] - if (typeof command === 'string') { - for (const cmd of extractBashCommands(command)) pushUnique(bashCommands, cmd) - } - } - } - } - - return { tools, bashCommands, firstCallId } -} - function splitSourcePath(path: string): { dbPath: string; conversationId: string } | null { const idx = path.lastIndexOf(':') if (idx < 0) return null return { dbPath: path.slice(0, idx), conversationId: path.slice(idx + 1) } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts +// pricing pass fills `costUSD` from the token buckets. Bash base-name +// extraction (and its `strip-ansi` dependency) stays CLI-side: the core +// decoder carries the raw command strings; the host reduces them here. +function toProviderCall(rich: ForgeDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - if (!isSqliteAvailable()) { - process.stderr.write(getSqliteLoadError() + '\n') - return - } - - const split = splitSourcePath(source.path) - if (!split) return - - let db: SqliteDatabase - try { - db = openDatabase(split.dbPath) - } catch { - return - } - - try { - if (!validateSchema(db)) return - const rows = db.query( - `SELECT conversation_id, title, CAST(workspace_id AS TEXT) AS workspace_id, context, created_at, updated_at - FROM conversations - WHERE conversation_id = ?`, - [split.conversationId], - ) - const row = rows[0] - if (!row?.context) return - - let parsed: unknown - try { - parsed = JSON.parse(row.context) - } catch { - return - } - const messages = Array.isArray((parsed as { messages?: unknown }).messages) - ? (parsed as { messages: ContextMessage[] }).messages - : [] - - let userMessage = '' - for (let i = 0; i < messages.length; i++) { - const text = messages[i]?.message?.text - const role = typeof text?.role === 'string' ? text.role.toLowerCase() : '' - const content = typeof text?.content === 'string' ? text.content : '' - - if (role === 'user') { - userMessage = content.length > 500 ? content.slice(0, 500) : content - continue - } - if (role !== 'assistant') continue - - const promptTokens = usageActual(messages[i]?.usage, 'prompt_tokens') - const outputTokens = usageActual(messages[i]?.usage, 'completion_tokens') - const cachedInputTokens = usageActual(messages[i]?.usage, 'cached_tokens') - const inputTokens = Math.max(0, promptTokens - cachedInputTokens) - if (inputTokens === 0 && outputTokens === 0) continue - - const model = typeof text?.model === 'string' ? text.model : 'unknown' - const calls = toolCalls(text?.tool_calls) - const { tools, bashCommands, firstCallId } = extractToolsAndCommands(calls) - const stableId = firstCallId ?? `${model}:${promptTokens}:${outputTokens}:${i}` - const deduplicationKey = `forge:${row.conversation_id}:${stableId}` - if (seenKeys.has(deduplicationKey)) continue - seenKeys.add(deduplicationKey) - yield { - provider: 'forge', - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: cachedInputTokens, - cachedInputTokens, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - tools, - bashCommands, - timestamp: sqliteTimestampToIso(row.updated_at ?? row.created_at), - speed: 'standard', - deduplicationKey, - userMessage, - sessionId: row.conversation_id, - } - } - } finally { - db.close() - } - }, + provider: 'forge', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))], + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } @@ -257,7 +90,7 @@ async function discoverFromDb(dbPath: string): Promise { } export function createForgeProvider(dbPath = DEFAULT_DB_PATH): Provider { - return { + return createBridgedProvider({ name: 'forge', displayName: 'Forge', @@ -274,10 +107,43 @@ export function createForgeProvider(dbPath = DEFAULT_DB_PATH): Provider { return discoverFromDb(dbPath) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: open the db and run the conversation-row query (sqlite-side). + // The `context` JSON blob is handed to the core decoder still serialized; + // parsing it is decode logic. + async readRecords(source: SessionSource): Promise { + if (!isSqliteAvailable()) { + process.stderr.write(getSqliteLoadError() + '\n') + return null + } + + const split = splitSourcePath(source.path) + if (!split) return null + + let db: SqliteDatabase + try { + db = openDatabase(split.dbPath) + } catch { + return null + } + + try { + if (!validateSchema(db)) return null + const rows = db.query( + `SELECT conversation_id, title, CAST(workspace_id AS TEXT) AS workspace_id, context, created_at, updated_at + FROM conversations + WHERE conversation_id = ?`, + [split.conversationId], + ) + if (rows.length === 0) return null + return [rows[0]!] + } finally { + db.close() + } }, - } + + decode: decodeForge, + toProviderCall, + }) } export const forge = createForgeProvider() diff --git a/packages/cli/src/providers/goose.ts b/packages/cli/src/providers/goose.ts index ca149e8c..dacbbe39 100644 --- a/packages/cli/src/providers/goose.ts +++ b/packages/cli/src/providers/goose.ts @@ -1,13 +1,16 @@ import { join } from 'path' import { homedir, platform } from 'os' +import { decodeGoose, gooseToolNameMap } from '@codeburn/core/providers/goose' +import type { GooseDecodedCall, GooseMessageRow, GooseSessionRecords, GooseSessionRow } from '@codeburn/core/providers/goose' + import { getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js' -import type { ToolCall } from '../types.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' -type SessionRow = { +type RawSessionRow = { id: string name: string working_dir: string | null @@ -19,33 +22,6 @@ type SessionRow = { model_config_json: Uint8Array | string | null } -type ModelConfig = { - model_name?: string - reasoning?: boolean -} - -type MessageRow = { - message_id: string - role: string - content_json: Uint8Array | string - created_timestamp: number -} - -type ContentItem = { - type: string - toolCall?: { value?: { name?: string; arguments?: Record } } -} - -const toolNameMap: Record = { - developer__shell: 'Bash', - developer__text_editor: 'Edit', - developer__read_file: 'Read', - developer__write_file: 'Write', - developer__list_directory: 'LS', - developer__search_files: 'Grep', - computercontroller__shell: 'Bash', -} - function sanitize(dir: string): string { return dir.replace(/^\//, '').replace(/\//g, '-') } @@ -72,157 +48,37 @@ function validateSchema(db: SqliteDatabase): boolean { } } -function parseModelConfig(raw: string | null): ModelConfig { - if (!raw) return {} - try { - return JSON.parse(raw) as ModelConfig - } catch { - return {} - } -} - -function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tools: string[]; bashCommands: string[]; toolSequence: ToolCall[][] } { - const tools: string[] = [] - const bashCommands: string[] = [] - const seen = new Set() - const toolSequence: ToolCall[][] = [] - - try { - const rows = db.query<{ content_json: Uint8Array | string }>( - "SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'assistant' AND content_json LIKE '%toolRequest%' ORDER BY created_timestamp ASC", - [sessionId], - ) - - for (const row of rows) { - let items: ContentItem[] - try { - items = JSON.parse(blobToText(row.content_json)) as ContentItem[] - } catch { - continue - } - const msgCalls: ToolCall[] = [] - for (const item of items) { - if (item.type !== 'toolRequest') continue - const rawName = item.toolCall?.value?.name ?? '' - if (!rawName) continue - const mapped = toolNameMap[rawName] ?? rawName.split('__').pop() ?? rawName - if (!seen.has(mapped)) { - seen.add(mapped) - tools.push(mapped) - } - const call: ToolCall = { tool: mapped } - const args = item.toolCall?.value?.arguments - if (args && typeof args === 'object') { - const fp = (args as Record)['file_path'] - if (typeof fp === 'string') call.file = fp - const cmd = (args as Record)['command'] - if (typeof cmd === 'string') call.command = cmd - } - msgCalls.push(call) - if (mapped === 'Bash') { - const cmd = item.toolCall?.value?.arguments?.command - if (typeof cmd === 'string') { - for (const c of extractBashCommands(cmd)) { - if (!bashCommands.includes(c)) bashCommands.push(c) - } - } - } - } - if (msgCalls.length > 0) toolSequence.push(msgCalls) - } - } catch { /* best-effort */ } - - return { tools, bashCommands, toolSequence } -} - -function getFirstUserMessage(db: SqliteDatabase, sessionId: string): string { - try { - const rows = db.query<{ content_json: Uint8Array | string }>( - "SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'user' ORDER BY created_timestamp ASC LIMIT 1", - [sessionId], - ) - if (rows.length === 0) return '' - const items = JSON.parse(blobToText(rows[0]!.content_json)) as ContentItem[] - const text = items.find(i => i.type === 'text') as { text?: string } | undefined - return (text?.text ?? '').slice(0, 500) - } catch { - return '' - } -} +const SESSION_COLUMNS = 'id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json' -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts +// pricing pass fills `costUSD` from the token buckets. Bash base-name +// extraction (and its `strip-ansi` dependency) stays CLI-side: the core +// decoder carries the raw command strings; the host reduces them here. +function toProviderCall(rich: GooseDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - if (!isSqliteAvailable()) { - process.stderr.write(getSqliteLoadError() + '\n') - return - } - - const segments = source.path.split(':') - const sessionId = segments[segments.length - 1]! - const dbPath = segments.slice(0, -1).join(':') - - let db: SqliteDatabase - try { - db = openDatabase(dbPath) - } catch (err) { - process.stderr.write(`codeburn: cannot open Goose database: ${err instanceof Error ? err.message : err}\n`) - return - } - - try { - if (!validateSchema(db)) return - - const rows = db.query( - 'SELECT id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json FROM sessions WHERE id = ?', - [sessionId], - ) - if (rows.length === 0) return - - const session = rows[0]! - const inputTokens = session.accumulated_input_tokens ?? 0 - const outputTokens = session.accumulated_output_tokens ?? 0 - if (inputTokens === 0 && outputTokens === 0) return - - const dedupKey = `goose:${sessionId}` - if (seenKeys.has(dedupKey)) return - seenKeys.add(dedupKey) - - const config = parseModelConfig(blobToText(session.model_config_json)) - const model = config.model_name ?? 'unknown' - - const { tools, bashCommands, toolSequence } = extractToolsFromMessages(db, sessionId) - const userMessage = getFirstUserMessage(db, sessionId) - - const raw = session.updated_at || session.created_at || '' - let ts = new Date(raw) - if (isNaN(ts.getTime())) ts = new Date(raw + 'Z') - if (isNaN(ts.getTime())) ts = new Date() - - yield { - provider: 'goose', - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - tools, - bashCommands, - toolSequence: toolSequence.length > 1 ? toolSequence : undefined, - timestamp: ts.toISOString(), - speed: 'standard', - deduplicationKey: dedupKey, - userMessage, - sessionId, - } - } finally { - db.close() - } - }, + provider: 'goose', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))], + toolSequence: rich.toolSequence, + // The pre-migration decode fell back to `new Date()` (the current time) + // when both updated_at/created_at were unparseable. That clock read can't + // live in the pure core decoder, so decodeGoose emits '' in that case and + // the fallback is applied here instead. + timestamp: rich.timestamp || new Date().toISOString(), + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } @@ -235,8 +91,8 @@ async function discoverFromDb(dbPath: string): Promise { } try { - const rows = db.query( - 'SELECT id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json FROM sessions ORDER BY updated_at DESC', + const rows = db.query( + `SELECT ${SESSION_COLUMNS} FROM sessions ORDER BY updated_at DESC`, ) return rows @@ -262,7 +118,7 @@ const modelDisplayNames: Record = { } export function createGooseProvider(): Provider { - return { + return createBridgedProvider({ name: 'goose', displayName: 'Goose', @@ -271,7 +127,7 @@ export function createGooseProvider(): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return gooseToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { @@ -280,10 +136,81 @@ export function createGooseProvider(): Provider { return discoverFromDb(dbPath) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: open the db and run the session query, the assistant + // tool-message query, and the first-user-message query (all sqlite-side), + // converting each BLOB column to text (the same charset-safe conversion + // fs-utils performs for file reads). The core decoder gets one composite + // record bundling all three; JSON parsing and per-message decode are pure + // and happen there. + async readRecords(source: SessionSource): Promise { + if (!isSqliteAvailable()) { + process.stderr.write(getSqliteLoadError() + '\n') + return null + } + + const segments = source.path.split(':') + const sessionId = segments[segments.length - 1]! + const dbPath = segments.slice(0, -1).join(':') + + let db: SqliteDatabase + try { + db = openDatabase(dbPath) + } catch (err) { + process.stderr.write(`codeburn: cannot open Goose database: ${err instanceof Error ? err.message : err}\n`) + return null + } + + try { + if (!validateSchema(db)) return null + + const rows = db.query( + `SELECT ${SESSION_COLUMNS} FROM sessions WHERE id = ?`, + [sessionId], + ) + if (rows.length === 0) return null + const raw = rows[0]! + + const session: GooseSessionRow = { + id: raw.id, + workingDir: raw.working_dir, + createdAt: raw.created_at, + updatedAt: raw.updated_at, + accumulatedInputTokens: raw.accumulated_input_tokens, + accumulatedOutputTokens: raw.accumulated_output_tokens, + modelConfigJson: blobToText(raw.model_config_json) || null, + } + + let assistantToolMessages: GooseMessageRow[] = [] + let firstUserMessage: GooseMessageRow | null = null + try { + const toolRows = db.query<{ content_json: Uint8Array | string }>( + "SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'assistant' AND content_json LIKE '%toolRequest%' ORDER BY created_timestamp ASC", + [sessionId], + ) + assistantToolMessages = toolRows.map(r => ({ contentJson: blobToText(r.content_json) })) + } catch { + // best-effort, matches the pre-migration decode + } + try { + const userRows = db.query<{ content_json: Uint8Array | string }>( + "SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'user' ORDER BY created_timestamp ASC LIMIT 1", + [sessionId], + ) + if (userRows.length > 0) firstUserMessage = { contentJson: blobToText(userRows[0]!.content_json) } + } catch { + // best-effort, matches the pre-migration decode + } + + const record: GooseSessionRecords = { sessionId, session, assistantToolMessages, firstUserMessage } + return [record] + } finally { + db.close() + } }, - } + + decode: decodeGoose, + toProviderCall, + }) } export const goose = createGooseProvider() diff --git a/packages/cli/src/providers/zcode.ts b/packages/cli/src/providers/zcode.ts index 6ffe757d..23aaf2e0 100644 --- a/packages/cli/src/providers/zcode.ts +++ b/packages/cli/src/providers/zcode.ts @@ -1,8 +1,12 @@ import { join } from 'path' import { homedir } from 'os' +import { decodeZcode } from '@codeburn/core/providers/zcode' +import type { ZcodeDecodedCall, ZcodeSessionRecords, ZcodeToolRow, ZcodeUsageRow } from '@codeburn/core/providers/zcode' + import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' /// ZCode (CLI v0.14.x) records usage in a single SQLite database at /// ~/.zcode/cli/db/db.sqlite. We read it because the other on-disk sources are @@ -16,24 +20,6 @@ type SessionRow = { directory: string } -type UsageRow = { - id: string - turn_id: string | null - model_id: string - input_tokens: number - output_tokens: number - reasoning_tokens: number - cache_creation_input_tokens: number - cache_read_input_tokens: number - started_at: number - completed_at: number | null -} - -type ToolRow = { - turn_id: string | null - tool_name: string -} - function getDbPath(override?: string): string { return override ?? join(homedir(), '.zcode', 'cli', 'db', 'db.sqlite') } @@ -42,11 +28,6 @@ function sanitizeProject(path: string): string { return path.replace(/^\//, '').replace(/\//g, '-') } -function epochMsToIso(ms: number | null): string { - if (ms === null || !Number.isFinite(ms) || ms <= 0) return new Date(0).toISOString() - return new Date(ms).toISOString() -} - function validateSchema(db: SqliteDatabase): boolean { try { db.query<{ cnt: number }>('SELECT COUNT(*) as cnt FROM model_usage LIMIT 1') @@ -85,51 +66,93 @@ function discover(dbPath: string): SessionSource[] { } } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts +// pricing pass fills `costUSD` from the token buckets. ZCode never captures a +// user message, so it is hardcoded empty here rather than carried through the +// rich decode. +function toProviderCall(rich: ZcodeDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { + provider: 'zcode', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: [], + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + turnId: rich.turnId, + userMessage: '', + sessionId: rich.sessionId, + } +} + +export function createZcodeProvider(dbPathOverride?: string): Provider { + const dbPath = getDbPath(dbPathOverride) + return createBridgedProvider({ + name: 'zcode', + displayName: 'ZCode', + + modelDisplayName(model: string): string { + return model + }, + + toolDisplayName(rawTool: string): string { + return rawTool + }, + + async discoverSessions(): Promise { + if (!isSqliteAvailable()) return [] + return discover(dbPath) + }, + + // I/O adapter: open the db, run the model_usage and tool_usage queries for + // this session (both sqlite-side), and hand the core decoder one combined + // record bundling both row sets. + async readRecords(source: SessionSource): Promise { if (!isSqliteAvailable()) { process.stderr.write(getSqliteLoadError() + '\n') - return + return null } // Source paths are `:`. Split from the right so a colon // in the path (Windows drive letter) doesn't corrupt the session id. const segments = source.path.split(':') const sessionId = segments[segments.length - 1]! - const dbPath = segments.slice(0, -1).join(':') + const readDbPath = segments.slice(0, -1).join(':') let db: SqliteDatabase try { - db = openDatabase(dbPath) + db = openDatabase(readDbPath) } catch (err) { process.stderr.write( `codeburn: cannot open ZCode database: ${err instanceof Error ? err.message : err}\n`, ) - return + return null } try { - if (!validateSchema(db)) return + if (!validateSchema(db)) return null // model_usage rows don't link to individual tool calls, only to a turn, // so collect each turn's tools and attach them to one request per turn - // (below) to avoid double-counting across a turn's multiple requests. - const toolRows = db.query( + // (in the decoder) to avoid double-counting across a turn's multiple + // requests. + const toolRows = db.query( `SELECT turn_id, tool_name FROM tool_usage WHERE session_id = ? AND turn_id IS NOT NULL ORDER BY started_at ASC`, [sessionId], ) - const toolsByTurn = new Map() - for (const tool of toolRows) { - if (!tool.turn_id) continue - const list = toolsByTurn.get(tool.turn_id) ?? [] - list.push(tool.tool_name) - toolsByTurn.set(tool.turn_id, list) - } - - const rows = db.query( + + const usageRows = db.query( `SELECT id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at FROM model_usage WHERE session_id = ? @@ -137,89 +160,16 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars [sessionId], ) - const turnsWithToolsEmitted = new Set() - - for (const row of rows) { - const cacheRead = row.cache_read_input_tokens ?? 0 - const cacheCreation = row.cache_creation_input_tokens ?? 0 - const output = row.output_tokens ?? 0 - const reasoning = row.reasoning_tokens ?? 0 - // ZCode folds cached tokens into input_tokens (OpenAI-style). Split - // them back out so fresh input bills at the input rate and cached at - // the cache-read rate, matching the pricing table's Anthropic-style - // semantics. - const freshInput = Math.max(0, (row.input_tokens ?? 0) - cacheRead - cacheCreation) - - if (freshInput === 0 && output === 0 && reasoning === 0 && cacheRead === 0 && cacheCreation === 0) { - continue - } - - const dedupKey = `zcode:${row.id}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - let tools: string[] = [] - if (row.turn_id && !turnsWithToolsEmitted.has(row.turn_id)) { - const turnTools = toolsByTurn.get(row.turn_id) - if (turnTools && turnTools.length > 0) { - tools = turnTools - turnsWithToolsEmitted.add(row.turn_id) - } - } - - const model = row.model_id - - yield { - provider: 'zcode', - model, - inputTokens: freshInput, - outputTokens: output, - cacheCreationInputTokens: cacheCreation, - cacheReadInputTokens: cacheRead, - cachedInputTokens: 0, - reasoningTokens: reasoning, - webSearchRequests: 0, - costBasis: 'estimated', - tools, - bashCommands: [], - timestamp: epochMsToIso(row.completed_at ?? row.started_at), - speed: 'standard', - deduplicationKey: dedupKey, - turnId: row.turn_id ?? undefined, - userMessage: '', - sessionId, - } - } + const record: ZcodeSessionRecords = { sessionId, usageRows, toolRows } + return [record] } finally { db.close() } }, - } -} - -export function createZcodeProvider(dbPathOverride?: string): Provider { - const dbPath = getDbPath(dbPathOverride) - return { - name: 'zcode', - displayName: 'ZCode', - - modelDisplayName(model: string): string { - return model - }, - toolDisplayName(rawTool: string): string { - return rawTool - }, - - async discoverSessions(): Promise { - if (!isSqliteAvailable()) return [] - return discover(dbPath) - }, - - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) - }, - } + decode: decodeZcode, + toProviderCall, + }) } export const zcode = createZcodeProvider() diff --git a/packages/cli/src/providers/zed.ts b/packages/cli/src/providers/zed.ts index 5951c0e3..45ed8f17 100644 --- a/packages/cli/src/providers/zed.ts +++ b/packages/cli/src/providers/zed.ts @@ -3,16 +3,24 @@ import { join } from 'path' import { homedir } from 'os' import zlib from 'zlib' +import { decodeZed } from '@codeburn/core/providers/zed' +import type { ZedDecodedCall, ZedThreadRow } from '@codeburn/core/providers/zed' + import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js' -import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { ParsedProviderCall, Provider, SessionSource } from './types.js' // Zed's built-in agent stores one row per thread in a single SQLite database; // the `data` blob is zstd-compressed JSON carrying `request_token_usage` -// (per-request Anthropic-shaped token counts) and the thread's model. -// Format documented in issue #480. +// (per-request Anthropic-shaped token counts) and the thread's model. The +// sqlite driver and SQL query stay CLI-side; decompression, JSON parsing, and +// per-request token accounting moved to @codeburn/core/providers/zed. Format +// documented in issue #480. // zstd landed in node:zlib in 22.15 / 23.8; the package floor is 22.13, so the -// provider degrades with a notice instead of assuming the export exists. +// provider degrades with a notice instead of assuming the export exists. This +// Node-version capability check is host-side by nature and stays here — the +// core decoder is only ever called once this has confirmed support exists. const zstdDecompress = (zlib as { zstdDecompressSync?: (buf: Buffer) => Buffer }).zstdDecompressSync function getZedThreadsDbPath(): string { @@ -31,154 +39,64 @@ const THREADS_QUERY = ` ORDER BY updated_at ASC ` -type ThreadRow = { - id: string - summary: string | null - updated_at: string | null - data_type: string | null - data: Uint8Array | null -} - -type TokenUsage = { - input_tokens?: number - output_tokens?: number - cache_creation_input_tokens?: number - cache_read_input_tokens?: number -} - -type ThreadJson = { - model?: { provider?: string; model?: string } - request_token_usage?: Record - cumulative_token_usage?: TokenUsage -} - -function num(value: number | undefined): number { - return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 -} - -function usageIsEmpty(usage: TokenUsage): boolean { - return ( - num(usage.input_tokens) === 0 && - num(usage.output_tokens) === 0 && - num(usage.cache_creation_input_tokens) === 0 && - num(usage.cache_read_input_tokens) === 0 - ) -} - -function buildCall(opts: { - threadId: string - requestKey: string - usage: TokenUsage - model: string - timestamp: string - userMessage: string -}): ParsedProviderCall { - const input = num(opts.usage.input_tokens) - const output = num(opts.usage.output_tokens) - const cacheWrite = num(opts.usage.cache_creation_input_tokens) - const cacheRead = num(opts.usage.cache_read_input_tokens) +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts +// pricing pass fills `costUSD` from the token buckets. Zed never captures tool +// calls, so `bashCommands` is always empty (no extractBashCommands needed). +function toProviderCall(rich: ZedDecodedCall): ParsedProviderCall { return { provider: 'zed', - model: opts.model, - inputTokens: input, - outputTokens: output, - cacheCreationInputTokens: cacheWrite, - cacheReadInputTokens: cacheRead, - cachedInputTokens: cacheRead, - reasoningTokens: 0, - webSearchRequests: 0, + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, costBasis: 'estimated', - tools: [], + tools: rich.tools, bashCommands: [], - timestamp: opts.timestamp, - speed: 'standard', - deduplicationKey: `zed:${opts.threadId}:${opts.requestKey}`, - userMessage: opts.userMessage, - sessionId: opts.threadId, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } -function parseThreads(db: SqliteDatabase, seenKeys: Set): ParsedProviderCall[] { - const calls: ParsedProviderCall[] = [] - let skipped = 0 - - let rows: ThreadRow[] - try { - rows = db.query(THREADS_QUERY) - } catch { - return calls - } - - for (const row of rows) { - try { - // Zed's DataType enum is "zstd" (current save path) or "json" (legacy - // uncompressed rows); anything else is unknown. - if (!row.id || !row.data || (row.data_type !== 'zstd' && row.data_type !== 'json')) { - if (row.data != null) skipped++ - continue - } - const parsedAt = new Date(row.updated_at ?? '') - if (Number.isNaN(parsedAt.getTime())) continue - const timestamp = parsedAt.toISOString() - - const jsonText = row.data_type === 'zstd' - ? zstdDecompress!(Buffer.from(row.data)).toString('utf-8') - : Buffer.from(row.data).toString('utf-8') - const thread = JSON.parse(jsonText) as ThreadJson - const model = thread.model?.model || 'unknown' - const userMessage = row.summary ?? '' +export function createZedProvider(dbPathOverride?: string): Provider { + return createBridgedProvider({ + name: 'zed', + displayName: 'Zed', - const requests = Object.entries(thread.request_token_usage ?? {}).filter(([, usage]) => usage != null && !usageIsEmpty(usage)) - // The per-request map is keyed by user message and does not cover every - // request (verified on a real thread: cumulative was ~3x the map sum), - // so a remainder entry tops the thread up to the exact cumulative - // counter. Threads with an empty map degrade to one cumulative call. - const entries: Array<[string, TokenUsage]> = [...requests] - const cumulative = thread.cumulative_token_usage - if (cumulative && !usageIsEmpty(cumulative)) { - let sumIn = 0, sumOut = 0, sumWrite = 0, sumRead = 0 - for (const [, usage] of requests) { - sumIn += num(usage.input_tokens) - sumOut += num(usage.output_tokens) - sumWrite += num(usage.cache_creation_input_tokens) - sumRead += num(usage.cache_read_input_tokens) - } - const remainder: TokenUsage = { - input_tokens: Math.max(0, num(cumulative.input_tokens) - sumIn), - output_tokens: Math.max(0, num(cumulative.output_tokens) - sumOut), - cache_creation_input_tokens: Math.max(0, num(cumulative.cache_creation_input_tokens) - sumWrite), - cache_read_input_tokens: Math.max(0, num(cumulative.cache_read_input_tokens) - sumRead), - } - if (!usageIsEmpty(remainder)) entries.push(['cumulative-remainder', remainder]) - } + modelDisplayName(model: string): string { + return model + }, - for (const [requestKey, usage] of entries) { - const call = buildCall({ threadId: row.id, requestKey, usage, model, timestamp, userMessage }) - if (seenKeys.has(call.deduplicationKey)) continue - seenKeys.add(call.deduplicationKey) - calls.push(call) - } - } catch { - skipped++ - } - } + toolDisplayName(rawTool: string): string { + return rawTool + }, - if (skipped > 0) { - process.stderr.write(`codeburn: skipped ${skipped} unreadable Zed threads\n`) - } - return calls -} + async discoverSessions(): Promise { + if (!isSqliteAvailable()) return [] + const dbPath = dbPathOverride ?? getZedThreadsDbPath() + if (!existsSync(dbPath)) return [] + return [{ path: dbPath, project: 'zed', provider: 'zed' }] + }, -function createParser(source: SessionSource, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { + // I/O adapter: open the db and run the threads query (sqlite-side). Rows + // (blob and all) are handed straight to the core decoder, which does the + // zstd decompression and JSON parsing now that Node-version support for it + // is confirmed. + async readRecords(source: SessionSource): Promise { if (!isSqliteAvailable()) { process.stderr.write(getSqliteLoadError() + '\n') - return + return null } if (!zstdDecompress) { process.stderr.write('codeburn: Zed threads need Node >= 22.15 (zstd support); skipping Zed usage.\n') - return + return null } let db: SqliteDatabase @@ -186,43 +104,30 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars db = openDatabase(source.path) } catch (err) { process.stderr.write(`codeburn: cannot open Zed database: ${err instanceof Error ? err.message : err}\n`) - return + return null } try { - for (const call of parseThreads(db, seenKeys)) { - yield call - } + return db.query(THREADS_QUERY) + } catch { + return [] } finally { db.close() } }, - } -} - -export function createZedProvider(dbPathOverride?: string): Provider { - return { - name: 'zed', - displayName: 'Zed', - - modelDisplayName(model: string): string { - return model - }, - - toolDisplayName(rawTool: string): string { - return rawTool - }, - async discoverSessions(): Promise { - if (!isSqliteAvailable()) return [] - const dbPath = dbPathOverride ?? getZedThreadsDbPath() - if (!existsSync(dbPath)) return [] - return [{ path: dbPath, project: 'zed', provider: 'zed' }] - }, - - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // The core decoder reports unreadable/unknown-shape threads as record + // diagnostics, which the bridge discards. The pre-migration decode printed + // one aggregate stderr line per scan for exactly those rows, so that notice + // is re-emitted here rather than silently dropped. + decode(input) { + const { calls, diagnostics } = decodeZed(input) + if (diagnostics.length > 0) { + process.stderr.write(`codeburn: skipped ${diagnostics.length} unreadable Zed threads\n`) + } + return { calls } }, - } + toProviderCall, + }) } export const zed = createZedProvider() diff --git a/packages/cli/tests/providers/crush-bridge.test.ts b/packages/cli/tests/providers/crush-bridge.test.ts new file mode 100644 index 00000000..2a3ac465 --- /dev/null +++ b/packages/cli/tests/providers/crush-bridge.test.ts @@ -0,0 +1,212 @@ +import { mkdtemp, mkdir, writeFile, rm } from 'fs/promises' +import { mkdirSync } from 'fs' +import { join } from 'path' +import { tmpdir } from 'os' +import { createRequire } from 'node:module' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { createCrushProvider } from '../../src/providers/crush.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +// Byte-identical parity gate for the crush bridge migration (phase 8, Category +// B / sqlite). Crush is not present in the frozen corpus, so a committed +// fixture golden is THE parity gate: the bridged provider (sqlite driver + SQL +// queries CLI-side, pure row->call decode delegated to +// @codeburn/core/providers/crush) must reproduce exactly what the +// pre-migration in-CLI decode produced for this fixture DB. The GOLDEN below +// was captured from the legacy provider before the migration. + +const requireForTest = createRequire(import.meta.url) + +// CREATE TABLE statements taken verbatim from charmbracelet/crush@v0.66.1 +// internal/db/migrations/20250424200609_initial.sql (same fixture shape as +// tests/providers/crush.test.ts). +function createCrushDb(dir: string): string { + mkdirSync(dir, { recursive: true }) + const dbPath = join(dir, 'crush.db') + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + db.exec(` + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + parent_session_id TEXT, + title TEXT NOT NULL, + message_count INTEGER NOT NULL DEFAULT 0 CHECK (message_count >= 0), + prompt_tokens INTEGER NOT NULL DEFAULT 0 CHECK (prompt_tokens >= 0), + completion_tokens INTEGER NOT NULL DEFAULT 0 CHECK (completion_tokens >= 0), + cost REAL NOT NULL DEFAULT 0.0 CHECK (cost >= 0.0), + updated_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + summary_message_id TEXT, + todos TEXT + ) + `) + db.exec(` + CREATE TABLE IF NOT EXISTS messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + parts TEXT NOT NULL DEFAULT '[]', + model TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER, + provider TEXT, + is_summary_message INTEGER DEFAULT 0 NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions (id) ON DELETE CASCADE + ) + `) + db.close() + return dbPath +} + +function seed(dbPath: string): void { + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + try { + db.prepare(` + INSERT INTO sessions (id, parent_session_id, title, message_count, prompt_tokens, completion_tokens, cost, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run('sess-alpha', null, 'test session', 3, 1500, 420, 0.0921, 1_700_000_100, 1_700_000_555) + for (const [id, model, ts] of [['m1', 'claude-sonnet-4-6', 1_700_000_100], ['m2', 'claude-sonnet-4-6', 1_700_000_200], ['m3', 'gpt-5', 1_700_000_300]] as const) { + db.prepare(`INSERT INTO messages (id, session_id, role, parts, model, created_at, updated_at) VALUES (?, ?, ?, '[]', ?, ?, ?)`) + .run(id, 'sess-alpha', 'assistant', model, ts, ts) + } + // No cost -> estimated fallback. + db.prepare(` + INSERT INTO sessions (id, parent_session_id, title, message_count, prompt_tokens, completion_tokens, cost, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run('sess-beta', null, 'test session 2', 1, 300, 80, 0, 1_700_000_600, 1_700_000_600) + } finally { + db.close() + } +} + +// Captured from the pre-migration crush decode (dominant model wins among +// tied assistant messages, measured cost from the DB, estimated fallback when +// cost is 0). `costUSD` in the GOLDEN below is the raw decoder output +// (pre-pricing-pass); the second test proves the pricing pass adds costUSD to +// the estimated row and leaves the measured row untouched. +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'crush', + model: 'claude-sonnet-4-6', + inputTokens: 1500, + outputTokens: 420, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costUSD: 0.0921, + costBasis: 'measured', + tools: [], + bashCommands: [], + timestamp: '2023-11-14T22:22:35.000Z', + speed: 'standard', + deduplicationKey: 'crush:sess-alpha', + userMessage: '', + sessionId: 'sess-alpha', + }, + { + provider: 'crush', + model: 'unknown', + inputTokens: 300, + outputTokens: 80, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2023-11-14T22:23:20.000Z', + speed: 'standard', + deduplicationKey: 'crush:sess-beta', + userMessage: '', + sessionId: 'sess-beta', + }, +] + +let tmpRoot: string + +beforeEach(async () => { + tmpRoot = await mkdtemp(join(tmpdir(), 'crush-bridge-')) +}) + +afterEach(async () => { + await rm(tmpRoot, { recursive: true, force: true }) +}) + +async function collect(): Promise { + const projectDir = join(tmpRoot, 'project-alpha') + const dbPath = createCrushDb(join(projectDir, '.crush')) + seed(dbPath) + + const globalData = join(tmpRoot, 'crush-global') + await mkdir(globalData, { recursive: true }) + await writeFile(join(globalData, 'projects.json'), JSON.stringify({ + 'proj-a': { path: projectDir, data_dir: '.crush' }, + })) + process.env['CRUSH_GLOBAL_DATA'] = globalData + + const provider = createCrushProvider() + const sources = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe.skipIf(!isSqliteAvailable())('crush bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD (for the estimated row) filled in', async () => { + const raw = await collect() + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + const { costUSD, ...rest } = call + const { costUSD: rawCostUSD, ...rawRest } = raw[i]! + expect(rest).toEqual(rawRest) + // The measured row's costUSD is untouched; the estimated row's is filled in. + if (raw[i]!.costBasis === 'measured') expect(costUSD).toBe(rawCostUSD) + }) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const projectDir = join(tmpRoot, 'project-alpha') + const dbPath = createCrushDb(join(projectDir, '.crush')) + seed(dbPath) + const globalData = join(tmpRoot, 'crush-global') + await mkdir(globalData, { recursive: true }) + await writeFile(join(globalData, 'projects.json'), JSON.stringify({ 'proj-a': { path: projectDir, data_dir: '.crush' } })) + process.env['CRUSH_GLOBAL_DATA'] = globalData + + const provider = createCrushProvider() + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(2) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/forge-bridge.test.ts b/packages/cli/tests/providers/forge-bridge.test.ts new file mode 100644 index 00000000..92fc1093 --- /dev/null +++ b/packages/cli/tests/providers/forge-bridge.test.ts @@ -0,0 +1,219 @@ +import { mkdtemp, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' +import { createRequire } from 'node:module' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { createForgeProvider } from '../../src/providers/forge.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +// Byte-identical parity gate for the forge bridge migration (phase 8, Category +// B / sqlite). Forge is not present in the frozen corpus, so a committed +// fixture golden is THE parity gate. The GOLDEN below was captured from the +// legacy provider before the migration. + +const requireForTest = createRequire(import.meta.url) + +type TestDb = { + exec(sql: string): void + prepare(sql: string): { run(...params: unknown[]): void } + close(): void +} + +function createForgeDb(dir: string): string { + const dbPath = join(dir, 'forge.db') + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + db.exec(`CREATE TABLE conversations( + conversation_id TEXT PRIMARY KEY NOT NULL, title TEXT, workspace_id BIGINT NOT NULL, + context TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP, metrics TEXT + )`) + db.close() + return dbPath +} + +function withTestDb(dbPath: string, fn: (db: TestDb) => void): void { + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + try { + fn(db) + } finally { + db.close() + } +} + +const CONTEXT = { + messages: [ + { message: { text: { role: 'User', content: 'implement forge bridge' } } }, + { + message: { + text: { + role: 'Assistant', content: '', model: 'claude-opus-4-6', + tool_calls: [ + { name: 'shell', call_id: 'call-1', arguments: { command: 'git status && npm test' } }, + { name: 'Read', call_id: 'call-2', arguments: { file_path: '/tmp/a' } }, + ], + }, + }, + usage: { prompt_tokens: { actual: 1200 }, completion_tokens: { actual: 300 }, cached_tokens: { actual: 200 } }, + }, + { message: { text: { role: 'User', content: 'now write tests' } } }, + { + message: { text: { role: 'Assistant', model: 'claude-sonnet-4-6', tool_calls: [{ name: 'unknown_tool', call_id: 'call-3', arguments: {} }] } }, + usage: { prompt_tokens: { actual: 400 }, completion_tokens: { actual: 90 } }, + }, + // Zero-token assistant message: must be skipped. + { message: { text: { role: 'Assistant', model: 'claude-sonnet-4-6' } }, usage: { prompt_tokens: { actual: 0 }, completion_tokens: { actual: 0 } } }, + // Tool names that collide with Object.prototype members. Tool names come + // straight from the conversation JSON, so mapping them through a plain + // object-literal lookup would resolve the INHERITED member (a Function, or + // Object.prototype itself for `__proto__`) instead of falling through to + // the identity default. The pre-migration decode used a `switch` and + // returned each name verbatim; this row pins that. + { + message: { + text: { + role: 'Assistant', model: 'claude-haiku-4-5', + tool_calls: [ + { name: 'constructor', call_id: 'call-4', arguments: {} }, + { name: 'toString', call_id: 'call-5', arguments: {} }, + { name: '__proto__', call_id: 'call-6', arguments: {} }, + { name: 'hasOwnProperty', call_id: 'call-7', arguments: {} }, + ], + }, + }, + usage: { prompt_tokens: { actual: 50 }, completion_tokens: { actual: 5 } }, + }, + ], +} + +function seed(dbPath: string): void { + withTestDb(dbPath, db => { + db.prepare(`INSERT INTO conversations (conversation_id, title, workspace_id, context, created_at, updated_at, metrics) VALUES (?, ?, ?, ?, ?, ?, ?)`) + .run('conv-1', 'Forge Project', 123, JSON.stringify(CONTEXT), '2026-05-06 15:00:00', '2026-05-06 15:20:41.379094', null) + }) +} + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'forge', + model: 'claude-opus-4-6', + inputTokens: 1000, + outputTokens: 300, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 200, + cachedInputTokens: 200, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Bash', 'Read'], + bashCommands: ['git', 'npm'], + timestamp: '2026-05-06T15:20:41.379Z', + speed: 'standard', + deduplicationKey: 'forge:conv-1:call-1', + userMessage: 'implement forge bridge', + sessionId: 'conv-1', + }, + { + provider: 'forge', + model: 'claude-sonnet-4-6', + inputTokens: 400, + outputTokens: 90, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['unknown_tool'], + bashCommands: [], + timestamp: '2026-05-06T15:20:41.379Z', + speed: 'standard', + deduplicationKey: 'forge:conv-1:call-3', + userMessage: 'now write tests', + sessionId: 'conv-1', + }, + { + provider: 'forge', + model: 'claude-haiku-4-5', + inputTokens: 50, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['constructor', 'toString', '__proto__', 'hasOwnProperty'], + bashCommands: [], + timestamp: '2026-05-06T15:20:41.379Z', + speed: 'standard', + deduplicationKey: 'forge:conv-1:call-4', + userMessage: 'now write tests', + sessionId: 'conv-1', + }, +] + +let tmpRoot: string + +beforeEach(async () => { + tmpRoot = await mkdtemp(join(tmpdir(), 'forge-bridge-')) +}) + +afterEach(async () => { + await rm(tmpRoot, { recursive: true, force: true }) +}) + +async function collect(dbPath: string, seen = new Set()): Promise { + const provider = createForgeProvider(dbPath) + const sources = await provider.discoverSessions() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe.skipIf(!isSqliteAvailable())('forge bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const dbPath = createForgeDb(tmpRoot) + seed(dbPath) + expect(await collect(dbPath)).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const dbPath = createForgeDb(tmpRoot) + seed(dbPath) + const raw = await collect(dbPath) + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const dbPath = createForgeDb(tmpRoot) + seed(dbPath) + const seen = new Set() + const first = await collect(dbPath, seen) + const second = await collect(dbPath, seen) + expect(first.length).toBe(3) + expect(second).toEqual([]) + }) + + it('every emitted tool name is a string (no inherited prototype member leaks in)', async () => { + const dbPath = createForgeDb(tmpRoot) + seed(dbPath) + for (const call of await collect(dbPath)) { + for (const tool of call.tools) expect(typeof tool).toBe('string') + } + }) +}) diff --git a/packages/cli/tests/providers/goose-bridge.test.ts b/packages/cli/tests/providers/goose-bridge.test.ts new file mode 100644 index 00000000..09082b6e --- /dev/null +++ b/packages/cli/tests/providers/goose-bridge.test.ts @@ -0,0 +1,178 @@ +import { mkdtemp, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' +import { createRequire } from 'node:module' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { createGooseProvider } from '../../src/providers/goose.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the goose bridge migration (phase 8, Category +// B / sqlite). Goose is not present in the frozen corpus and had no pre- +// existing fixture test, so this fixture-building pattern is modeled on the +// sibling sqlite providers (crush/zed/forge). The GOLDEN below was captured +// from the legacy in-CLI decode before the migration (createGooseProvider has +// no db-path override, so `createSessionParser` is exercised directly against +// a manually-built source, exactly as the capture script did). + +const requireForTest = createRequire(import.meta.url) + +function createGooseDb(dir: string): string { + const dbPath = join(dir, 'sessions.db') + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + db.exec(`CREATE TABLE sessions ( + id TEXT PRIMARY KEY, name TEXT, working_dir TEXT, created_at TEXT, updated_at TEXT, + accumulated_input_tokens INTEGER, accumulated_output_tokens INTEGER, provider_name TEXT, model_config_json BLOB + )`) + db.exec(`CREATE TABLE messages ( + message_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, role TEXT NOT NULL, content_json BLOB NOT NULL, created_timestamp INTEGER NOT NULL + )`) + db.close() + return dbPath +} + +function seed(dbPath: string, opts: { badTimestamps?: boolean; singleTurn?: boolean } = {}): void { + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + try { + const modelConfig = JSON.stringify({ model_name: 'gpt-5.5', reasoning: true }) + db.prepare(`INSERT INTO sessions (id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, model_config_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run('sess-1', 'goose session', '/Users/me/project', + opts.badTimestamps ? 'not-a-date' : '2026-06-01T10:00:00Z', + opts.badTimestamps ? 'also-not-a-date' : '2026-06-01T10:05:30Z', + 1500, 400, 'openai', Buffer.from(modelConfig)) + + db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`) + .run('msg-1', 'sess-1', 'user', Buffer.from(JSON.stringify([{ type: 'text', text: 'please refactor the shell wrapper and run tests' }])), 1_780_000_000) + + const assistantContent1 = [ + { type: 'text', text: 'working on it' }, + { type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'npm test && npm run lint' } } } }, + { type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: 'src/index.ts' } } } }, + ] + db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`) + .run('msg-2', 'sess-1', 'assistant', Buffer.from(JSON.stringify(assistantContent1)), 1_780_000_010) + + if (!opts.singleTurn) { + const assistantContent2 = [ + { type: 'toolRequest', toolCall: { value: { name: 'some_native_tool', arguments: {} } } }, + ] + db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`) + .run('msg-3', 'sess-1', 'assistant', Buffer.from(JSON.stringify(assistantContent2)), 1_780_000_020) + } + } finally { + db.close() + } +} + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'goose', + model: 'gpt-5.5', + inputTokens: 1500, + outputTokens: 400, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Bash', 'Read', 'some_native_tool'], + bashCommands: ['npm'], + toolSequence: [ + [ + { tool: 'Bash', command: 'npm test && npm run lint' }, + { tool: 'Read', file: 'src/index.ts' }, + ], + [ + { tool: 'some_native_tool' }, + ], + ], + timestamp: '2026-06-01T10:05:30.000Z', + speed: 'standard', + deduplicationKey: 'goose:sess-1', + userMessage: 'please refactor the shell wrapper and run tests', + sessionId: 'sess-1', + }, +] + +let tmpRoot: string + +beforeEach(async () => { + tmpRoot = await mkdtemp(join(tmpdir(), 'goose-bridge-')) +}) + +afterEach(async () => { + await rm(tmpRoot, { recursive: true, force: true }) +}) + +async function collect(dbPath: string, seen = new Set()): Promise { + const provider = createGooseProvider() + const source: SessionSource = { path: `${dbPath}:sess-1`, project: 'goose-bridge', provider: 'goose' } + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + return calls +} + +describe.skipIf(!isSqliteAvailable())('goose bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const dbPath = createGooseDb(tmpRoot) + seed(dbPath) + expect(await collect(dbPath)).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const dbPath = createGooseDb(tmpRoot) + seed(dbPath) + const raw = await collect(dbPath) + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const dbPath = createGooseDb(tmpRoot) + seed(dbPath) + const seen = new Set() + const first = await collect(dbPath, seen) + const second = await collect(dbPath, seen) + expect(first.length).toBe(1) + expect(second).toEqual([]) + }) + + it('omits toolSequence when only one turn carries tool calls', async () => { + const dbPath = createGooseDb(tmpRoot) + seed(dbPath, { singleTurn: true }) + const [call] = await collect(dbPath) + expect(call).toEqual({ + ...GOLDEN[0]!, + tools: ['Bash', 'Read'], + toolSequence: undefined, + }) + }) + + it('falls back to the current time when neither timestamp column parses', async () => { + const dbPath = createGooseDb(tmpRoot) + seed(dbPath, { badTimestamps: true }) + const before = Date.now() + const [call] = await collect(dbPath) + const after = Date.now() + // Every other field is identical to the golden; only the timestamp differs, + // and it must land on "now" exactly as the pre-migration `new Date()` did. + const { timestamp, ...rest } = call! + const { timestamp: _goldenTimestamp, ...goldenRest } = GOLDEN[0]! + expect(rest).toEqual(goldenRest) + expect(Date.parse(timestamp)).toBeGreaterThanOrEqual(before - 1000) + expect(Date.parse(timestamp)).toBeLessThanOrEqual(after + 1000) + }) +}) diff --git a/packages/cli/tests/providers/zcode-bridge.test.ts b/packages/cli/tests/providers/zcode-bridge.test.ts new file mode 100644 index 00000000..14223fd0 --- /dev/null +++ b/packages/cli/tests/providers/zcode-bridge.test.ts @@ -0,0 +1,172 @@ +import { mkdtemp, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' +import { createRequire } from 'node:module' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { createZcodeProvider } from '../../src/providers/zcode.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +// Byte-identical parity gate for the zcode bridge migration (phase 8, Category +// B / sqlite). ZCode is not present in the frozen corpus, so a committed +// fixture golden is THE parity gate. The GOLDEN below was captured from the +// legacy provider before the migration. + +const requireForTest = createRequire(import.meta.url) + +function createZcodeDb(dir: string): string { + const dbPath = join(dir, 'db.sqlite') + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + db.exec(`CREATE TABLE session (id TEXT PRIMARY KEY, directory TEXT NOT NULL)`) + db.exec(`CREATE TABLE model_usage ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL, turn_id TEXT, model_id TEXT NOT NULL, + input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0, + reasoning_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0, + cache_read_input_tokens INTEGER NOT NULL DEFAULT 0, started_at INTEGER NOT NULL, completed_at INTEGER)`) + db.exec(`CREATE TABLE tool_usage (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, turn_id TEXT, tool_name TEXT NOT NULL, started_at INTEGER NOT NULL)`) + db.close() + return dbPath +} + +function seed(dbPath: string): void { + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + try { + db.prepare('INSERT INTO session (id, directory) VALUES (?, ?)').run('sess-1', '/Users/me/proj') + db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run('mu-1', 'sess-1', 'turn-1', 'GLM-5.2', 9125, 27, 12, 0, 8064, 1781981181862, 1781981202412) + db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run('mu-2', 'sess-1', 'turn-1', 'GLM-5.2', 200, 40, 0, 0, 0, 1781981210000, 1781981220000) + // Zero-token row: must be skipped entirely. + db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run('mu-zero', 'sess-1', 'turn-2', 'GLM-5.2', 0, 0, 0, 0, 0, 1781981230000, 1781981231000) + // No turn_id -> no tools attached. + db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + .run('mu-3', 'sess-1', null, 'GLM-5.2', 500, 60, 0, 100, 0, 1781981240000, null) + + db.prepare('INSERT INTO tool_usage (id, session_id, turn_id, tool_name, started_at) VALUES (?, ?, ?, ?, ?)').run('tu-1', 'sess-1', 'turn-1', 'Bash', 1781981185000) + db.prepare('INSERT INTO tool_usage (id, session_id, turn_id, tool_name, started_at) VALUES (?, ?, ?, ?, ?)').run('tu-2', 'sess-1', 'turn-1', 'Read', 1781981190000) + } finally { + db.close() + } +} + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'zcode', + model: 'GLM-5.2', + inputTokens: 1061, + outputTokens: 27, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 8064, + cachedInputTokens: 0, + reasoningTokens: 12, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Bash', 'Read'], + bashCommands: [], + timestamp: '2026-06-20T18:46:42.412Z', + speed: 'standard', + deduplicationKey: 'zcode:mu-1', + turnId: 'turn-1', + userMessage: '', + sessionId: 'sess-1', + }, + { + provider: 'zcode', + model: 'GLM-5.2', + inputTokens: 200, + outputTokens: 40, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-20T18:47:00.000Z', + speed: 'standard', + deduplicationKey: 'zcode:mu-2', + turnId: 'turn-1', + userMessage: '', + sessionId: 'sess-1', + }, + { + provider: 'zcode', + model: 'GLM-5.2', + inputTokens: 400, + outputTokens: 60, + cacheCreationInputTokens: 100, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-20T18:47:20.000Z', + speed: 'standard', + deduplicationKey: 'zcode:mu-3', + userMessage: '', + sessionId: 'sess-1', + }, +] + +let tmpRoot: string + +beforeEach(async () => { + tmpRoot = await mkdtemp(join(tmpdir(), 'zcode-bridge-')) +}) + +afterEach(async () => { + await rm(tmpRoot, { recursive: true, force: true }) +}) + +async function collect(dbPath: string, seen = new Set()): Promise { + const provider = createZcodeProvider(dbPath) + const sources = await provider.discoverSessions() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe.skipIf(!isSqliteAvailable())('zcode bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const dbPath = createZcodeDb(tmpRoot) + seed(dbPath) + expect(await collect(dbPath)).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const dbPath = createZcodeDb(tmpRoot) + seed(dbPath) + const raw = await collect(dbPath) + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costUSD).toBeGreaterThan(0) + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const dbPath = createZcodeDb(tmpRoot) + seed(dbPath) + const seen = new Set() + const first = await collect(dbPath, seen) + const second = await collect(dbPath, seen) + expect(first.length).toBe(3) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/zed-bridge.test.ts b/packages/cli/tests/providers/zed-bridge.test.ts new file mode 100644 index 00000000..052645b8 --- /dev/null +++ b/packages/cli/tests/providers/zed-bridge.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtemp, rm } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { createRequire } from 'node:module' +import zlib from 'zlib' + +import { createZedProvider } from '../../src/providers/zed.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +// Byte-identical parity gate for the zed bridge migration (phase 8, Category B +// / sqlite). Zed is not present in the frozen corpus, so a committed fixture +// golden is THE parity gate. The GOLDEN below was captured from the legacy +// provider before the migration. + +const requireForTest = createRequire(import.meta.url) +const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync + +const skipReason = !isSqliteAvailable() + ? 'node:sqlite not available — needs Node 22+; skipping' + : !zstd + ? 'zlib zstd not available — needs Node 22.15+; skipping' + : null + +let tmpDir: string + +beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), 'zed-bridge-')) +}) + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }) +}) + +function buildDb(fn: (db: { + exec(sql: string): void + prepare(sql: string): { run(...params: unknown[]): void } + close(): void +}) => void): string { + const dbPath = join(tmpDir, 'threads.db') + const { DatabaseSync: Database } = requireForTest('node:sqlite') + const db = new Database(dbPath) + db.exec(`CREATE TABLE threads ( + id TEXT PRIMARY KEY, + summary TEXT NOT NULL, + updated_at TEXT NOT NULL, + data_type TEXT NOT NULL, + data BLOB NOT NULL, + parent_id TEXT, folder_paths TEXT, folder_paths_order TEXT, created_at TEXT + )`) + fn(db) + db.close() + return dbPath +} + +function insertThread(db: { + prepare(sql: string): { run(...params: unknown[]): void } +}, opts: { + id: string + summary?: string + updatedAt?: string + dataType?: string + thread?: unknown + rawData?: Buffer +}): void { + const data = opts.rawData ?? zstd!(Buffer.from(JSON.stringify(opts.thread ?? {}))) + db.prepare('INSERT INTO threads (id, summary, updated_at, data_type, data) VALUES (?, ?, ?, ?, ?)').run( + opts.id, + opts.summary ?? 'a thread', + opts.updatedAt ?? '2026-06-20T10:00:00Z', + opts.dataType ?? 'zstd', + data, + ) +} + +function seedDb(): string { + return buildDb((db) => { + insertThread(db, { + id: 'thread-1', + summary: 'refactor the parser', + updatedAt: '2026-06-21T09:30:00Z', + thread: { + model: { provider: 'anthropic', model: 'claude-opus-4-8' }, + request_token_usage: { + 'req-1': { input_tokens: 1200, output_tokens: 300, cache_creation_input_tokens: 5000, cache_read_input_tokens: 90000 }, + 'req-2': { input_tokens: 800, output_tokens: 150, cache_creation_input_tokens: 0, cache_read_input_tokens: 95000 }, + }, + cumulative_token_usage: { input_tokens: 2000, output_tokens: 450 }, + }, + }) + insertThread(db, { id: 'bad-type', dataType: 'protobuf', rawData: Buffer.from('{}') }) + insertThread(db, { id: 'bad-blob', rawData: Buffer.from('not zstd at all') }) + insertThread(db, { + id: 'legacy', + dataType: 'json', + updatedAt: '2026-06-22T11:00:00Z', + rawData: Buffer.from(JSON.stringify({ + model: { model: 'claude-sonnet-4-6' }, + request_token_usage: { 'req-1': { input_tokens: 40, output_tokens: 8 } }, + })), + }) + insertThread(db, { + id: 'zero-usage', + thread: { + model: { model: 'claude-opus-4-8' }, + request_token_usage: { 'req-1': { input_tokens: 0, output_tokens: 0 } }, + cumulative_token_usage: { input_tokens: 0, output_tokens: 0 }, + }, + }) + }) +} + +// Captured from the pre-migration zed decode: two per-request calls for +// thread-1 (cumulative exactly covered by the map, no remainder), one call for +// the legacy uncompressed row, and the bad-type/bad-blob/zero-usage rows +// skipped without dropping the healthy threads. +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'zed', + model: 'claude-opus-4-8', + inputTokens: 1200, + outputTokens: 300, + cacheCreationInputTokens: 5000, + cacheReadInputTokens: 90000, + cachedInputTokens: 90000, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-21T09:30:00.000Z', + speed: 'standard', + deduplicationKey: 'zed:thread-1:req-1', + userMessage: 'refactor the parser', + sessionId: 'thread-1', + }, + { + provider: 'zed', + model: 'claude-opus-4-8', + inputTokens: 800, + outputTokens: 150, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 95000, + cachedInputTokens: 95000, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-21T09:30:00.000Z', + speed: 'standard', + deduplicationKey: 'zed:thread-1:req-2', + userMessage: 'refactor the parser', + sessionId: 'thread-1', + }, + { + provider: 'zed', + model: 'claude-sonnet-4-6', + inputTokens: 40, + outputTokens: 8, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-22T11:00:00.000Z', + speed: 'standard', + deduplicationKey: 'zed:legacy:req-1', + userMessage: 'a thread', + sessionId: 'legacy', + }, +] + +async function collectCalls(dbPath: string, seenKeys = new Set()): Promise { + const provider = createZedProvider(dbPath) + const sources = await provider.discoverSessions() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seenKeys).parse()) { + calls.push(call) + } + } + return calls +} + +describe.skipIf(skipReason !== null)('zed bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const dbPath = seedDb() + expect(await collectCalls(dbPath)).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const dbPath = seedDb() + const raw = await collectCalls(dbPath) + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => { + const dbPath = seedDb() + const seen = new Set() + const first = await collectCalls(dbPath, seen) + const second = await collectCalls(dbPath, seen) + expect(first.length).toBe(3) + expect(second).toEqual([]) + }) + + it('still prints the pre-migration aggregate stderr line for unreadable threads', async () => { + const dbPath = seedDb() + const written: string[] = [] + const original = process.stderr.write.bind(process.stderr) + ;(process.stderr as { write: unknown }).write = (chunk: unknown) => { written.push(String(chunk)); return true } + try { + await collectCalls(dbPath) + } finally { + ;(process.stderr as { write: unknown }).write = original + } + // bad-type (unknown data_type) + bad-blob (zstd decompress throws) = 2. + expect(written).toEqual(['codeburn: skipped 2 unreadable Zed threads\n']) + }) +}) diff --git a/packages/core/package.json b/packages/core/package.json index 2346dbf0..43f7d111 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -98,6 +98,26 @@ "./providers/pi": { "types": "./dist/providers/pi/index.d.ts", "import": "./dist/providers/pi/index.js" + }, + "./providers/crush": { + "types": "./dist/providers/crush/index.d.ts", + "import": "./dist/providers/crush/index.js" + }, + "./providers/zcode": { + "types": "./dist/providers/zcode/index.d.ts", + "import": "./dist/providers/zcode/index.js" + }, + "./providers/zed": { + "types": "./dist/providers/zed/index.d.ts", + "import": "./dist/providers/zed/index.js" + }, + "./providers/forge": { + "types": "./dist/providers/forge/index.d.ts", + "import": "./dist/providers/forge/index.js" + }, + "./providers/goose": { + "types": "./dist/providers/goose/index.d.ts", + "import": "./dist/providers/goose/index.js" } }, "files": [ diff --git a/packages/core/src/providers/crush/decode.ts b/packages/core/src/providers/crush/decode.ts new file mode 100644 index 00000000..e467f418 --- /dev/null +++ b/packages/core/src/providers/crush/decode.ts @@ -0,0 +1,73 @@ +// @codeburn/core Crush decoder: pure decode over host-supplied session records. +// The host runs the sqlite queries (session row + dominant model) and hands +// this decoder one combined record per session; no fs/env/sqlite/pricing here. +// Crush already stores cost in dollars, so a row with cost > 0 carries +// `measuredCostUSD` instead of estimated token pricing (Phase 0 pattern B). + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CrushDecodedCall, CrushRawRecord } from './types.js' + +function epochSecondsToIso(epochSeconds: number | null): string { + if (epochSeconds === null || !Number.isFinite(epochSeconds)) { + return new Date(0).toISOString() + } + return new Date(epochSeconds * 1000).toISOString() +} + +export type CrushDecodeInput = { + records: unknown[] + context: DecodeContext + // Optional live dedup set the host mutates in place. Threaded exactly like + // codex/qwen; Crush never persists resume state. + seenKeys?: Set +} + +export type CrushDecodeResult = { + calls: CrushDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode Crush session records (one per source) into rich, cost-free-or-measured + * calls. A session with zero tokens and zero cost is skipped; dedup is keyed on + * `crush:` against the live `seenKeys` set (host-owned). + */ +export function decodeCrush({ records, seenKeys: liveSeen }: CrushDecodeInput): CrushDecodeResult { + const seen = liveSeen ?? new Set() + const calls: CrushDecodedCall[] = [] + + for (const raw of records) { + const session = raw as CrushRawRecord + + const inputTokens = session.prompt_tokens ?? 0 + const outputTokens = session.completion_tokens ?? 0 + const cost = session.cost ?? 0 + if (inputTokens === 0 && outputTokens === 0 && cost === 0) continue + + const dedupKey = `crush:${session.id}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + calls.push({ + provider: 'crush', + model: session.model, + inputTokens, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + ...(cost > 0 ? { measuredCostUSD: cost } : {}), + tools: [], + rawBashCommands: [], + timestamp: epochSecondsToIso(session.updated_at ?? session.created_at), + speed: 'standard', + deduplicationKey: dedupKey, + sessionId: session.id, + }) + } + + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/crush/index.ts b/packages/core/src/providers/crush/index.ts new file mode 100644 index 00000000..77a0cbef --- /dev/null +++ b/packages/core/src/providers/crush/index.ts @@ -0,0 +1,26 @@ +// @codeburn/core Crush provider. +// +// Two layers: +// - Rich pure decode (`decodeCrush`): host-facing, NOT part of the stable +// minimized surface. Pure over host-supplied session records (the sqlite +// driver and SQL queries stay CLI-side, per the Category B recipe). +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeCrush, + type CrushDecodeInput, + type CrushDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichCrushSessionDecode, + type CrushToObservationsContext, +} from './observations.js' + +export type { + CrushDecodedCall, + CrushRawRecord, + CrushSessionRow, +} from './types.js' diff --git a/packages/core/src/providers/crush/observations.ts b/packages/core/src/providers/crush/observations.ts new file mode 100644 index 00000000..e89342fd --- /dev/null +++ b/packages/core/src/providers/crush/observations.ts @@ -0,0 +1,83 @@ +// Minimizing transform: rich Crush decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. Crush never captures a user message or file +// paths, so there is nothing else to fingerprint or drop here. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { CrushDecodedCall } from './types.js' + +/** One Crush session's rich decode, as the host holds it before minimization. */ +export interface RichCrushSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order (Crush emits one per session). */ + calls: CrushDecodedCall[] +} + +export interface CrushToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: CrushDecodedCall, turnIndex: number): CallObservation { + const measured = call.measuredCostUSD !== undefined + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: measured ? 'measured' : 'estimated', + ...(measured ? { measuredCostUSD: call.measuredCostUSD } : {}), + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichCrushSessionDecode, ctx: CrushToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'crush' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Crush decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichCrushSessionDecode | RichCrushSessionDecode[], + ctx: CrushToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/crush/types.ts b/packages/core/src/providers/crush/types.ts new file mode 100644 index 00000000..3f036835 --- /dev/null +++ b/packages/core/src/providers/crush/types.ts @@ -0,0 +1,45 @@ +// Raw record + rich-decode types for the Crush provider. +// +// Crush stores per-project SQLite databases; the sqlite driver and SQL queries +// stay CLI-side (Category B). The host runs both the session-row query and the +// dominant-model query (a GROUP BY over messages) and hands the decoder one +// combined record per session — this decoder performs no further DB access. + +/** The `sessions` row Crush records, moved verbatim from the CLI. */ +export type CrushSessionRow = { + id: string + prompt_tokens: number | null + completion_tokens: number | null + cost: number | null + created_at: number | null + updated_at: number | null + message_count: number | null +} + +/** + * The record the host hands to the core decoder: the session row plus the + * dominant model resolved by the host's second query (unchanged from the + * pre-migration decode, which ran `dominantModel()` inline). + */ +export type CrushRawRecord = CrushSessionRow & { model: string } + +/** The rich decode of one Crush session, pre-pricing. */ +export type CrushDecodedCall = { + provider: 'crush' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + /** Crush already records cost in dollars; present only when the row's cost > 0. */ + measuredCostUSD?: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + sessionId: string +} diff --git a/packages/core/src/providers/forge/decode.ts b/packages/core/src/providers/forge/decode.ts new file mode 100644 index 00000000..a27d29c6 --- /dev/null +++ b/packages/core/src/providers/forge/decode.ts @@ -0,0 +1,188 @@ +// @codeburn/core Forge decoder: pure decode over a host-supplied conversation +// row (the sqlite driver + query stay CLI-side; the `context` JSON blob is +// still a serialized string when it reaches this decoder). No fs/env/sqlite/ +// pricing/strip-ansi here — bash base-name extraction stays host-side, this +// decoder emits raw command strings only. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { ForgeConversationRow, ForgeContextMessage, ForgeDecodedCall } from './types.js' + +function sqliteTimestampToIso(value: string | null | undefined): string { + if (!value) return new Date(0).toISOString() + + const match = value.match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?$/) + if (match) { + const ms = (match[3] ?? '').padEnd(3, '0').slice(0, 3) + const parsed = new Date(`${match[1]}T${match[2]}.${ms}Z`) + if (!Number.isNaN(parsed.getTime())) return parsed.toISOString() + } + + const parsed = new Date(value) + return Number.isNaN(parsed.getTime()) ? new Date(0).toISOString() : parsed.toISOString() +} + +function actual(value: unknown): number { + if (!value || typeof value !== 'object') return 0 + const raw = (value as Record)['actual'] + return typeof raw === 'number' && Number.isFinite(raw) ? raw : 0 +} + +function usageActual(usage: unknown, key: string): number { + if (!usage || typeof usage !== 'object') return 0 + return actual((usage as Record)[key]) +} + +// Kept as a `switch` (not a lookup object) and not exported: tool names come +// straight from the conversation JSON, so an object-literal map would resolve +// inherited `Object.prototype` members ("constructor", "toString", "__proto__") +// to non-string values instead of falling through to the identity default. The +// pre-migration CLI's `toolDisplayName` was an identity function anyway (raw +// tool ids never surfaced; `tools` already carries the mapped canonical name +// from this decode), so this has no consumer outside the extraction below. +function mapToolName(name: string): string { + switch (name) { + case 'shell': + case 'bash': + return 'Bash' + case 'read': + case 'Read': + return 'Read' + case 'write': + case 'Write': + return 'Write' + case 'patch': + case 'Edit': + case 'edit': + return 'Edit' + case 'fs_search': + case 'grep': + return 'Grep' + case 'task': + case 'dispatch_agent': + return 'Agent' + default: + return name + } +} + +function pushUnique(values: string[], value: string): void { + if (!values.includes(value)) values.push(value) +} + +function toolCallsOf(value: unknown): Record[] { + return Array.isArray(value) ? value.filter(v => v && typeof v === 'object') as Record[] : [] +} + +function extractToolsAndCommands(calls: Record[]): { tools: string[]; rawBashCommands: string[]; firstCallId?: string } { + const tools: string[] = [] + const rawBashCommands: string[] = [] + let firstCallId: string | undefined + + for (const call of calls) { + const rawName = call['name'] + if (typeof rawName !== 'string') continue + if (!firstCallId && typeof call['call_id'] === 'string') firstCallId = call['call_id'] + + const tool = mapToolName(rawName) + pushUnique(tools, tool) + + if (tool === 'Bash') { + const args = call['arguments'] + if (args && typeof args === 'object') { + const command = (args as Record)['command'] + if (typeof command === 'string') { + rawBashCommands.push(command) + } + } + } + } + + return { tools, rawBashCommands, firstCallId } +} + +export type ForgeDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type ForgeDecodeResult = { + calls: ForgeDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one Forge `conversations` row into rich, cost-free calls, one per + * assistant message that carries token usage. The nearest previous user + * message in the same conversation is attributed as `userMessage`. Dedup keys + * on the tool call's `call_id` when present, else a positional fallback, + * against the live `seenKeys` set (host-owned). + */ +export function decodeForge({ records, seenKeys: liveSeen }: ForgeDecodeInput): ForgeDecodeResult { + const seen = liveSeen ?? new Set() + const calls: ForgeDecodedCall[] = [] + + for (const raw of records) { + const row = raw as ForgeConversationRow + if (!row.context) continue + + let parsed: unknown + try { + parsed = JSON.parse(row.context) + } catch { + continue + } + const messages = Array.isArray((parsed as { messages?: unknown }).messages) + ? (parsed as { messages: ForgeContextMessage[] }).messages + : [] + + let userMessage = '' + for (let i = 0; i < messages.length; i++) { + const text = messages[i]?.message?.text + const role = typeof text?.role === 'string' ? text.role.toLowerCase() : '' + const content = typeof text?.content === 'string' ? text.content : '' + + if (role === 'user') { + userMessage = content.length > 500 ? content.slice(0, 500) : content + continue + } + if (role !== 'assistant') continue + + const promptTokens = usageActual(messages[i]?.usage, 'prompt_tokens') + const outputTokens = usageActual(messages[i]?.usage, 'completion_tokens') + const cachedInputTokens = usageActual(messages[i]?.usage, 'cached_tokens') + const inputTokens = Math.max(0, promptTokens - cachedInputTokens) + if (inputTokens === 0 && outputTokens === 0) continue + + const model = typeof text?.model === 'string' ? text.model : 'unknown' + const toolCalls = toolCallsOf(text?.tool_calls) + const { tools, rawBashCommands, firstCallId } = extractToolsAndCommands(toolCalls) + const stableId = firstCallId ?? `${model}:${promptTokens}:${outputTokens}:${i}` + const deduplicationKey = `forge:${row.conversation_id}:${stableId}` + if (seen.has(deduplicationKey)) continue + seen.add(deduplicationKey) + + calls.push({ + provider: 'forge', + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: cachedInputTokens, + cachedInputTokens, + reasoningTokens: 0, + webSearchRequests: 0, + tools, + rawBashCommands, + timestamp: sqliteTimestampToIso(row.updated_at ?? row.created_at), + speed: 'standard', + deduplicationKey, + userMessage, + sessionId: row.conversation_id, + }) + } + } + + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/forge/index.ts b/packages/core/src/providers/forge/index.ts new file mode 100644 index 00000000..93ccf8fe --- /dev/null +++ b/packages/core/src/providers/forge/index.ts @@ -0,0 +1,27 @@ +// @codeburn/core Forge provider. +// +// Two layers: +// - Rich pure decode (`decodeForge`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied `conversations` row (the +// sqlite driver and SQL query stay CLI-side, Category B); parses the +// `context` JSON blob itself. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeForge, + type ForgeDecodeInput, + type ForgeDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichForgeSessionDecode, + type ForgeToObservationsContext, +} from './observations.js' + +export type { + ForgeConversationRow, + ForgeContextMessage, + ForgeDecodedCall, +} from './types.js' diff --git a/packages/core/src/providers/forge/observations.ts b/packages/core/src/providers/forge/observations.ts new file mode 100644 index 00000000..bb4c9793 --- /dev/null +++ b/packages/core/src/providers/forge/observations.ts @@ -0,0 +1,83 @@ +// Minimizing transform: rich Forge decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the user message or a bash command line. +// Forge never captures file targets for its tool calls, so there is nothing to +// fingerprint into resourceReads/resourceEdits. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { ForgeDecodedCall } from './types.js' + +/** One Forge conversation's rich decode, as the host holds it before minimization. */ +export interface RichForgeSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order (one per assistant message with usage). */ + calls: ForgeDecodedCall[] +} + +export interface ForgeToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: ForgeDecodedCall, turnIndex: number): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichForgeSessionDecode, ctx: ForgeToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'forge' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Forge decode (one or many conversations) into the minimized + * observation layer. Returns the `sessions` array plus any per-record + * `diagnostics`. + */ +export function toObservations( + decode: RichForgeSessionDecode | RichForgeSessionDecode[], + ctx: ForgeToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/forge/types.ts b/packages/core/src/providers/forge/types.ts new file mode 100644 index 00000000..8a055f60 --- /dev/null +++ b/packages/core/src/providers/forge/types.ts @@ -0,0 +1,50 @@ +// Raw record + rich-decode types for the Forge provider. +// +// Forge stores one row per conversation in a single SQLite database; the +// `context` column is a JSON blob carrying the full message list (including +// per-message token usage and tool calls). The sqlite driver and SQL query stay +// CLI-side (Category B); the host hands this decoder the raw conversation row +// (with the still-serialized `context` string) — JSON parsing and per-message +// decode are pure and happen here. + +/** The `conversations` row Forge records, moved verbatim from the CLI. */ +export type ForgeConversationRow = { + conversation_id: string + title: string | null + workspace_id: number | string + context: string | null + created_at: string | null + updated_at: string | null +} + +export type ForgeContextMessage = { + message?: { + text?: { + role?: unknown + content?: unknown + model?: unknown + tool_calls?: unknown + } + } + usage?: unknown +} + +/** The rich decode of one Forge assistant message, pre-pricing. */ +export type ForgeDecodedCall = { + provider: 'forge' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/src/providers/goose/decode.ts b/packages/core/src/providers/goose/decode.ts new file mode 100644 index 00000000..e7b9addc --- /dev/null +++ b/packages/core/src/providers/goose/decode.ts @@ -0,0 +1,171 @@ +// @codeburn/core Goose decoder: pure decode over a host-supplied composite +// session record (session row + assistant tool-message rows + first user +// message row, all BLOB columns already converted to text host-side). No +// fs/env/sqlite/pricing/clock here — the pre-migration decode fell back to +// `new Date()` when both timestamp columns were unparseable; that clock read +// stays host-side (the CLI's `toProviderCall` applies it), so this decoder +// emits an empty timestamp in that case instead. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { + GooseContentItem, + GooseDecodedCall, + GooseMessageRow, + GooseModelConfig, + GooseSessionRecords, + GooseToolCall, +} from './types.js' + +export const gooseToolNameMap: Record = { + developer__shell: 'Bash', + developer__text_editor: 'Edit', + developer__read_file: 'Read', + developer__write_file: 'Write', + developer__list_directory: 'LS', + developer__search_files: 'Grep', + computercontroller__shell: 'Bash', +} + +function parseModelConfig(raw: string | null): GooseModelConfig { + if (!raw) return {} + try { + return JSON.parse(raw) as GooseModelConfig + } catch { + return {} + } +} + +function extractToolsFromMessages(messages: GooseMessageRow[]): { tools: string[]; rawBashCommands: string[]; toolSequence: GooseToolCall[][] } { + const tools: string[] = [] + const rawBashCommands: string[] = [] + const seen = new Set() + const toolSequence: GooseToolCall[][] = [] + + for (const row of messages) { + let items: GooseContentItem[] + try { + items = JSON.parse(row.contentJson) as GooseContentItem[] + } catch { + continue + } + const msgCalls: GooseToolCall[] = [] + for (const item of items) { + if (item.type !== 'toolRequest') continue + const rawName = item.toolCall?.value?.name ?? '' + if (!rawName) continue + const mapped = gooseToolNameMap[rawName] ?? rawName.split('__').pop() ?? rawName + if (!seen.has(mapped)) { + seen.add(mapped) + tools.push(mapped) + } + const call: GooseToolCall = { tool: mapped } + const args = item.toolCall?.value?.arguments + if (args && typeof args === 'object') { + const fp = (args as Record)['file_path'] + if (typeof fp === 'string') call.file = fp + const cmd = (args as Record)['command'] + if (typeof cmd === 'string') call.command = cmd + } + msgCalls.push(call) + if (mapped === 'Bash') { + const cmd = args && typeof args === 'object' ? (args as Record)['command'] : undefined + if (typeof cmd === 'string') rawBashCommands.push(cmd) + } + } + if (msgCalls.length > 0) toolSequence.push(msgCalls) + } + + return { tools, rawBashCommands, toolSequence } +} + +function getFirstUserMessage(row: GooseMessageRow | null): string { + if (!row) return '' + try { + const items = JSON.parse(row.contentJson) as GooseContentItem[] + const text = items.find(i => i.type === 'text') as { text?: string } | undefined + return (text?.text ?? '').slice(0, 500) + } catch { + return '' + } +} + +// Resolve updated_at/created_at into an ISO timestamp, exactly as the +// pre-migration decode did, EXCEPT the final `new Date()` (current time) +// fallback: that clock read is not deterministic and must stay host-side, so +// this returns '' when neither column parses. +function resolveTimestamp(updatedAt: string | null, createdAt: string | null): string { + const raw = updatedAt || createdAt || '' + let ts = new Date(raw) + if (isNaN(ts.getTime())) ts = new Date(raw + 'Z') + if (isNaN(ts.getTime())) return '' + return ts.toISOString() +} + +function isGooseSessionRecords(value: unknown): value is GooseSessionRecords { + return value !== null && typeof value === 'object' && 'session' in (value as object) && 'assistantToolMessages' in (value as object) +} + +export type GooseDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type GooseDecodeResult = { + calls: GooseDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one Goose session's composite record into a single rich, cost-free + * call. A session with zero accumulated tokens is skipped; dedup is keyed on + * `goose:` against the live `seenKeys` set (host-owned). + * `toolSequence` is included only when it has MORE THAN ONE turn (matches the + * pre-migration `toolSequence.length > 1 ? toolSequence : undefined` exactly). + */ +export function decodeGoose({ records, seenKeys: liveSeen }: GooseDecodeInput): GooseDecodeResult { + const seen = liveSeen ?? new Set() + const calls: GooseDecodedCall[] = [] + + const session = records.find(isGooseSessionRecords) + if (!session) return { calls, diagnostics: [] } + + const { sessionId, session: sessionRow, assistantToolMessages, firstUserMessage } = session + + const inputTokens = sessionRow.accumulatedInputTokens ?? 0 + const outputTokens = sessionRow.accumulatedOutputTokens ?? 0 + if (inputTokens === 0 && outputTokens === 0) return { calls, diagnostics: [] } + + const dedupKey = `goose:${sessionId}` + if (seen.has(dedupKey)) return { calls, diagnostics: [] } + seen.add(dedupKey) + + const config = parseModelConfig(sessionRow.modelConfigJson) + const model = config.model_name ?? 'unknown' + + const { tools, rawBashCommands, toolSequence } = extractToolsFromMessages(assistantToolMessages) + const userMessage = getFirstUserMessage(firstUserMessage) + + calls.push({ + provider: 'goose', + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools, + rawBashCommands, + toolSequence: toolSequence.length > 1 ? toolSequence : undefined, + timestamp: resolveTimestamp(sessionRow.updatedAt, sessionRow.createdAt), + speed: 'standard', + deduplicationKey: dedupKey, + userMessage, + sessionId, + }) + + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/goose/index.ts b/packages/core/src/providers/goose/index.ts new file mode 100644 index 00000000..b84fd050 --- /dev/null +++ b/packages/core/src/providers/goose/index.ts @@ -0,0 +1,31 @@ +// @codeburn/core Goose provider. +// +// Two layers: +// - Rich pure decode (`decodeGoose`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied session/message row bundle +// (the sqlite driver and SQL queries stay CLI-side, Category B). +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeGoose, + gooseToolNameMap, + type GooseDecodeInput, + type GooseDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichGooseSessionDecode, + type GooseToObservationsContext, +} from './observations.js' + +export type { + GooseContentItem, + GooseDecodedCall, + GooseMessageRow, + GooseModelConfig, + GooseSessionRecords, + GooseSessionRow, + GooseToolCall, +} from './types.js' diff --git a/packages/core/src/providers/goose/observations.ts b/packages/core/src/providers/goose/observations.ts new file mode 100644 index 00000000..ba90005b --- /dev/null +++ b/packages/core/src/providers/goose/observations.ts @@ -0,0 +1,83 @@ +// Minimizing transform: rich Goose decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the user message, a bash command line, or +// a raw file path. Fingerprint file paths through `extractResourceRefs`. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import { extractResourceRefs } from '../resource-refs.js' +import type { GooseDecodedCall } from './types.js' + +/** One Goose session's rich decode, as the host holds it before minimization. */ +export interface RichGooseSessionDecode { + sessionId: string + /** Absolute project path (the session's working_dir); fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order (Goose emits one per session). */ + calls: GooseDecodedCall[] +} + +export interface GooseToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: GooseDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + ...extractResourceRefs(privacyKey, call.toolSequence), + } +} + +function toSessionObservation(decode: RichGooseSessionDecode, ctx: GooseToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'goose' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Goose decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichGooseSessionDecode | RichGooseSessionDecode[], + ctx: GooseToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/goose/types.ts b/packages/core/src/providers/goose/types.ts new file mode 100644 index 00000000..c9a52ec8 --- /dev/null +++ b/packages/core/src/providers/goose/types.ts @@ -0,0 +1,78 @@ +// Raw record + rich-decode types for the Goose provider. +// +// Goose stores one row per session plus a `messages` table with a per-message +// `content_json` BLOB column. The sqlite driver and SQL queries stay CLI-side +// (Category B); the host runs the session query, the assistant tool-message +// query, and the first-user-message query, converts each BLOB column to text +// (`blobToText`, the same charset-safe conversion `fs-utils` performs for file +// reads), and hands this decoder one composite record bundling all three — +// JSON parsing and per-message decode are pure and happen here. + +/** The `sessions` row Goose records, pre-resolved to text columns by the host. */ +export type GooseSessionRow = { + id: string + workingDir: string | null + createdAt: string | null + updatedAt: string | null + accumulatedInputTokens: number | null + accumulatedOutputTokens: number | null + /** `model_config_json`, already converted from BLOB to text host-side. */ + modelConfigJson: string | null +} + +export type GooseModelConfig = { + model_name?: string + reasoning?: boolean +} + +/** One `messages` row, `content_json` already converted from BLOB to text host-side. */ +export type GooseMessageRow = { + contentJson: string +} + +export type GooseContentItem = { + type: string + text?: string + toolCall?: { value?: { name?: string; arguments?: Record } } +} + +/** One tool invocation as captured from a Goose transcript. */ +export type GooseToolCall = { + tool: string + file?: string + command?: string +} + +/** The composite record the host hands to the core decoder for one session. */ +export type GooseSessionRecords = { + sessionId: string + session: GooseSessionRow + /** Assistant messages whose content_json contains a toolRequest, ordered by created_timestamp ASC. */ + assistantToolMessages: GooseMessageRow[] + /** The first user message (by created_timestamp ASC), if any. */ + firstUserMessage: GooseMessageRow | null +} + +/** The rich decode of one Goose session, pre-pricing. */ +export type GooseDecodedCall = { + provider: 'goose' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + toolSequence?: GooseToolCall[][] + /** Resolved from updated_at/created_at; empty when neither parses (the host + * falls back to the current time, matching the pre-migration decode — that + * clock read must stay host-side, so this decode never touches it). */ + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/src/providers/zcode/decode.ts b/packages/core/src/providers/zcode/decode.ts new file mode 100644 index 00000000..171fe995 --- /dev/null +++ b/packages/core/src/providers/zcode/decode.ts @@ -0,0 +1,103 @@ +// @codeburn/core ZCode decoder: pure decode over a host-supplied composite +// session record (model_usage rows + tool_usage rows). No fs/env/sqlite/pricing +// here — the host runs both queries and hands the rows straight through. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { ZcodeDecodedCall, ZcodeSessionRecords, ZcodeToolRow, ZcodeUsageRow } from './types.js' + +function epochMsToIso(ms: number | null): string { + if (ms === null || !Number.isFinite(ms) || ms <= 0) return new Date(0).toISOString() + return new Date(ms).toISOString() +} + +function isZcodeSessionRecords(value: unknown): value is ZcodeSessionRecords { + return value !== null && typeof value === 'object' && 'usageRows' in (value as object) && 'toolRows' in (value as object) +} + +export type ZcodeDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type ZcodeDecodeResult = { + calls: ZcodeDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one ZCode session's composite record (model_usage rows joined with + * tool_usage rows by turn) into rich, cost-free calls. model_usage rows don't + * link to individual tool calls, only to a turn, so each turn's tools attach to + * the FIRST usage row of that turn only, to avoid double-counting across a + * turn's multiple requests — matching the pre-migration decode exactly. + */ +export function decodeZcode({ records, seenKeys: liveSeen }: ZcodeDecodeInput): ZcodeDecodeResult { + const seen = liveSeen ?? new Set() + const calls: ZcodeDecodedCall[] = [] + + const session = records.find(isZcodeSessionRecords) + if (!session) return { calls, diagnostics: [] } + + const { sessionId, usageRows, toolRows }: { sessionId: string; usageRows: ZcodeUsageRow[]; toolRows: ZcodeToolRow[] } = session + + const toolsByTurn = new Map() + for (const tool of toolRows) { + if (!tool.turn_id) continue + const list = toolsByTurn.get(tool.turn_id) ?? [] + list.push(tool.tool_name) + toolsByTurn.set(tool.turn_id, list) + } + + const turnsWithToolsEmitted = new Set() + + for (const row of usageRows) { + const cacheRead = row.cache_read_input_tokens ?? 0 + const cacheCreation = row.cache_creation_input_tokens ?? 0 + const output = row.output_tokens ?? 0 + const reasoning = row.reasoning_tokens ?? 0 + // ZCode folds cached tokens into input_tokens (OpenAI-style). Split them + // back out so fresh input bills at the input rate and cached at the + // cache-read rate, matching the pricing table's Anthropic-style semantics. + const freshInput = Math.max(0, (row.input_tokens ?? 0) - cacheRead - cacheCreation) + + if (freshInput === 0 && output === 0 && reasoning === 0 && cacheRead === 0 && cacheCreation === 0) { + continue + } + + const dedupKey = `zcode:${row.id}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + let tools: string[] = [] + if (row.turn_id && !turnsWithToolsEmitted.has(row.turn_id)) { + const turnTools = toolsByTurn.get(row.turn_id) + if (turnTools && turnTools.length > 0) { + tools = turnTools + turnsWithToolsEmitted.add(row.turn_id) + } + } + + calls.push({ + provider: 'zcode', + model: row.model_id, + inputTokens: freshInput, + outputTokens: output, + cacheCreationInputTokens: cacheCreation, + cacheReadInputTokens: cacheRead, + cachedInputTokens: 0, + reasoningTokens: reasoning, + webSearchRequests: 0, + tools, + rawBashCommands: [], + timestamp: epochMsToIso(row.completed_at ?? row.started_at), + speed: 'standard', + deduplicationKey: dedupKey, + turnId: row.turn_id ?? undefined, + sessionId, + }) + } + + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/zcode/index.ts b/packages/core/src/providers/zcode/index.ts new file mode 100644 index 00000000..e46cb03d --- /dev/null +++ b/packages/core/src/providers/zcode/index.ts @@ -0,0 +1,27 @@ +// @codeburn/core ZCode provider. +// +// Two layers: +// - Rich pure decode (`decodeZcode`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied model_usage/tool_usage row +// bundle (the sqlite driver and SQL queries stay CLI-side, Category B). +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeZcode, + type ZcodeDecodeInput, + type ZcodeDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichZcodeSessionDecode, + type ZcodeToObservationsContext, +} from './observations.js' + +export type { + ZcodeDecodedCall, + ZcodeSessionRecords, + ZcodeToolRow, + ZcodeUsageRow, +} from './types.js' diff --git a/packages/core/src/providers/zcode/observations.ts b/packages/core/src/providers/zcode/observations.ts new file mode 100644 index 00000000..2f26dd18 --- /dev/null +++ b/packages/core/src/providers/zcode/observations.ts @@ -0,0 +1,81 @@ +// Minimizing transform: rich ZCode decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. ZCode never captures a user message or file +// paths, so there is nothing else to fingerprint or drop here. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { ZcodeDecodedCall } from './types.js' + +/** One ZCode session's rich decode, as the host holds it before minimization. */ +export interface RichZcodeSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order (one per model_usage row). */ + calls: ZcodeDecodedCall[] +} + +export interface ZcodeToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: ZcodeDecodedCall, turnIndex: number): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichZcodeSessionDecode, ctx: ZcodeToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'zcode' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich ZCode decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichZcodeSessionDecode | RichZcodeSessionDecode[], + ctx: ZcodeToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/zcode/types.ts b/packages/core/src/providers/zcode/types.ts new file mode 100644 index 00000000..4b1d4f5c --- /dev/null +++ b/packages/core/src/providers/zcode/types.ts @@ -0,0 +1,53 @@ +// Raw record + rich-decode types for the ZCode provider. +// +// ZCode (CLI v0.14.x) records usage in a single SQLite database. The sqlite +// driver and SQL queries stay CLI-side (Category B); the host runs both the +// model_usage query and the tool_usage query for one session and hands the +// decoder a single composite record bundling both row sets. + +/** One `model_usage` row, moved verbatim from the CLI. */ +export type ZcodeUsageRow = { + id: string + turn_id: string | null + model_id: string + input_tokens: number + output_tokens: number + reasoning_tokens: number + cache_creation_input_tokens: number + cache_read_input_tokens: number + started_at: number + completed_at: number | null +} + +/** One `tool_usage` row, moved verbatim from the CLI. */ +export type ZcodeToolRow = { + turn_id: string | null + tool_name: string +} + +/** The composite record the host hands to the core decoder for one session. */ +export type ZcodeSessionRecords = { + sessionId: string + usageRows: ZcodeUsageRow[] + toolRows: ZcodeToolRow[] +} + +/** The rich decode of one ZCode model_usage row, pre-pricing. */ +export type ZcodeDecodedCall = { + provider: 'zcode' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + turnId?: string + sessionId: string +} diff --git a/packages/core/src/providers/zed/decode.ts b/packages/core/src/providers/zed/decode.ts new file mode 100644 index 00000000..d8355f30 --- /dev/null +++ b/packages/core/src/providers/zed/decode.ts @@ -0,0 +1,143 @@ +// @codeburn/core Zed decoder: pure decode over host-supplied `threads` rows. +// The host runs the sqlite query and the Node-version zstd-availability gate; +// this decoder receives the raw rows (blob and all) and does the decompression, +// JSON parsing, and per-request token accounting. No fs/env/sqlite/pricing. +// +// zstd landed in node:zlib in 22.15 / 23.8. The host only calls this decoder +// after confirming `zstdDecompressSync` exists (see the CLI's `readRecords`), so +// calling it here is safe; the decoder does not re-check Node version support. + +import zlib from 'node:zlib' + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { ZedDecodedCall, ZedThreadJson, ZedThreadRow, ZedTokenUsage } from './types.js' + +const zstdDecompressSync = (zlib as { zstdDecompressSync?: (buf: Buffer) => Buffer }).zstdDecompressSync + +function num(value: number | undefined): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 +} + +function usageIsEmpty(usage: ZedTokenUsage): boolean { + return ( + num(usage.input_tokens) === 0 && + num(usage.output_tokens) === 0 && + num(usage.cache_creation_input_tokens) === 0 && + num(usage.cache_read_input_tokens) === 0 + ) +} + +function buildCall(opts: { + threadId: string + requestKey: string + usage: ZedTokenUsage + model: string + timestamp: string + userMessage: string +}): ZedDecodedCall { + const input = num(opts.usage.input_tokens) + const output = num(opts.usage.output_tokens) + const cacheWrite = num(opts.usage.cache_creation_input_tokens) + const cacheRead = num(opts.usage.cache_read_input_tokens) + return { + provider: 'zed', + model: opts.model, + inputTokens: input, + outputTokens: output, + cacheCreationInputTokens: cacheWrite, + cacheReadInputTokens: cacheRead, + cachedInputTokens: cacheRead, + reasoningTokens: 0, + webSearchRequests: 0, + tools: [], + rawBashCommands: [], + timestamp: opts.timestamp, + speed: 'standard', + deduplicationKey: `zed:${opts.threadId}:${opts.requestKey}`, + userMessage: opts.userMessage, + sessionId: opts.threadId, + } +} + +export type ZedDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type ZedDecodeResult = { + calls: ZedDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode Zed `threads` rows into rich, cost-free calls. The per-request map is + * keyed by user message and does not cover every request (verified on a real + * thread: cumulative was ~3x the map sum), so a remainder entry tops the thread + * up to the exact cumulative counter. Threads with an empty map degrade to one + * cumulative call. Dedup is keyed on `zed::` against the + * live `seenKeys` set (host-owned). + */ +export function decodeZed({ records, seenKeys: liveSeen }: ZedDecodeInput): ZedDecodeResult { + const seen = liveSeen ?? new Set() + const calls: ZedDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + records.forEach((raw, index) => { + const row = raw as ZedThreadRow + try { + // Zed's DataType enum is "zstd" (current save path) or "json" (legacy + // uncompressed rows); anything else is unknown. + if (!row.id || !row.data || (row.data_type !== 'zstd' && row.data_type !== 'json')) { + if (row.data != null) diagnostics.push({ index, code: 'unknown-shape' }) + return + } + const parsedAt = new Date(row.updated_at ?? '') + if (Number.isNaN(parsedAt.getTime())) return + const timestamp = parsedAt.toISOString() + + const jsonText = row.data_type === 'zstd' + ? zstdDecompressSync!(Buffer.from(row.data)).toString('utf-8') + : Buffer.from(row.data).toString('utf-8') + const thread = JSON.parse(jsonText) as ZedThreadJson + const model = thread.model?.model || 'unknown' + const userMessage = row.summary ?? '' + + const requests = Object.entries(thread.request_token_usage ?? {}).filter(([, usage]) => usage != null && !usageIsEmpty(usage)) + // The per-request map is keyed by user message and does not cover every + // request, so a remainder entry tops the thread up to the exact + // cumulative counter. Threads with an empty map degrade to one + // cumulative call. + const entries: Array<[string, ZedTokenUsage]> = [...requests] + const cumulative = thread.cumulative_token_usage + if (cumulative && !usageIsEmpty(cumulative)) { + let sumIn = 0, sumOut = 0, sumWrite = 0, sumRead = 0 + for (const [, usage] of requests) { + sumIn += num(usage.input_tokens) + sumOut += num(usage.output_tokens) + sumWrite += num(usage.cache_creation_input_tokens) + sumRead += num(usage.cache_read_input_tokens) + } + const remainder: ZedTokenUsage = { + input_tokens: Math.max(0, num(cumulative.input_tokens) - sumIn), + output_tokens: Math.max(0, num(cumulative.output_tokens) - sumOut), + cache_creation_input_tokens: Math.max(0, num(cumulative.cache_creation_input_tokens) - sumWrite), + cache_read_input_tokens: Math.max(0, num(cumulative.cache_read_input_tokens) - sumRead), + } + if (!usageIsEmpty(remainder)) entries.push(['cumulative-remainder', remainder]) + } + + for (const [requestKey, usage] of entries) { + const call = buildCall({ threadId: row.id, requestKey, usage, model, timestamp, userMessage }) + if (seen.has(call.deduplicationKey)) continue + seen.add(call.deduplicationKey) + calls.push(call) + } + } catch { + diagnostics.push({ index, code: 'malformed-json' }) + } + }) + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/zed/index.ts b/packages/core/src/providers/zed/index.ts new file mode 100644 index 00000000..2b07b555 --- /dev/null +++ b/packages/core/src/providers/zed/index.ts @@ -0,0 +1,28 @@ +// @codeburn/core Zed provider. +// +// Two layers: +// - Rich pure decode (`decodeZed`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied `threads` rows (the sqlite +// driver and SQL query stay CLI-side, Category B); decompresses and parses +// the zstd/json blob itself. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeZed, + type ZedDecodeInput, + type ZedDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichZedSessionDecode, + type ZedToObservationsContext, +} from './observations.js' + +export type { + ZedDecodedCall, + ZedThreadJson, + ZedThreadRow, + ZedTokenUsage, +} from './types.js' diff --git a/packages/core/src/providers/zed/observations.ts b/packages/core/src/providers/zed/observations.ts new file mode 100644 index 00000000..b01ca7b0 --- /dev/null +++ b/packages/core/src/providers/zed/observations.ts @@ -0,0 +1,82 @@ +// Minimizing transform: rich Zed decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the thread summary Zed captures as the +// rich decode's user message. Zed never records file paths, so there is +// nothing to fingerprint. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { ZedDecodedCall } from './types.js' + +/** One Zed thread's rich decode, as the host holds it before minimization. */ +export interface RichZedSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order (one per request / cumulative-remainder). */ + calls: ZedDecodedCall[] +} + +export interface ZedToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: ZedDecodedCall, turnIndex: number): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichZedSessionDecode, ctx: ZedToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'zed' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Zed decode (one or many threads) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichZedSessionDecode | RichZedSessionDecode[], + ctx: ZedToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/zed/types.ts b/packages/core/src/providers/zed/types.ts new file mode 100644 index 00000000..4d7bfc82 --- /dev/null +++ b/packages/core/src/providers/zed/types.ts @@ -0,0 +1,50 @@ +// Raw record + rich-decode types for the Zed provider. +// +// Zed's built-in agent stores one row per thread in a single SQLite database; +// the `data` blob is zstd-compressed JSON carrying `request_token_usage` +// (per-request Anthropic-shaped token counts) and the thread's model. The +// sqlite driver and SQL query stay CLI-side (Category B); the host hands this +// decoder the raw `threads` rows, blob and all — decompression + JSON parsing +// are pure and happen here. Format documented in issue #480. + +/** One `threads` row, moved verbatim from the CLI. */ +export type ZedThreadRow = { + id: string + summary: string | null + updated_at: string | null + data_type: string | null + data: Uint8Array | null +} + +export type ZedTokenUsage = { + input_tokens?: number + output_tokens?: number + cache_creation_input_tokens?: number + cache_read_input_tokens?: number +} + +export type ZedThreadJson = { + model?: { provider?: string; model?: string } + request_token_usage?: Record + cumulative_token_usage?: ZedTokenUsage +} + +/** The rich decode of one Zed request (per-request or cumulative-remainder), pre-pricing. */ +export type ZedDecodedCall = { + provider: 'zed' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index bc532460..19edab77 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -151,6 +151,12 @@ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/kimicode/types.ts', 'src/providers/pi/decode.ts', 'src/providers/pi/types.ts', + 'src/providers/zed/decode.ts', + 'src/providers/zed/types.ts', + 'src/providers/forge/decode.ts', + 'src/providers/forge/types.ts', + 'src/providers/goose/decode.ts', + 'src/providers/goose/types.ts', ]) describe('architecture gate: no classification or free text in @codeburn/core source', () => { diff --git a/packages/core/tests/content-smuggling.test.ts b/packages/core/tests/content-smuggling.test.ts index b1c23123..89a8c3b0 100644 --- a/packages/core/tests/content-smuggling.test.ts +++ b/packages/core/tests/content-smuggling.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs' import { dirname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' +import zlib from 'node:zlib' import { describe, expect, it } from 'vitest' @@ -25,7 +26,11 @@ import { decodeKimi, toObservations as toKimiObservations } from '../src/provide import { decodeCodeWhale, toObservations as toCodeWhaleObservations } from '../src/providers/codewhale/index.js' import { decodeCodebuff, toObservations as toCodebuffObservations } from '../src/providers/codebuff/index.js' import { decodeOpenClaw, toObservations as toOpenClawObservations } from '../src/providers/openclaw/index.js' +import { decodeZed, toObservations as toZedObservations } from '../src/providers/zed/index.js' +import { decodeForge, toObservations as toForgeObservations } from '../src/providers/forge/index.js' +import { decodeGoose, toObservations as toGooseObservations } from '../src/providers/goose/index.js' import type { DecodeContext } from '../src/contracts.js' +import type { ZedThreadRow } from '../src/providers/zed/index.js' const here = dirname(fileURLToPath(import.meta.url)) const goldenEnvelope = JSON.parse( @@ -687,6 +692,191 @@ describe('content-smuggling guardrail: real openclaw decode -> toObservations is }) }) +describe('content-smuggling guardrail: real zed decode -> toObservations is secret-free', () => { + // A hostile Zed thread planting every secret in the free-text fields the + // decode captures: the thread summary (user message) and the project path. + // Zed never records tool calls or file paths, so there is no command-line or + // path-fingerprint surface to probe here. + const zedContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'zed', sourceRef: 'ref' } + const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync + + function decodeAndMinimize() { + const thread = { + model: { provider: 'anthropic', model: 'claude-opus-4-8' }, + request_token_usage: { + 'req-1': { input_tokens: 500, output_tokens: 200 }, + }, + } + const row: ZedThreadRow = { + id: 'thread-hostile', + summary: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}`, + updated_at: '2026-07-17T10:00:00.000Z', + data_type: 'zstd', + data: zstd!(Buffer.from(JSON.stringify(thread))), + } + const { calls } = decodeZed({ records: [row], context: zedContext }) + const { sessions } = toZedObservations( + { sessionId: 'thread-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'zed' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it.skipIf(!zstd)('produces a schema-valid envelope from the hostile thread', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it.skipIf(!zstd)('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) +}) + +describe('content-smuggling guardrail: real forge decode -> toObservations is secret-free', () => { + // A hostile Forge conversation planting every secret in the free-text fields + // the decode captures: the user message, a Bash command, and a tool NAME + // carrying a command line (must fail the canonical-name filter). The `context` + // column arrives as a still-serialized JSON string, exactly as sqlite returns it. + const forgeContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'forge', sourceRef: 'ref' } + + function decodeAndMinimize() { + const context = { + messages: [ + { message: { text: { role: 'User', content: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` } } }, + { + message: { + text: { + role: 'Assistant', model: 'claude-opus-4-6', + tool_calls: [ + { name: 'shell', call_id: 'call-1', arguments: { command: SECRETS.commandLine } }, + // A hostile tool NAME carrying a command line: fails canonical charset. + { name: SECRETS.commandLine, call_id: 'call-2', arguments: {} }, + ], + }, + }, + usage: { prompt_tokens: { actual: 500 }, completion_tokens: { actual: 200 } }, + }, + ], + } + const records = [{ + conversation_id: 'conv-hostile', + title: 'hostile', + workspace_id: 1, + context: JSON.stringify(context), + created_at: '2026-07-17 10:00:00', + updated_at: '2026-07-17 10:00:05', + }] + const { calls } = decodeForge({ records, context: forgeContext }) + const { sessions } = toForgeObservations( + { sessionId: 'conv-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'forge' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile conversation', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + +describe('content-smuggling guardrail: real goose decode -> toObservations is secret-free', () => { + // A hostile Goose session planting every secret in the free-text fields the + // decode captures: the first user message, a Bash command, a read_file path, + // and a tool NAME carrying a command line. BLOB columns arrive pre-converted + // to text (blobToText), exactly as the host hands them over. + const gooseContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'goose', sourceRef: 'ref' } + + function decodeAndMinimize() { + const records = [{ + sessionId: 'sess-hostile', + session: { + id: 'sess-hostile', + workingDir: SECRETS.absPath, + createdAt: '2026-07-17T10:00:00Z', + updatedAt: '2026-07-17T10:00:05Z', + accumulatedInputTokens: 500, + accumulatedOutputTokens: 200, + modelConfigJson: JSON.stringify({ model_name: 'gpt-5.5' }), + }, + assistantToolMessages: [ + { contentJson: JSON.stringify([ + { type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: SECRETS.commandLine } } } }, + { type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: SECRETS.absPath } } } }, + // A hostile tool NAME carrying a command line: fails canonical charset. + { type: 'toolRequest', toolCall: { value: { name: SECRETS.commandLine, arguments: {} } } }, + ]) }, + // A second turn so toolSequence (included only when it has MORE THAN ONE + // turn) actually reaches toObservations for fingerprinting. + { contentJson: JSON.stringify([ + { type: 'toolRequest', toolCall: { value: { name: 'developer__list_directory', arguments: {} } } }, + ]) }, + ], + firstUserMessage: { contentJson: JSON.stringify([{ type: 'text', text: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` }]) }, + }] + const { calls } = decodeGoose({ records, context: gooseContext }) + const { sessions } = toGooseObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'goose' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile session', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash/Read) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).toContain('Read') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) + + it('fingerprints the read_file path into a 16-hex resourceRead, never the raw path', () => { + const env = decodeAndMinimize() + const reads = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + expect(reads.length).toBeGreaterThan(0) + for (const ref of reads) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + expect(allStrings(reads)).not.toContain(SECRETS.absPath) + }) +}) + describe('content-smuggling guardrail: diagnostic detail rejects paths', () => { it('rejects an absolute path', () => { expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false) diff --git a/packages/core/tests/providers/crush-decode.test.ts b/packages/core/tests/providers/crush-decode.test.ts new file mode 100644 index 00000000..5b634c7c --- /dev/null +++ b/packages/core/tests/providers/crush-decode.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' + +import { decodeCrush, toObservations } from '../../src/providers/crush/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { CrushRawRecord } from '../../src/providers/crush/index.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'crush', sourceRef: 'ref' } + +function record(overrides: Partial = {}): CrushRawRecord { + return { + id: 'sess-a', + prompt_tokens: 1234, + completion_tokens: 567, + cost: 0.0789, + created_at: 1_700_000_010, + updated_at: 1_700_000_999, + message_count: 3, + model: 'claude-sonnet-4-6', + ...overrides, + } +} + +describe('crush rich decode (moved to @codeburn/core)', () => { + it('decodes a session row with a real cost into a measured, cost-free rich call', () => { + const { calls } = decodeCrush({ records: [record()], context }) + expect(calls).toHaveLength(1) + const call = calls[0]! + + // No pricing crosses into the decode layer as a costBasis marker; the + // dollar figure is carried as `measuredCostUSD` only, for the host to map. + expect(call).not.toHaveProperty('costBasis') + expect(call.measuredCostUSD).toBeCloseTo(0.0789, 6) + expect(call.model).toBe('claude-sonnet-4-6') + expect(call.inputTokens).toBe(1234) + expect(call.outputTokens).toBe(567) + expect(call.deduplicationKey).toBe('crush:sess-a') + expect(call.sessionId).toBe('sess-a') + // Crush stores epoch seconds; 1_700_000_999 sec -> 2023-11-14T22:29:59.000Z. + expect(call.timestamp).toBe(new Date(1_700_000_999 * 1000).toISOString()) + }) + + it('omits measuredCostUSD when cost is zero (estimated fallback)', () => { + const { calls } = decodeCrush({ records: [record({ cost: 0, prompt_tokens: 5, completion_tokens: 5 })], context }) + expect(calls).toHaveLength(1) + expect(calls[0]).not.toHaveProperty('measuredCostUSD') + }) + + it('skips a session with zero tokens and zero cost', () => { + const { calls } = decodeCrush({ records: [record({ prompt_tokens: 0, completion_tokens: 0, cost: 0 })], context }) + expect(calls).toEqual([]) + }) + + it('threads a live seenKeys set so a repeated session id across passes drops', () => { + const seen = new Set() + const first = decodeCrush({ records: [record()], context, seenKeys: seen }).calls + expect(first).toHaveLength(1) + const again = decodeCrush({ records: [record()], context, seenKeys: seen }).calls + expect(again).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const { calls } = decodeCrush({ records: [record()], context }) + const { sessions } = toObservations( + { sessionId: 'sess-a', projectPath: '/Users/t/alpha', calls }, + { privacyKey: 'test-privacy-key', provider: 'crush' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + expect(sessions[0]?.calls[0]?.costBasis).toBe('measured') + expect(sessions[0]?.calls[0]?.measuredCostUSD).toBeCloseTo(0.0789, 6) + }) +}) diff --git a/packages/core/tests/providers/forge-decode.test.ts b/packages/core/tests/providers/forge-decode.test.ts new file mode 100644 index 00000000..e4b51dcc --- /dev/null +++ b/packages/core/tests/providers/forge-decode.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest' + +import { decodeForge, toObservations } from '../../src/providers/forge/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { ForgeConversationRow } from '../../src/providers/forge/index.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'forge', sourceRef: 'ref' } + +function row(context_: unknown, overrides: Partial = {}): ForgeConversationRow { + return { + conversation_id: 'conv-1', + title: 'Forge Project', + workspace_id: 123, + context: JSON.stringify(context_), + created_at: '2026-05-06 15:00:00', + updated_at: '2026-05-06 15:20:41.379094', + ...overrides, + } +} + +describe('forge rich decode (moved to @codeburn/core)', () => { + it('maps tool calls, attributes the nearest previous user message, and emits raw bash commands', () => { + const conv = { + messages: [ + { message: { text: { role: 'User', content: 'implement forge' } } }, + { + message: { + text: { + role: 'Assistant', content: '', model: 'claude-opus-4-6', + tool_calls: [ + { name: 'shell', call_id: 'call-1', arguments: { command: 'git status && npm test' } }, + { name: 'Read', call_id: 'call-2', arguments: { file_path: '/tmp/a' } }, + ], + }, + }, + usage: { prompt_tokens: { actual: 1200 }, completion_tokens: { actual: 300 }, cached_tokens: { actual: 200 } }, + }, + ], + } + const { calls } = decodeForge({ records: [row(conv)], context }) + expect(calls).toHaveLength(1) + const call = calls[0]! + expect(call).not.toHaveProperty('costUSD') + expect(call).not.toHaveProperty('costBasis') + expect(call.model).toBe('claude-opus-4-6') + expect(call.inputTokens).toBe(1000) // 1200 - 200 cached + expect(call.cacheReadInputTokens).toBe(200) + expect(call.tools).toEqual(['Bash', 'Read']) + // Raw command strings survive host-side; base-name extraction is the CLI's job. + expect(call.rawBashCommands).toEqual(['git status && npm test']) + expect(call.userMessage).toBe('implement forge') + expect(call.deduplicationKey).toBe('forge:conv-1:call-1') + expect(call.timestamp).toBe('2026-05-06T15:20:41.379Z') + }) + + it('skips zero-token assistant messages', () => { + const conv = { + messages: [ + { message: { text: { role: 'User', content: 'zero tokens' } } }, + { message: { text: { role: 'Assistant', model: 'claude-opus-4-6' } }, usage: { prompt_tokens: { actual: 0 }, completion_tokens: { actual: 0 } } }, + ], + } + expect(decodeForge({ records: [row(conv)], context }).calls).toEqual([]) + }) + + it('yields nothing for invalid JSON context without throwing', () => { + expect(decodeForge({ records: [row(null, { context: '{invalid' })], context }).calls).toEqual([]) + }) + + it('threads a live seenKeys set so a repeated call_id across passes drops', () => { + const conv = { + messages: [ + { message: { text: { role: 'User', content: 'hi' } } }, + { message: { text: { role: 'Assistant', model: 'm', tool_calls: [{ name: 'shell', call_id: 'call-1', arguments: {} }] } }, usage: { prompt_tokens: { actual: 5 }, completion_tokens: { actual: 5 } } }, + ], + } + const seen = new Set() + expect(decodeForge({ records: [row(conv)], context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeForge({ records: [row(conv)], context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const conv = { + messages: [ + { message: { text: { role: 'User', content: 'implement forge' } } }, + { message: { text: { role: 'Assistant', model: 'claude-opus-4-6', tool_calls: [{ name: 'shell', call_id: 'call-1', arguments: { command: 'npm test' } }] } }, usage: { prompt_tokens: { actual: 100 }, completion_tokens: { actual: 20 } } }, + ], + } + const { calls } = decodeForge({ records: [row(conv)], context }) + const { sessions } = toObservations( + { sessionId: 'conv-1', projectPath: '/Users/t/proj', calls }, + { privacyKey: 'test-privacy-key', provider: 'forge' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + expect(JSON.stringify(envelope)).not.toContain('implement forge') + }) +}) diff --git a/packages/core/tests/providers/goose-decode.test.ts b/packages/core/tests/providers/goose-decode.test.ts new file mode 100644 index 00000000..ebb4f2b1 --- /dev/null +++ b/packages/core/tests/providers/goose-decode.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' + +import { decodeGoose, toObservations } from '../../src/providers/goose/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { GooseSessionRecords } from '../../src/providers/goose/index.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'goose', sourceRef: 'ref' } + +function sessionRecords(overrides: Partial = {}): GooseSessionRecords { + return { + sessionId: 'sess-1', + session: { + id: 'sess-1', + workingDir: '/Users/me/project', + createdAt: '2026-06-01T10:00:00Z', + updatedAt: '2026-06-01T10:05:30Z', + accumulatedInputTokens: 1500, + accumulatedOutputTokens: 400, + modelConfigJson: JSON.stringify({ model_name: 'gpt-5.5', reasoning: true }), + }, + assistantToolMessages: [ + { contentJson: JSON.stringify([ + { type: 'text', text: 'working on it' }, + { type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'npm test && npm run lint' } } } }, + { type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: 'src/index.ts' } } } }, + ]) }, + { contentJson: JSON.stringify([ + { type: 'toolRequest', toolCall: { value: { name: 'some_native_tool', arguments: {} } } }, + ]) }, + ], + firstUserMessage: { contentJson: JSON.stringify([{ type: 'text', text: 'please refactor the shell wrapper and run tests' }]) }, + ...overrides, + } +} + +describe('goose rich decode (moved to @codeburn/core)', () => { + it('decodes a session into a single rich call with mapped tools, raw commands, and a two-turn toolSequence', () => { + const { calls } = decodeGoose({ records: [sessionRecords()], context }) + expect(calls).toHaveLength(1) + const call = calls[0]! + + expect(call).not.toHaveProperty('costUSD') + expect(call).not.toHaveProperty('costBasis') + expect(call.model).toBe('gpt-5.5') + expect(call.inputTokens).toBe(1500) + expect(call.outputTokens).toBe(400) + expect(call.tools).toEqual(['Bash', 'Read', 'some_native_tool']) + expect(call.rawBashCommands).toEqual(['npm test && npm run lint']) + expect(call.userMessage).toBe('please refactor the shell wrapper and run tests') + expect(call.deduplicationKey).toBe('goose:sess-1') + expect(call.timestamp).toBe('2026-06-01T10:05:30.000Z') + // toolSequence included only when it has MORE than one turn. + expect(call.toolSequence).toHaveLength(2) + expect(call.toolSequence?.[0]?.[1]).toEqual({ tool: 'Read', file: 'src/index.ts' }) + }) + + it('omits toolSequence when only one turn used tools', () => { + const records = sessionRecords({ + assistantToolMessages: [ + { contentJson: JSON.stringify([{ type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'ls' } } } }]) }, + ], + }) + const { calls } = decodeGoose({ records: [records], context }) + expect(calls[0]!.toolSequence).toBeUndefined() + }) + + it('falls back to "unknown" model when model_config_json is missing/malformed', () => { + const records = sessionRecords({ session: { ...sessionRecords().session, modelConfigJson: null } }) + expect(decodeGoose({ records: [records], context }).calls[0]!.model).toBe('unknown') + }) + + it('skips a session with zero accumulated tokens', () => { + const records = sessionRecords({ session: { ...sessionRecords().session, accumulatedInputTokens: 0, accumulatedOutputTokens: 0 } }) + expect(decodeGoose({ records: [records], context }).calls).toEqual([]) + }) + + it('resolves to an empty timestamp (never the clock) when both timestamp columns are unparseable', () => { + const records = sessionRecords({ session: { ...sessionRecords().session, createdAt: 'not-a-date', updatedAt: null } }) + expect(decodeGoose({ records: [records], context }).calls[0]!.timestamp).toBe('') + }) + + it('threads a live seenKeys set so a repeated session id across passes drops', () => { + const seen = new Set() + expect(decodeGoose({ records: [sessionRecords()], context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeGoose({ records: [sessionRecords()], context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope and fingerprints the read_file path', () => { + const { calls } = decodeGoose({ records: [sessionRecords()], context }) + const { sessions } = toObservations( + { sessionId: 'sess-1', projectPath: '/Users/me/project', calls }, + { privacyKey: 'test-privacy-key', provider: 'goose' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + expect(reads.length).toBeGreaterThan(0) + for (const ref of reads) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + expect(JSON.stringify(envelope)).not.toContain('please refactor') + }) +}) diff --git a/packages/core/tests/providers/zcode-decode.test.ts b/packages/core/tests/providers/zcode-decode.test.ts new file mode 100644 index 00000000..c3a652e3 --- /dev/null +++ b/packages/core/tests/providers/zcode-decode.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' + +import { decodeZcode, toObservations } from '../../src/providers/zcode/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { ZcodeSessionRecords } from '../../src/providers/zcode/index.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'zcode', sourceRef: 'ref' } + +function records(): ZcodeSessionRecords { + return { + sessionId: 'sess-1', + usageRows: [ + { id: 'mu-1', turn_id: 'turn-1', model_id: 'GLM-5.2', input_tokens: 9125, output_tokens: 27, reasoning_tokens: 12, cache_creation_input_tokens: 0, cache_read_input_tokens: 8064, started_at: 1781981181862, completed_at: 1781981202412 }, + { id: 'mu-2', turn_id: 'turn-1', model_id: 'GLM-5.2', input_tokens: 200, output_tokens: 40, reasoning_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, started_at: 1781981210000, completed_at: 1781981220000 }, + // Zero-token row: skipped. + { id: 'mu-zero', turn_id: 'turn-2', model_id: 'GLM-5.2', input_tokens: 0, output_tokens: 0, reasoning_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, started_at: 1781981230000, completed_at: 1781981231000 }, + // No turn_id: no tools attached. + { id: 'mu-3', turn_id: null, model_id: 'GLM-5.2', input_tokens: 500, output_tokens: 60, reasoning_tokens: 0, cache_creation_input_tokens: 100, cache_read_input_tokens: 0, started_at: 1781981240000, completed_at: null }, + ], + toolRows: [ + { turn_id: 'turn-1', tool_name: 'Bash' }, + { turn_id: 'turn-1', tool_name: 'Read' }, + ], + } +} + +describe('zcode rich decode (moved to @codeburn/core)', () => { + it('decodes usage rows, splitting cached tokens and skipping zero-token rows', () => { + const { calls } = decodeZcode({ records: [records()], context }) + expect(calls).toHaveLength(3) + + const [first, second, third] = calls + expect(first).not.toHaveProperty('costUSD') + expect(first).not.toHaveProperty('costBasis') + expect(first!.inputTokens).toBe(1061) // 9125 - 8064 cached + expect(first!.cacheReadInputTokens).toBe(8064) + expect(first!.reasoningTokens).toBe(12) + expect(first!.tools).toEqual(['Bash', 'Read']) + expect(first!.turnId).toBe('turn-1') + expect(first!.deduplicationKey).toBe('zcode:mu-1') + + // Same turn's second row gets no tools (already attached to the first). + expect(second!.tools).toEqual([]) + expect(second!.turnId).toBe('turn-1') + + // No turn_id -> no tools, turnId undefined. + expect(third!.tools).toEqual([]) + expect(third!.turnId).toBeUndefined() + expect(third!.cacheCreationInputTokens).toBe(100) + }) + + it('threads a live seenKeys set so a repeated row id across passes drops', () => { + const seen = new Set() + const first = decodeZcode({ records: [records()], context, seenKeys: seen }).calls + expect(first).toHaveLength(3) + const again = decodeZcode({ records: [records()], context, seenKeys: seen }).calls + expect(again).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const { calls } = decodeZcode({ records: [records()], context }) + const { sessions } = toObservations( + { sessionId: 'sess-1', projectPath: '/Users/me/proj', calls }, + { privacyKey: 'test-privacy-key', provider: 'zcode' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + expect(sessions[0]?.calls.every(c => c.costBasis === 'estimated')).toBe(true) + }) +}) diff --git a/packages/core/tests/providers/zed-decode.test.ts b/packages/core/tests/providers/zed-decode.test.ts new file mode 100644 index 00000000..77e5cec2 --- /dev/null +++ b/packages/core/tests/providers/zed-decode.test.ts @@ -0,0 +1,106 @@ +import zlib from 'node:zlib' + +import { describe, expect, it } from 'vitest' + +import { decodeZed, toObservations } from '../../src/providers/zed/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { ZedThreadRow } from '../../src/providers/zed/index.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'zed', sourceRef: 'ref' } + +const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync +const skipReason = !zstd ? 'zlib zstd not available — needs Node 22.15+; skipping' : null + +function zstdRow(id: string, thread: unknown, opts: Partial = {}): ZedThreadRow { + return { + id, + summary: opts.summary ?? 'a thread', + updated_at: opts.updated_at ?? '2026-06-20T10:00:00Z', + data_type: 'zstd', + data: zstd!(Buffer.from(JSON.stringify(thread))), + ...opts, + } +} + +describe.skipIf(skipReason !== null)('zed rich decode (moved to @codeburn/core)', () => { + it('emits one call per request, plus a cumulative-remainder entry when the map undercounts', () => { + const row = zstdRow('thread-1', { + model: { provider: 'anthropic', model: 'claude-opus-4-8' }, + request_token_usage: { + 'req-1': { input_tokens: 1200, output_tokens: 300, cache_creation_input_tokens: 5000, cache_read_input_tokens: 90000 }, + 'req-2': { input_tokens: 800, output_tokens: 150 }, + }, + cumulative_token_usage: { input_tokens: 2000, output_tokens: 450 }, + }, { summary: 'refactor the parser', updated_at: '2026-06-21T09:30:00Z' }) + + const { calls } = decodeZed({ records: [row], context }) + expect(calls).toHaveLength(2) + const first = calls.find(c => c.deduplicationKey === 'zed:thread-1:req-1')! + expect(first.inputTokens).toBe(1200) + expect(first.cacheCreationInputTokens).toBe(5000) + expect(first.model).toBe('claude-opus-4-8') + expect(first.userMessage).toBe('refactor the parser') + expect(first.timestamp).toBe('2026-06-21T09:30:00.000Z') + expect(calls.reduce((s, c) => s + c.inputTokens, 0)).toBe(2000) + }) + + it('skips non-zstd/json rows and malformed blobs, recording a diagnostic without dropping healthy threads', () => { + const badType: ZedThreadRow = { id: 'bad-type', summary: null, updated_at: '2026-06-20T10:00:00Z', data_type: 'protobuf', data: Buffer.from('{}') } + const badBlob: ZedThreadRow = { id: 'bad-blob', summary: null, updated_at: '2026-06-20T10:00:00Z', data_type: 'zstd', data: Buffer.from('not zstd at all') } + const good = zstdRow('good', { model: { model: 'claude-opus-4-8' }, request_token_usage: { 'req-1': { input_tokens: 10, output_tokens: 5 } } }) + + const { calls, diagnostics } = decodeZed({ records: [badType, badBlob, good], context }) + expect(calls).toHaveLength(1) + expect(calls[0]!.sessionId).toBe('good') + expect(diagnostics.length).toBeGreaterThan(0) + }) + + it('reads legacy uncompressed json rows', () => { + const row: ZedThreadRow = { + id: 'legacy', summary: null, updated_at: '2026-06-22T11:00:00Z', data_type: 'json', + data: Buffer.from(JSON.stringify({ model: { model: 'claude-sonnet-4-6' }, request_token_usage: { 'req-1': { input_tokens: 40, output_tokens: 8 } } })), + } + const { calls } = decodeZed({ records: [row], context }) + expect(calls).toHaveLength(1) + expect(calls[0]!.inputTokens).toBe(40) + expect(calls[0]!.model).toBe('claude-sonnet-4-6') + }) + + it('skips threads whose usage is entirely zero', () => { + const row = zstdRow('zero', { + model: { model: 'claude-opus-4-8' }, + request_token_usage: { 'req-1': { input_tokens: 0, output_tokens: 0 } }, + cumulative_token_usage: { input_tokens: 0, output_tokens: 0 }, + }) + expect(decodeZed({ records: [row], context }).calls).toEqual([]) + }) + + it('threads a live seenKeys set so a repeated thread across passes drops', () => { + const row = zstdRow('thread-3', { model: { model: 'claude-opus-4-8' }, request_token_usage: { 'req-1': { input_tokens: 100, output_tokens: 50 } } }) + const seen = new Set() + expect(decodeZed({ records: [row], context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeZed({ records: [row], context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const row = zstdRow('thread-4', { + model: { model: 'claude-opus-4-8' }, + request_token_usage: { 'req-1': { input_tokens: 100, output_tokens: 50 } }, + }, { summary: 'do the thing' }) + const { calls } = decodeZed({ records: [row], context }) + const { sessions } = toObservations( + { sessionId: 'thread-4', projectPath: '/Users/t/proj', calls }, + { privacyKey: 'test-privacy-key', provider: 'zed' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + // The thread summary must never cross into the envelope. + expect(JSON.stringify(envelope)).not.toContain('do the thing') + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 1f181980..d84bb475 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -27,6 +27,11 @@ export default defineConfig({ 'src/providers/gemini/index.ts', 'src/providers/kimicode/index.ts', 'src/providers/pi/index.ts', + 'src/providers/crush/index.ts', + 'src/providers/zcode/index.ts', + 'src/providers/zed/index.ts', + 'src/providers/forge/index.ts', + 'src/providers/goose/index.ts', ], format: ['esm'], target: 'node20',