diff --git a/packages/cli/src/providers/droid.ts b/packages/cli/src/providers/droid.ts index 7689f7c4..098b407a 100644 --- a/packages/cli/src/providers/droid.ts +++ b/packages/cli/src/providers/droid.ts @@ -2,86 +2,27 @@ import { readdir, stat, readFile } from 'fs/promises' import { join } from 'path' import { homedir } from 'os' +import { decodeDroid, droidToolNameMap, stripModelPrefix } from '@codeburn/core/providers/droid' +import type { DecodeContext } from '@codeburn/core' +import type { DroidDecodedCall, DroidJsonlEntry, DroidSettings } from '@codeburn/core/providers/droid' + import { readSessionFile, readSessionLines } from '../fs-utils.js' import { getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import { normalizeContentBlocks } from '../content-utils.js' -import type { - Provider, - SessionSource, - SessionParser, - ParsedProviderCall, -} from './types.js' - -const toolNameMap: Record = { - Read: 'Read', - Create: 'Create', - Edit: 'Edit', - MultiEdit: 'MultiEdit', - LS: 'LS', - Glob: 'Glob', - Grep: 'Grep', - Execute: 'Bash', - AskUser: 'AskUser', - TodoWrite: 'TodoWrite', - Skill: 'Skill', - Task: 'Agent', - WebSearch: 'WebSearch', - FetchUrl: 'FetchUrl', - GenerateDroid: 'GenerateDroid', - ExitSpecMode: 'ExitSpecMode', -} - -type DroidSettings = { - model?: string - tokenUsage?: { - inputTokens: number - outputTokens: number - cacheCreationTokens: number - cacheReadTokens: number - thinkingTokens: number - } -} +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' -type DroidContent = { - type: string - text?: string - name?: string - input?: Record -} - -type DroidMessage = { - role: string - content?: DroidContent[] -} - -type DroidJsonlEntry = { - type: string - id?: string - timestamp?: string - message?: DroidMessage - title?: string - cwd?: string -} +// Host-derived scalars the pure core decode needs, packed alongside the JSONL +// records so they cross the bridge's fixed decode signature. +type DroidMeta = { settings: DroidSettings } +type DroidPacked = { meta: DroidMeta; records: unknown[] } function getFactoryDir(): string { return process.env['FACTORY_DIR'] ?? join(homedir(), '.factory') } - -// Strip Droid-specific wrapper to get the model's display name. -// e.g. "custom:GLM-5.1-[Proxy]-0" -> "GLM-5.1" -// Cost lookup is handled by codeburn's existing calculateCost/getCanonicalName -// which normalizes case and strips date suffixes automatically. -function stripModelPrefix(raw: string): string { - return raw - .replace(/^custom:/, '') - .replace(/\[.*?\]/g, '') - .replace(/-\d+$/, '') - .replace(/-+$/, '') - .replace(/^-/, '') -} - +// Display-name reduction (report layer). Reuses core's stripModelPrefix so the +// wrapper-stripping logic lives in exactly one place. function parseModelForDisplay(raw: string): string { const stripped = stripModelPrefix(raw) const lower = stripped.toLowerCase() @@ -97,191 +38,49 @@ function parseModelForDisplay(raw: string): string { } /** - * Extract meaningful shell command names from a Droid Execute call. - * Droid frequently passes multi-line scripts (python -c "...", heredocs, etc.) - * where splitting on ;/&&/| produces noise tokens like '}', 'await', 'import'. - * Instead, extract only the primary command from each logical line. + * Extract meaningful shell command names from a Droid Execute call. Droid + * frequently passes multi-line scripts (python -c "...", heredocs, etc.) where + * splitting on ;/&&/| produces noise tokens like '}', 'await', 'import'. Reduce + * to the primary command on the first logical line. Host-side (with its + * strip-ansi dependency); the core decoder carries the raw command strings. */ function extractDroidBashCommands(command: string): string[] { if (!command || !command.trim()) return [] - const firstLine = command.split('\n')[0]!.trim() return extractBashCommands(firstLine) } -function createParser( - source: SessionSource, - seenKeys: Set, -): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here (`costBasis: 'estimated'`); the Droid base-name extraction runs +// on the raw command strings the decoder carried through. +function toProviderCall(rich: DroidDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - const content = await readSessionFile(source.path) - if (content === null) return - - // Read the companion settings file for token usage - const settingsPath = source.path.replace(/\.jsonl$/, '.settings.json') - let settings: DroidSettings = {} - try { - const raw = await readFile(settingsPath, 'utf-8') - settings = JSON.parse(raw) as DroidSettings - } catch { - // No settings file or parse error - } - - const lines = content.split('\n').filter(l => l.trim()) - let sessionId = '' - let sessionModelDisplay = settings.model ? stripModelPrefix(settings.model) : 'unknown' - let currentUserMessage = '' - - // Collect all assistant messages with their tools - const assistantCalls: Array<{ - id: string - timestamp: string - tools: string[] - bashCommands: string[] - }> = [] - - let pendingTools: string[] = [] - let pendingBashCommands: string[] = [] - - for (const line of lines) { - let entry: DroidJsonlEntry - try { - entry = JSON.parse(line) as DroidJsonlEntry - } catch { - continue - } - - if (entry.type === 'session_start') { - sessionId = entry.id ?? '' - continue - } - - if (entry.type !== 'message' || !entry.message) continue - - const msg = entry.message - - if (msg.role === 'user') { - // Extract user text from content - const texts = normalizeContentBlocks(msg.content) - .filter(c => c.type === 'text' && c.text) - .map(c => c.text!) - .filter(Boolean) - // Skip system-reminder-only messages - const nonSystemTexts = texts.filter(t => !t.startsWith('')) - if (nonSystemTexts.length > 0) { - currentUserMessage = nonSystemTexts.join(' ').slice(0, 500) - } - continue - } - - if (msg.role === 'assistant') { - const toolUses = normalizeContentBlocks(msg.content).filter(c => c.type === 'tool_use') - - for (const tu of toolUses) { - const toolName = tu.name ?? '' - pendingTools.push(toolNameMap[toolName] ?? toolName) - - if (toolName === 'Execute' && tu.input && typeof tu.input['command'] === 'string') { - pendingBashCommands.push(...extractDroidBashCommands(tu.input['command'] as string)) - } - } - - // Check if this assistant message has any text content (non-thinking) - const hasText = normalizeContentBlocks(msg.content).some(c => c.type === 'text' && c.text) - - // Only emit a call entry if there are tools or substantial text - if (pendingTools.length > 0 || hasText) { - assistantCalls.push({ - id: entry.id ?? `msg-${assistantCalls.length}`, - timestamp: entry.timestamp ?? '', - tools: [...pendingTools], - bashCommands: [...pendingBashCommands], - }) - pendingTools = [] - pendingBashCommands = [] - } - continue - } - } - - if (assistantCalls.length === 0) return - - // KNOWN LIMITATION: Droid records token usage only at session level - // (settings.tokenUsage), not per-message. We split evenly across the - // emitted assistant calls and price all of them at settings.model - // (the latest model the session used). For sessions where the user - // switched models mid-stream, costs are approximate — we have no - // ground-truth breakdown to attribute tokens per model. - const totalTokens = settings.tokenUsage - if (!totalTokens) return - - const totalInput = totalTokens.inputTokens ?? 0 - const totalOutput = totalTokens.outputTokens ?? 0 - const totalCacheCreation = totalTokens.cacheCreationTokens ?? 0 - const totalCacheRead = totalTokens.cacheReadTokens ?? 0 - const totalThinking = totalTokens.thinkingTokens ?? 0 - const numCalls = assistantCalls.length - - // Distribute evenly across calls - const inputPerCall = Math.floor(totalInput / numCalls) - const outputPerCall = Math.floor(totalOutput / numCalls) - const cacheCreationPerCall = Math.floor(totalCacheCreation / numCalls) - const cacheReadPerCall = Math.floor(totalCacheRead / numCalls) - const thinkingPerCall = Math.floor(totalThinking / numCalls) - - for (let i = 0; i < assistantCalls.length; i++) { - const call = assistantCalls[i] - - // Assign remainder to the last call - const isLast = i === assistantCalls.length - 1 - const inputTokens = isLast - ? totalInput - inputPerCall * (numCalls - 1) - : inputPerCall - const outputTokens = isLast - ? totalOutput - outputPerCall * (numCalls - 1) - : outputPerCall - const cacheCreationTokens = isLast - ? totalCacheCreation - cacheCreationPerCall * (numCalls - 1) - : cacheCreationPerCall - const cacheReadTokens = isLast - ? totalCacheRead - cacheReadPerCall * (numCalls - 1) - : cacheReadPerCall - const thinkingTokens = isLast - ? totalThinking - thinkingPerCall * (numCalls - 1) - : thinkingPerCall - - const dedupKey = `droid:${sessionId}:${call.id}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - // Use the call's timestamp, or session_start timestamp - const timestamp = call.timestamp || '' - - yield { - provider: 'droid', - model: sessionModelDisplay, - inputTokens, - outputTokens, - cacheCreationInputTokens: cacheCreationTokens, - cacheReadInputTokens: cacheReadTokens, - cachedInputTokens: cacheReadTokens, - reasoningTokens: thinkingTokens, - webSearchRequests: 0, - costBasis: 'estimated', - tools: call.tools, - bashCommands: call.bashCommands, - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: i === 0 ? currentUserMessage : '', - sessionId, - } - } - }, + provider: 'droid', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: rich.rawBashCommands.flatMap(c => extractDroidBashCommands(c)), + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } +function decode(input: { records: unknown[]; context: DecodeContext; seenKeys: Set }): { calls: DroidDecodedCall[] } { + const packed = input.records[0] as DroidPacked | undefined + if (!packed) return { calls: [] } + return decodeDroid({ records: packed.records, context: input.context, settings: packed.meta.settings, seenKeys: input.seenKeys }) +} + function isInternalSession(cwd: string, factoryDir: string): boolean { // Skip sessions whose cwd is the .factory directory itself (internal housekeeping) const normalized = cwd.replace(/\/+$/, '') @@ -370,7 +169,7 @@ export function createDroidProvider(factoryDir?: string): Provider { const base = factoryDir ?? getFactoryDir() const sessionsDir = join(base, 'sessions') - return { + return createBridgedProvider({ name: 'droid', displayName: 'Droid', @@ -379,20 +178,37 @@ export function createDroidProvider(factoryDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return droidToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { return discoverSessionsInDir(sessionsDir, base) }, - createSessionParser( - source: SessionSource, - seenKeys: Set, - ): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: read the JSONL turns and the companion settings file (which + // carries session-level token usage + model). Both are packed so the pure + // decoder — which distributes the session token totals across calls — can + // consume them through the bridge. + async readRecords(source: SessionSource): Promise { + const content = await readSessionFile(source.path) + if (content === null) return null + + const settingsPath = source.path.replace(/\.jsonl$/, '.settings.json') + let settings: DroidSettings = {} + try { + settings = JSON.parse(await readFile(settingsPath, 'utf-8')) as DroidSettings + } catch { + // No settings file or parse error — decoder yields nothing without usage. + } + + const lines = content.split('\n').filter(l => l.trim()) + const packed: DroidPacked = { meta: { settings }, records: lines } + return [packed] }, - } + + decode, + toProviderCall, + }) } export const droid = createDroidProvider() diff --git a/packages/cli/src/providers/lingtai-tui.ts b/packages/cli/src/providers/lingtai-tui.ts index 63f622d8..50ed71bd 100644 --- a/packages/cli/src/providers/lingtai-tui.ts +++ b/packages/cli/src/providers/lingtai-tui.ts @@ -2,36 +2,17 @@ import { readdir, readFile, stat } from 'fs/promises' import { basename, delimiter, dirname, join, resolve } from 'path' import { homedir } from 'os' +import { decodeLingTaiTui } from '@codeburn/core/providers/lingtai-tui' +import type { DecodeContext } from '@codeburn/core' +import type { LingTaiTuiDecodedCall, LingTaiAgentManifest } from '@codeburn/core/providers/lingtai-tui' + import { readSessionLines } from '../fs-utils.js' import { getShortModelName } from '../models.js' -import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { ParsedProviderCall, Provider, SessionSource } from './types.js' type JsonObject = Record -type LingTaiAgentManifest = { - agent_id?: string - agent_name?: string - address?: string - nickname?: string | null - llm?: { - model?: string - base_url?: string - } -} - -type LingTaiLedgerEntry = { - source?: string - em_id?: string - run_id?: string - ts?: string | number - input?: number | string - output?: number | string - thinking?: number | string - cached?: number | string - model?: string - endpoint?: string -} - type LingTaiProviderOptions = { lingtaiHomeOverride?: string defaultHomeOverride?: string @@ -44,6 +25,16 @@ type LingTaiHome = { projectPrefix?: string } +// Host-derived scalars the pure core decode needs, packed with the ledger lines. +type LingTaiMeta = { + agentId: string + fallbackModel: string + fallbackEndpoint: string + projectPath: string + project: string +} +type LingTaiPacked = { meta: LingTaiMeta; records: unknown[] } + function normalizeOptions(options?: string | LingTaiProviderOptions): LingTaiProviderOptions { return typeof options === 'string' ? { lingtaiHomeOverride: options } @@ -165,13 +156,6 @@ function stringField(obj: JsonObject | null, key: string): string | undefined { return typeof value === 'string' && value.trim() ? value : undefined } -function numericField(obj: JsonObject, key: keyof LingTaiLedgerEntry): number { - const raw = obj[key] - const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN - if (!Number.isFinite(n) || n <= 0) return 0 - return Math.trunc(n) -} - async function readJson(path: string): Promise { const raw = await readFile(path, 'utf-8').catch(() => null) if (!raw) return null @@ -217,64 +201,6 @@ function projectFromManifest(manifest: LingTaiAgentManifest | null, fallback: st return prefix ? `${prefix}-${name}` : name } -function parseTimestamp(raw: unknown): string { - if (typeof raw === 'number' && Number.isFinite(raw)) { - const ms = raw < 1e12 ? raw * 1000 : raw - return new Date(ms).toISOString() - } - if (typeof raw !== 'string' || !raw.trim()) return '' - const d = new Date(raw) - return Number.isNaN(d.getTime()) ? '' : d.toISOString() -} - -function parseLedgerLine(line: string | Buffer): LingTaiLedgerEntry | null { - const text = Buffer.isBuffer(line) ? line.toString('utf-8') : line - if (!text.trim()) return null - try { - const parsed = JSON.parse(text) as unknown - const obj = asObject(parsed) - return obj ? obj as LingTaiLedgerEntry : null - } catch { - return null - } -} - -function activityForSource(sourceLabel: string): { userMessage: string; tools: string[]; subagentTypes: string[] } { - const normalized = sourceLabel.trim().toLowerCase() - - if (normalized === 'tc_wake' || normalized.startsWith('tc_') || normalized.includes('wake')) { - return { - userMessage: 'LingTai task coordinator wake', - tools: ['Agent'], - subagentTypes: ['lingtai-task-coordinator'], - } - } - - if (normalized === 'daemon') { - return { - userMessage: 'LingTai daemon task', - tools: ['Agent'], - subagentTypes: ['lingtai-daemon'], - } - } - - if (normalized === 'summarize_apriori' || normalized.includes('summar')) { - return { - userMessage: 'LingTai planning summary', - tools: ['EnterPlanMode'], - subagentTypes: [], - } - } - - return { - userMessage: normalized === 'main' - ? 'LingTai main conversation' - : `LingTai ${sourceLabel || 'main'} conversation`, - tools: [], - subagentTypes: [], - } -} - async function discoverLedgersInHome(home: LingTaiHome): Promise { const entries = await readdir(home.path, { withFileTypes: true }).catch(() => []) const sources: SessionSource[] = [] @@ -313,91 +239,45 @@ async function discoverLedgers(homes: LingTaiHome[]): Promise { return sources } -function createParser(source: SessionSource): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. +// LingTai ledgers carry synthetic activity (no shell commands), so bashCommands +// is the decoder's empty raw list. +function toProviderCall(rich: LingTaiTuiDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - const agentDir = agentDirFromLedgerPath(source.path) - const manifest = await readAgentManifest(agentDir) - const agentId = manifest?.agent_id ?? basename(agentDir) - const fallbackModel = manifest?.llm?.model ?? 'unknown' - const fallbackEndpoint = manifest?.llm?.base_url ?? '' - const project = source.project || projectFromManifest(manifest, basename(agentDir)) - const projectPath = agentDir - - let lineNo = 0 - for await (const line of readSessionLines(source.path)) { - lineNo += 1 - const entry = parseLedgerLine(line) - if (!entry) continue - - const obj = entry as JsonObject - const inputTotal = numericField(obj, 'input') - const outputTokens = numericField(obj, 'output') - const reasoningTokens = numericField(obj, 'thinking') - const cachedInputTokens = numericField(obj, 'cached') - const totalTokens = inputTotal + outputTokens + reasoningTokens + cachedInputTokens - if (totalTokens === 0) continue - - // LingTai records provider-normalized input totals plus a separate - // cached count. Match CodeBurn's normal shape by billing cached tokens - // in cacheReadInputTokens, not again as fresh input. - const inputTokens = Math.max(0, inputTotal - cachedInputTokens) - const model = stringField(obj, 'model') ?? fallbackModel - const endpoint = stringField(obj, 'endpoint') ?? fallbackEndpoint - const timestamp = parseTimestamp(entry.ts) - const sourceLabel = stringField(obj, 'source') ?? 'main' - const emId = stringField(obj, 'em_id') ?? '' - const runId = stringField(obj, 'run_id') ?? '' - const sessionId = runId || `${agentId}:${sourceLabel}` - const activity = activityForSource(sourceLabel) - const dedupKey = [ - 'lingtai-tui', - source.path, - lineNo, - timestamp, - model, - endpoint, - sourceLabel, - emId, - runId, - inputTotal, - outputTokens, - reasoningTokens, - cachedInputTokens, - ].join(':') - - yield { - provider: 'lingtai-tui', - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: cachedInputTokens, - cachedInputTokens, - reasoningTokens, - webSearchRequests: 0, - costBasis: 'estimated', - tools: activity.tools, - bashCommands: [], - subagentTypes: activity.subagentTypes, - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - turnId: `${sessionId}:line:${lineNo}`, - userMessage: activity.userMessage, - sessionId, - project, - projectPath, - } - } - }, + provider: 'lingtai-tui', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: rich.rawBashCommands, + subagentTypes: rich.subagentTypes, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + turnId: rich.turnId, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + projectPath: rich.projectPath, + ...(rich.project !== undefined ? { project: rich.project } : {}), } } +function decode(input: { records: unknown[]; context: DecodeContext; seenKeys: Set }): { calls: LingTaiTuiDecodedCall[] } { + const packed = input.records[0] as LingTaiPacked | undefined + if (!packed) return { calls: [] } + return decodeLingTaiTui({ records: packed.records, context: input.context, ...packed.meta, seenKeys: input.seenKeys }) +} + export function createLingTaiTuiProvider(options?: string | LingTaiProviderOptions): Provider { const providerOptions = normalizeOptions(options) - return { + return createBridgedProvider({ name: 'lingtai-tui', displayName: 'LingTai TUI', @@ -413,10 +293,27 @@ export function createLingTaiTuiProvider(options?: string | LingTaiProviderOptio return discoverLedgers(await getLingTaiHomes(providerOptions)) }, - createSessionParser(source: SessionSource): SessionParser { - return createParser(source) + // I/O adapter: read the agent manifest (for model/endpoint/agent-id/project + // fallbacks) and the ledger JSONL lines, packing them for the pure decoder. + async readRecords(source: SessionSource): Promise { + const agentDir = agentDirFromLedgerPath(source.path) + const manifest = await readAgentManifest(agentDir) + const meta: LingTaiMeta = { + agentId: manifest?.agent_id ?? basename(agentDir), + fallbackModel: manifest?.llm?.model ?? 'unknown', + fallbackEndpoint: manifest?.llm?.base_url ?? '', + projectPath: agentDir, + project: source.project || projectFromManifest(manifest, basename(agentDir)), + } + const lines: string[] = [] + for await (const line of readSessionLines(source.path)) lines.push(line) + const packed: LingTaiPacked = { meta, records: lines } + return [packed] }, - } + + decode, + toProviderCall, + }) } export const lingtaiTui = createLingTaiTuiProvider() diff --git a/packages/cli/src/providers/mux.ts b/packages/cli/src/providers/mux.ts index dab5aa9a..0a813bf7 100644 --- a/packages/cli/src/providers/mux.ts +++ b/packages/cli/src/providers/mux.ts @@ -2,45 +2,19 @@ import { readdir, readFile, stat } from 'fs/promises' import { basename, dirname, join, resolve } from 'path' import { homedir } from 'os' +import { decodeMux, muxToolNameMap } from '@codeburn/core/providers/mux' +import type { DecodeContext } from '@codeburn/core' +import type { MuxDecodedCall } from '@codeburn/core/providers/mux' + import { readSessionLines } from '../fs-utils.js' import { getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' -import { safeNumber } from '../parser.js' - -const toolNameMap: Record = { - bash: 'Bash', - file_read: 'Read', - file_edit_replace_string: 'Edit', - file_edit_replace_lines: 'Edit', - file_edit_insert: 'Edit', - file_edit_operation: 'Edit', - web_fetch: 'WebFetch', - web_search: 'WebSearch', - task: 'Agent', - todo: 'TodoWrite', -} - -type MuxPart = { - type?: string - text?: string - toolName?: string - input?: unknown -} +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' -type MuxMessage = { - id?: string - role?: string - parts?: MuxPart[] - createdAt?: string - metadata?: { - model?: string - timestamp?: number - historySequence?: number - usage?: unknown - providerMetadata?: Record - } -} +// Host-derived scalars the pure core decode needs, packed with the JSONL lines. +type MuxMeta = { workspaceId: string } +type MuxPacked = { meta: MuxMeta; records: unknown[] } function expandHome(p: string): string { if (p === '~') return homedir() @@ -58,6 +32,7 @@ function getMuxRoot(override?: string): string { } // Splits on the first colon only, leaving any colon inside the id intact. +// Display-layer only; the decoder strips the prefix for the emitted call model. function stripProvider(model: string): string { const i = model.indexOf(':') return i >= 0 ? model.slice(i + 1) : model @@ -67,15 +42,6 @@ function asRecord(v: unknown): Record | undefined { return v !== null && typeof v === 'object' ? (v as Record) : undefined } -// Guard against non-finite / out-of-range ms, which make toISOString() throw. -function toIsoTimestamp(ts: unknown, createdAt: unknown): string { - if (typeof ts === 'number' && Number.isFinite(ts)) { - const d = new Date(ts) - if (!Number.isNaN(d.getTime())) return d.toISOString() - } - return typeof createdAt === 'string' ? createdAt : '' -} - // config.json shape: { projects: [[projectPath, { workspaces: [{ id }] }], ...] } async function loadProjectMap(root: string): Promise> { const map = new Map() @@ -147,108 +113,41 @@ async function discoverSessions(root: string): Promise { return sources } -function createParser(source: SessionSource, seenKeys: Set): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost +// re-enters here; extractBashCommands (with its strip-ansi dependency) runs on +// the raw scripts the decoder carried through. +function toProviderCall(rich: MuxDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - const workspaceId = basename(dirname(source.path)) - let pendingUserMessage = '' - let lineIdx = 0 - - for await (const line of readSessionLines(source.path)) { - lineIdx++ - let msg: MuxMessage - try { - msg = JSON.parse(line) as MuxMessage - } catch { - continue - } - if (!msg || typeof msg !== 'object') continue - - if (msg.role === 'user') { - const texts = (Array.isArray(msg.parts) ? msg.parts : []) - .filter(p => p?.type === 'text' && typeof p.text === 'string') - .map(p => p.text as string) - .filter(Boolean) - if (texts.length > 0) pendingUserMessage = texts.join(' ').slice(0, 500) - continue - } - - if (msg.role !== 'assistant') continue - const meta = msg.metadata - const usage = asRecord(meta?.usage) - if (!meta || !usage) continue - - const pm = meta.providerMetadata ?? {} - const anthropic = asRecord(pm['anthropic']) - - // mux reports inputTokens inclusive of cache read+creation and - // outputTokens inclusive of reasoning; decompose to codeburn's - // cache/reasoning-exclusive convention. Cache creation is Anthropic-only. - // The AI SDK v6 normalizes reasoning into usage.reasoningTokens across - // every provider family, so that field is the single source of truth. - const cacheRead = safeNumber(usage['cachedInputTokens']) - const cacheCreate = safeNumber(anthropic?.['cacheCreationInputTokens']) - const reasoning = safeNumber(usage['reasoningTokens']) - const inputTokens = Math.max(0, safeNumber(usage['inputTokens']) - cacheRead - cacheCreate) - const outputTokens = Math.max(0, safeNumber(usage['outputTokens']) - reasoning) - - if (inputTokens === 0 && outputTokens === 0 && cacheRead === 0 && cacheCreate === 0 && reasoning === 0) { - continue - } - - // Strip the "provider:" prefix — codeburn's getCanonicalName only strips - // slash prefixes, so a colon-prefixed model would price at $0. - const rawModel = typeof meta.model === 'string' && meta.model ? meta.model : 'unknown' - const model = stripProvider(rawModel) - const id = typeof msg.id === 'string' && msg.id ? msg.id : `L${lineIdx}` - const dedupKey = `mux:${workspaceId}:${id}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const toolParts = (Array.isArray(msg.parts) ? msg.parts : []).filter( - p => p?.type === 'dynamic-tool' && typeof p.toolName === 'string', - ) - const tools = toolParts.map(p => toolNameMap[p.toolName!] ?? p.toolName!) - const bashCommands = toolParts - .filter(p => p.toolName === 'bash') - .flatMap(p => { - const input = asRecord(p.input) - const script = input?.['script'] ?? input?.['command'] - return typeof script === 'string' ? extractBashCommands(script) : [] - }) - - const timestamp = toIsoTimestamp(meta.timestamp, msg.createdAt) - - yield { - provider: 'mux', - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: cacheCreate, - cacheReadInputTokens: cacheRead, - cachedInputTokens: cacheRead, - reasoningTokens: reasoning, - webSearchRequests: 0, - costBasis: 'estimated', - tools, - bashCommands, - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: pendingUserMessage, - sessionId: workspaceId, - } - - pendingUserMessage = '' - } - }, + provider: 'mux', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)), + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, } } +function decode(input: { records: unknown[]; context: DecodeContext; seenKeys: Set }): { calls: MuxDecodedCall[] } { + const packed = input.records[0] as MuxPacked | undefined + if (!packed) return { calls: [] } + return decodeMux({ records: packed.records, context: input.context, workspaceId: packed.meta.workspaceId, seenKeys: input.seenKeys }) +} + export function createMuxProvider(muxRoot?: string): Provider { const root = getMuxRoot(muxRoot) - return { + return createBridgedProvider({ name: 'mux', displayName: 'Mux', @@ -257,17 +156,26 @@ export function createMuxProvider(muxRoot?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return muxToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { return discoverSessions(root) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: read the chat JSONL lines and pack the workspace id (derived + // from the source path) the decoder folds into dedup keys and session ids. + async readRecords(source: SessionSource): Promise { + const workspaceId = basename(dirname(source.path)) + const lines: string[] = [] + for await (const line of readSessionLines(source.path)) lines.push(line) + const packed: MuxPacked = { meta: { workspaceId }, records: lines } + return [packed] }, - } + + decode, + toProviderCall, + }) } export const mux = createMuxProvider() diff --git a/packages/cli/src/providers/open-design.ts b/packages/cli/src/providers/open-design.ts index 6cb0d63b..4ce059a8 100644 --- a/packages/cli/src/providers/open-design.ts +++ b/packages/cli/src/providers/open-design.ts @@ -2,8 +2,13 @@ import { readdir, stat } from 'fs/promises' import { basename, dirname, join } from 'path' import { homedir, platform } from 'os' +import { decodeOpenDesign } from '@codeburn/core/providers/open-design' +import type { DecodeContext } from '@codeburn/core' +import type { OpenDesignDecodedCall } from '@codeburn/core/providers/open-design' + import { readSessionLines } from '../fs-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' const PROVIDER_NAME = 'open-design' const ENV_DIR = 'CODEBURN_OPEN_DESIGN_DIR' @@ -14,65 +19,9 @@ const modelDisplayNames = new Map([ ['GLM-5.2', 'GLM-5.2'], ]) -type OpenDesignEntry = { - id?: unknown - event?: unknown - data?: unknown - timestamp?: unknown -} - -type TokenUsage = { - inputTokens: number - outputTokens: number - cacheReadTokens: number - reasoningTokens: number -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} - -function tokenValue(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0 -} - -function timestampValue(value: unknown): string { - const text = stringValue(value) - if (text) return text - if (typeof value !== 'number' || !Number.isFinite(value)) return '' - - const date = new Date(value) - return Number.isNaN(date.getTime()) ? '' : date.toISOString() -} - -function parseEvent(line: string | Buffer): OpenDesignEntry | null { - const text = (typeof line === 'string' ? line : line.toString('utf-8')).trim() - if (!text) return null - - try { - const parsed = JSON.parse(text) as unknown - return isRecord(parsed) ? parsed : null - } catch { - return null - } -} - -function parseUsage(data: unknown): TokenUsage | null { - if (!isRecord(data) || data['type'] !== 'usage') return null - const usage = data['usage'] - if (!isRecord(usage)) return null - - return { - inputTokens: tokenValue(usage['input_tokens']), - outputTokens: tokenValue(usage['output_tokens']), - cacheReadTokens: tokenValue(usage['cached_read_tokens']), - reasoningTokens: tokenValue(usage['thought_tokens']), - } -} +// Host-derived scalars the pure core decode needs, packed with the event lines. +type OpenDesignMeta = { sessionId: string; project: string } +type OpenDesignPacked = { meta: OpenDesignMeta; records: unknown[] } function getOpenDesignDir(): string { const override = process.env[ENV_DIR] @@ -161,71 +110,39 @@ async function discoverOpenDesignSessions(baseDir: string): Promise): SessionParser { +// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Open +// Design records usage events only — no tools or shell commands. +function toProviderCall(rich: OpenDesignDecodedCall): ParsedProviderCall { return { - async *parse(): AsyncGenerator { - const sessionId = basename(dirname(source.path)) - let currentModel = '' - let fallbackEventCounter = 0 - - for await (const line of readSessionLines(source.path)) { - const entry = parseEvent(line) - if (!entry) continue - - const eventName = stringValue(entry.event) - const data = entry.data - - if (eventName === 'start' && isRecord(data)) { - const model = stringValue(data['model']) - if (model) currentModel = model - continue - } - - if (eventName !== 'agent' || !isRecord(data)) continue - - if (data['type'] === 'status') { - const model = stringValue(data['model']) - if (model) currentModel = model - continue - } - - const usage = parseUsage(data) - if (!usage || !currentModel) continue - - const eventId = stringValue(entry.id) ?? `line-${fallbackEventCounter++}` - const dedupKey = `${PROVIDER_NAME}:${sessionId}:${eventId}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const uncachedInputTokens = Math.max(0, usage.inputTokens - usage.cacheReadTokens) - - yield { - provider: PROVIDER_NAME, - sessionId, - project: source.project, - model: currentModel, - inputTokens: uncachedInputTokens, - outputTokens: usage.outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: usage.cacheReadTokens, - cachedInputTokens: usage.cacheReadTokens, - reasoningTokens: usage.reasoningTokens, - webSearchRequests: 0, - costBasis: 'estimated', - tools: [], - bashCommands: [], - timestamp: timestampValue(entry.timestamp), - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: '', - } - } - }, + provider: PROVIDER_NAME, + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: rich.rawBashCommands, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + ...(rich.project !== undefined ? { project: rich.project } : {}), } } +function decode(input: { records: unknown[]; context: DecodeContext; seenKeys: Set }): { calls: OpenDesignDecodedCall[] } { + const packed = input.records[0] as OpenDesignPacked | undefined + if (!packed) return { calls: [] } + return decodeOpenDesign({ records: packed.records, context: input.context, ...packed.meta, seenKeys: input.seenKeys }) +} + export function createOpenDesignProvider(overrideDir?: string): Provider { - return { + return createBridgedProvider({ name: PROVIDER_NAME, displayName: 'Open Design', @@ -241,10 +158,20 @@ export function createOpenDesignProvider(overrideDir?: string): Provider { return discoverOpenDesignSessions(overrideDir ?? getOpenDesignDir()) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + // I/O adapter: read the run's events.jsonl lines and pack the session id + // (derived from the run dir) plus the discovered project so the decoder can + // reproduce the pre-migration call shape. + async readRecords(source: SessionSource): Promise { + const sessionId = basename(dirname(source.path)) + const lines: string[] = [] + for await (const line of readSessionLines(source.path)) lines.push(line) + const packed: OpenDesignPacked = { meta: { sessionId, project: source.project }, records: lines } + return [packed] }, - } + + decode, + toProviderCall, + }) } export const openDesign = createOpenDesignProvider() diff --git a/packages/cli/src/providers/zerostack.ts b/packages/cli/src/providers/zerostack.ts index bbceeeec..85a82d6a 100644 --- a/packages/cli/src/providers/zerostack.ts +++ b/packages/cli/src/providers/zerostack.ts @@ -2,49 +2,61 @@ import { readdir } from 'fs/promises' import { basename, join } from 'path' import { homedir, platform } from 'os' +import { decodeZerostack, zerostackToolNameMap } from '@codeburn/core/providers/zerostack' +import type { DecodeContext } from '@codeburn/core' +import type { ZerostackDecodedCall } from '@codeburn/core/providers/zerostack' + import { readSessionFile } from '../fs-utils.js' import { getShortModelName } from '../models.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' - -// zerostack (https://github.com/gi-dellav/zerostack) is a minimal Rust coding -// agent. Each session is a single JSON file under /zerostack/sessions/. -// Token counts are stored as CUMULATIVE session totals (total_input_tokens, -// total_output_tokens, total_cost) — there is no per-call breakdown — so we emit -// one ParsedProviderCall per session. - -const toolNameMap: Record = { - bash: 'Bash', - read: 'Read', - write: 'Write', - edit: 'Edit', - grep: 'Grep', - glob: 'Glob', - fetch: 'WebFetch', - search: 'WebSearch', - task: 'Agent', +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' + +// The host-derived scalars the core decoder needs, packed alongside the session +// records so they reach the pure decode through the bridge's fixed decode +// signature (records + context + seenKeys only). +type ZerostackMeta = { project: string; sessionIdFallback: string } +type ZerostackPacked = { meta: ZerostackMeta; records: unknown[] } + +async function readSession(path: string): Promise { + const content = await readSessionFile(path) + if (content === null) return null + try { + return JSON.parse(content) as unknown + } catch { + return null + } } -type ZerostackMessage = { - role?: string - content?: string | Array<{ text?: string }> +function toProviderCall(rich: ZerostackDecodedCall): ParsedProviderCall { + return { + provider: 'zerostack', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: rich.rawBashCommands, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + ...(rich.project !== undefined ? { project: rich.project } : {}), + ...(rich.projectPath !== undefined ? { projectPath: rich.projectPath } : {}), + } } -type ZerostackSession = { - id?: string - messages?: ZerostackMessage[] - created_at?: string - updated_at?: string - total_input_tokens?: number - total_output_tokens?: number - model?: string - provider?: string - working_dir?: string +function decode(input: { records: unknown[]; context: DecodeContext; seenKeys: Set }): { calls: ZerostackDecodedCall[] } { + const packed = input.records[0] as ZerostackPacked | undefined + if (!packed) return { calls: [] } + return decodeZerostack({ records: packed.records, context: input.context, ...packed.meta, seenKeys: input.seenKeys }) } -// zerostack uses the platform data dir (Rust `dirs::data_dir`): macOS maps to -// ~/Library/Application Support, everything else to $XDG_DATA_HOME or -// ~/.local/share, then a `zerostack` subdir. ZS_DATA_DIR overrides the whole -// data dir (sessions live directly under it). Matches src/session/storage.rs. function defaultSessionsDir(): string { const override = process.env['ZS_DATA_DIR'] if (override) return join(override, 'sessions') @@ -55,72 +67,10 @@ function defaultSessionsDir(): string { return join(base, 'zerostack', 'sessions') } -function firstUserMessage(messages: ZerostackMessage[]): string { - const msg = messages.find(m => m.role === 'user') - if (!msg) return '' - if (typeof msg.content === 'string') return msg.content - return (msg.content ?? []).map(c => c.text ?? '').filter(Boolean).join(' ') -} - -async function readSession(path: string): Promise { - const content = await readSessionFile(path) - if (content === null) return null - try { - return JSON.parse(content) as ZerostackSession - } catch { - return null - } -} - -function createParser(source: SessionSource, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { - const session = await readSession(source.path) - if (!session) return - - const input = session.total_input_tokens ?? 0 - const output = session.total_output_tokens ?? 0 - if (input === 0 && output === 0) return - - const timestamp = session.updated_at ?? session.created_at ?? '' - const sessionId = session.id ?? basename(source.path, '.json') - const dedupKey = `${source.provider}:${source.path}:${timestamp}:${sessionId}` - if (seenKeys.has(dedupKey)) return - seenKeys.add(dedupKey) - - const model = session.model ?? '' - - yield { - provider: source.provider, - model, - inputTokens: input, - outputTokens: output, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - // zerostack persists only final assistant text, not tool-call records, - // so there is nothing to extract here. - tools: [], - bashCommands: [], - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: firstUserMessage(session.messages ?? []), - sessionId, - project: source.project, - projectPath: session.working_dir, - } - }, - } -} - export function createZerostackProvider(sessionsDir?: string): Provider { const dir = sessionsDir ?? defaultSessionsDir() - return { + return createBridgedProvider({ name: 'zerostack', displayName: 'Zerostack', @@ -130,7 +80,7 @@ export function createZerostackProvider(sessionsDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return zerostackToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { @@ -147,16 +97,26 @@ export function createZerostackProvider(sessionsDir?: string): Provider { const path = join(dir, file) const session = await readSession(path) if (!session) continue - const project = session.working_dir ? basename(session.working_dir) : basename(file, '.json') + const sessionObj = session as Record | null + const project = sessionObj?.working_dir ? basename(sessionObj.working_dir as string) : basename(file, '.json') sources.push({ path, project, provider: 'zerostack' }) } return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + async readRecords(source: SessionSource): Promise { + const session = await readSession(source.path) + if (!session) return null + const packed: ZerostackPacked = { + meta: { project: source.project, sessionIdFallback: basename(source.path, '.json') }, + records: [session], + } + return [packed] }, - } + + decode, + toProviderCall, + }) } export const zerostack = createZerostackProvider() diff --git a/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.jsonl b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.jsonl new file mode 100644 index 00000000..0a4813db --- /dev/null +++ b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.jsonl @@ -0,0 +1,4 @@ +{"type":"session_start","id":"sess-droid","cwd":"/Users/test/projects/proj-a","title":"parity"} +{"type":"message","id":"u1","timestamp":"2026-05-01T10:00:00.000Z","message":{"role":"user","content":[{"type":"text","text":"ignore me"},{"type":"text","text":"add a feature"}]}} +{"type":"message","id":"a1","timestamp":"2026-05-01T10:00:01.000Z","message":{"role":"assistant","content":[{"type":"tool_use","name":"Execute","input":{"command":"python3 - <<'PY'\nimport os\n}\nPY"}},{"type":"tool_use","name":"Read","input":{"file_path":"/tmp/a"}},{"type":"tool_use","name":"Task","input":{"prompt":"do work"}}]}} +{"type":"message","id":"a2","timestamp":"2026-05-01T10:00:02.000Z","message":{"role":"assistant","content":[{"type":"text","text":"second"}]}} diff --git a/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.settings.json b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.settings.json new file mode 100644 index 00000000..205cc323 --- /dev/null +++ b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid.settings.json @@ -0,0 +1 @@ +{"model":"custom:gpt-5-[Proxy]-0","tokenUsage":{"inputTokens":101,"outputTokens":51,"cacheCreationTokens":7,"cacheReadTokens":11,"thinkingTokens":5}} diff --git a/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.jsonl b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.jsonl new file mode 100644 index 00000000..57c94644 --- /dev/null +++ b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.jsonl @@ -0,0 +1,2 @@ +{"type":"session_start","id":"sess-droid2","cwd":"/Users/test/projects/proj-b"} +{"type":"message","id":"b1","timestamp":"2026-05-02T10:00:00.000Z","message":{"role":"assistant","content":[{"type":"text","text":"hi"}]}} diff --git a/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.settings.json b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.settings.json new file mode 100644 index 00000000..174ddc66 --- /dev/null +++ b/packages/cli/tests/fixtures/droid-parity/sessions/proj-a/sess-droid2.settings.json @@ -0,0 +1 @@ +{"tokenUsage":{"inputTokens":10,"outputTokens":5,"cacheCreationTokens":0,"cacheReadTokens":0,"thinkingTokens":0}} diff --git a/packages/cli/tests/fixtures/lingtai-parity/agent-x/.agent.json b/packages/cli/tests/fixtures/lingtai-parity/agent-x/.agent.json new file mode 100644 index 00000000..c82b93cc --- /dev/null +++ b/packages/cli/tests/fixtures/lingtai-parity/agent-x/.agent.json @@ -0,0 +1 @@ +{"agent_id":"agent-001","agent_name":"Operator","llm":{"model":"gpt-5.5","base_url":"example-endpoint"}} diff --git a/packages/cli/tests/fixtures/lingtai-parity/agent-x/logs/token_ledger.jsonl b/packages/cli/tests/fixtures/lingtai-parity/agent-x/logs/token_ledger.jsonl new file mode 100644 index 00000000..9aacb201 --- /dev/null +++ b/packages/cli/tests/fixtures/lingtai-parity/agent-x/logs/token_ledger.jsonl @@ -0,0 +1,3 @@ +{"source":"main","ts":"2026-06-04T01:25:09Z","input":100,"output":20,"thinking":5,"cached":10,"model":"gpt-5.5","endpoint":"example-endpoint"} +{"source":"tc_wake","ts":"2026-06-04T01:28:24Z","input":25,"output":5,"thinking":0,"cached":10} +{"source":"daemon","em_id":"em-1","run_id":"run-1","ts":"2026-06-04T01:30:24Z","input":50,"output":10,"thinking":0,"cached":50} diff --git a/packages/cli/tests/fixtures/mux-parity/config.json b/packages/cli/tests/fixtures/mux-parity/config.json new file mode 100644 index 00000000..9f1d6cbc --- /dev/null +++ b/packages/cli/tests/fixtures/mux-parity/config.json @@ -0,0 +1 @@ +{"projects":[["/Users/test/myproject",{"workspaces":[{"id":"ws-abc","name":"main","path":"/Users/test/myproject/wt-ws-abc"}]}]]} diff --git a/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/chat.jsonl b/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/chat.jsonl new file mode 100644 index 00000000..32426ccd --- /dev/null +++ b/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/chat.jsonl @@ -0,0 +1,2 @@ +{"id":"u1","role":"user","parts":[{"type":"text","text":"implement the feature"}],"metadata":{"historySequence":0,"timestamp":1776023210000}} +{"id":"msg-1","role":"assistant","parts":[{"type":"text","text":"done"},{"type":"dynamic-tool","toolName":"file_read","input":{}},{"type":"dynamic-tool","toolName":"bash","input":{"script":"git status && bun test"}}],"metadata":{"historySequence":1,"model":"anthropic:claude-opus-4-8","timestamp":1776023230000,"usage":{"inputTokens":1000,"outputTokens":230,"reasoningTokens":30,"cachedInputTokens":200},"providerMetadata":{"anthropic":{"cacheCreationInputTokens":50}}}} diff --git a/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/subagent-transcripts/child-a/chat.jsonl b/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/subagent-transcripts/child-a/chat.jsonl new file mode 100644 index 00000000..eb4fbd30 --- /dev/null +++ b/packages/cli/tests/fixtures/mux-parity/sessions/ws-abc/subagent-transcripts/child-a/chat.jsonl @@ -0,0 +1 @@ +{"id":"c1","role":"assistant","parts":[{"type":"text","text":"sub work"}],"metadata":{"historySequence":0,"model":"anthropic:claude-sonnet-4-6","timestamp":1776023240000,"usage":{"inputTokens":500,"outputTokens":100}}} diff --git a/packages/cli/tests/fixtures/zerostack/sess-abc.json b/packages/cli/tests/fixtures/zerostack/sess-abc.json new file mode 100644 index 00000000..d0aa8bed --- /dev/null +++ b/packages/cli/tests/fixtures/zerostack/sess-abc.json @@ -0,0 +1,19 @@ +{ + "id": "sess-abc", + "name": "", + "messages": [ + { "role": "user", "content": "hello, what is this repo about?", "estimated_tokens": 7 }, + { "role": "assistant", "content": "It is a minimal coding agent in Rust.", "estimated_tokens": 92 } + ], + "compactions": [], + "created_at": "2026-06-19T11:33:34.022836+00:00", + "updated_at": "2026-06-19T11:34:14.140631+00:00", + "total_input_tokens": 34119, + "total_output_tokens": 961, + "total_cost": 0.015677835, + "total_estimated_tokens": 446, + "model": "deepseek/deepseek-v4-pro", + "provider": "openrouter", + "working_dir": "/Users/test/myproject", + "permission_allowlist": [] +} diff --git a/packages/cli/tests/fixtures/zerostack/sess-array.json b/packages/cli/tests/fixtures/zerostack/sess-array.json new file mode 100644 index 00000000..1af49771 --- /dev/null +++ b/packages/cli/tests/fixtures/zerostack/sess-array.json @@ -0,0 +1,13 @@ +{ + "messages": [ + { "role": "user", "content": [{ "text": "part one" }, { "text": "part two" }] }, + { "role": "assistant", "content": "ok" } + ], + "compactions": [], + "created_at": "2026-06-20T09:00:00.000000+00:00", + "total_input_tokens": 500, + "total_output_tokens": 200, + "total_cost": 0.0, + "provider": "openrouter", + "working_dir": "/Users/test/another" +} diff --git a/packages/cli/tests/providers/droid-bridge.test.ts b/packages/cli/tests/providers/droid-bridge.test.ts new file mode 100644 index 00000000..2e7c0971 --- /dev/null +++ b/packages/cli/tests/providers/droid-bridge.test.ts @@ -0,0 +1,115 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, it, expect } from 'vitest' + +import { createDroidProvider } from '../../src/providers/droid.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the droid bridge migration (phase 8). Droid is +// not in the frozen corpus, so a committed fixture golden is THE parity gate: +// the bridged provider (discovery + companion-settings I/O CLI-side, pure decode +// delegated to @codeburn/core/providers/droid) must reproduce exactly what the +// pre-migration in-CLI decode produced. Covers: session-level token distribution +// with the remainder assigned to the last call, model-wrapper stripping and the +// 'unknown' fallback when settings omit a model, tool mapping (Execute->Bash, +// Task->Agent), Droid's first-line bash base-name extraction over a heredoc, +// system-reminder-only user text being dropped, and session:id dedup. +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/droid-parity') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'droid', + model: 'gpt-5', + inputTokens: 50, + outputTokens: 25, + cacheCreationInputTokens: 3, + cacheReadInputTokens: 5, + cachedInputTokens: 5, + reasoningTokens: 2, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Bash', 'Read', 'Agent'], + bashCommands: ['python3'], + timestamp: '2026-05-01T10:00:01.000Z', + speed: 'standard', + deduplicationKey: 'droid:sess-droid:a1', + userMessage: 'add a feature', + sessionId: 'sess-droid', + }, + { + provider: 'droid', + model: 'gpt-5', + inputTokens: 51, + outputTokens: 26, + cacheCreationInputTokens: 4, + cacheReadInputTokens: 6, + cachedInputTokens: 6, + reasoningTokens: 3, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-05-01T10:00:02.000Z', + speed: 'standard', + deduplicationKey: 'droid:sess-droid:a2', + userMessage: '', + sessionId: 'sess-droid', + }, + { + provider: 'droid', + model: 'unknown', + inputTokens: 10, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-05-02T10:00:00.000Z', + speed: 'standard', + deduplicationKey: 'droid:sess-droid2:b1', + userMessage: '', + sessionId: 'sess-droid2', + }, +] + +async function collect(seen = new Set()): Promise { + const provider = createDroidProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + } + return calls +} + +describe('droid bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + raw.map(priceProviderCall).forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('the shared seenKeys set dedups a repeat scan', async () => { + const seen = new Set() + const first = await collect(seen) + const second = await collect(seen) + expect(first).toEqual(GOLDEN) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/grok-bridge.test.ts b/packages/cli/tests/providers/grok-bridge.test.ts index 857d290c..65a298fc 100644 --- a/packages/cli/tests/providers/grok-bridge.test.ts +++ b/packages/cli/tests/providers/grok-bridge.test.ts @@ -33,7 +33,9 @@ const GOLDEN: ParsedProviderCall[] = [ subagentTypes: ['general-purpose'], timestamp: '2026-06-19T11:31:12.282793Z', speed: 'standard', - deduplicationKey: 'grok:/Users/torukmakto/Projects/codeburn/codeburn/.claude/worktrees/kimi-p8a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001:2026-06-19T11:31:12.282793Z:019edf9c-0000-7000-8000-000000000001', + // The key embeds the session dir's absolute path — compute it from + // FIXTURE_DIR so the golden is portable across checkouts. + deduplicationKey: `grok:${resolve(FIXTURE_DIR, '%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001')}:2026-06-19T11:31:12.282793Z:019edf9c-0000-7000-8000-000000000001`, userMessage: 'User asks about the repo', sessionId: '019edf9c-0000-7000-8000-000000000001', project: 'myproject', diff --git a/packages/cli/tests/providers/lingtai-tui-bridge.test.ts b/packages/cli/tests/providers/lingtai-tui-bridge.test.ts new file mode 100644 index 00000000..2632ee3f --- /dev/null +++ b/packages/cli/tests/providers/lingtai-tui-bridge.test.ts @@ -0,0 +1,140 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, it, expect } from 'vitest' + +import { createLingTaiTuiProvider } from '../../src/providers/lingtai-tui.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the lingtai-tui bridge migration (phase 8). +// Not in the frozen corpus, so a committed fixture golden is THE parity gate: +// the bridged provider (discovery + manifest/ledger I/O CLI-side, pure decode +// delegated to @codeburn/core/providers/lingtai-tui) must reproduce exactly what +// the pre-migration in-CLI decode produced. Covers: cached-input separation, +// per-source-label activity synthesis (main / tc_wake / daemon => userMessage + +// tools + subagentTypes), model/endpoint fallback from the manifest when a +// ledger row omits them, run_id vs `${agentId}:${label}` session ids, the +// composite dedup key threaded on the SOURCE PATH (not the agent dir), turnId, +// and the manifest-derived project / projectPath carried onto the call. +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/lingtai-parity') + +function dedup( + sourcePath: string, + lineNo: number, + ts: string, + model: string, + endpoint: string, + label: string, + emId: string, + runId: string, + input: number, + output: number, + thinking: number, + cached: number, +): string { + return ['lingtai-tui', sourcePath, lineNo, ts, model, endpoint, label, emId, runId, input, output, thinking, cached].join(':') +} + +function golden(sourcePath: string, agentDir: string): ParsedProviderCall[] { + return [ + { + provider: 'lingtai-tui', + model: 'gpt-5.5', + inputTokens: 90, + outputTokens: 20, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 10, + cachedInputTokens: 10, + reasoningTokens: 5, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + subagentTypes: [], + timestamp: '2026-06-04T01:25:09.000Z', + speed: 'standard', + deduplicationKey: dedup(sourcePath, 1, '2026-06-04T01:25:09.000Z', 'gpt-5.5', 'example-endpoint', 'main', '', '', 100, 20, 5, 10), + turnId: 'agent-001:main:line:1', + userMessage: 'LingTai main conversation', + sessionId: 'agent-001:main', + projectPath: agentDir, + project: 'Operator', + }, + { + provider: 'lingtai-tui', + model: 'gpt-5.5', + inputTokens: 15, + outputTokens: 5, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 10, + cachedInputTokens: 10, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Agent'], + bashCommands: [], + subagentTypes: ['lingtai-task-coordinator'], + timestamp: '2026-06-04T01:28:24.000Z', + speed: 'standard', + deduplicationKey: dedup(sourcePath, 2, '2026-06-04T01:28:24.000Z', 'gpt-5.5', 'example-endpoint', 'tc_wake', '', '', 25, 5, 0, 10), + turnId: 'agent-001:tc_wake:line:2', + userMessage: 'LingTai task coordinator wake', + sessionId: 'agent-001:tc_wake', + projectPath: agentDir, + project: 'Operator', + }, + { + provider: 'lingtai-tui', + model: 'gpt-5.5', + inputTokens: 0, + outputTokens: 10, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 50, + cachedInputTokens: 50, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Agent'], + bashCommands: [], + subagentTypes: ['lingtai-daemon'], + timestamp: '2026-06-04T01:30:24.000Z', + speed: 'standard', + deduplicationKey: dedup(sourcePath, 3, '2026-06-04T01:30:24.000Z', 'gpt-5.5', 'example-endpoint', 'daemon', 'em-1', 'run-1', 50, 10, 0, 50), + turnId: 'run-1:line:3', + userMessage: 'LingTai daemon task', + sessionId: 'run-1', + projectPath: agentDir, + project: 'Operator', + }, + ] +} + +async function sourceAndCalls(seen = new Set()): Promise<{ source: SessionSource; calls: ParsedProviderCall[] }> { + const provider = createLingTaiTuiProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + expect(sources).toHaveLength(1) + const source = sources[0]! + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + return { source, calls } +} + +describe('lingtai-tui bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + const { source, calls } = await sourceAndCalls() + const agentDir = dirname(dirname(source.path)) + expect(calls).toEqual(golden(source.path, agentDir)) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const { calls } = await sourceAndCalls() + calls.map(priceProviderCall).forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(calls[i]) + }) + }) +}) diff --git a/packages/cli/tests/providers/mux-bridge.test.ts b/packages/cli/tests/providers/mux-bridge.test.ts new file mode 100644 index 00000000..0d5a4df7 --- /dev/null +++ b/packages/cli/tests/providers/mux-bridge.test.ts @@ -0,0 +1,96 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, it, expect } from 'vitest' + +import { createMuxProvider } from '../../src/providers/mux.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the mux bridge migration (phase 8). Mux is not +// in the frozen corpus, so a committed fixture golden is THE parity gate: the +// bridged provider (discovery + JSONL I/O CLI-side, pure decode delegated to +// @codeburn/core/providers/mux) must reproduce exactly what the pre-migration +// in-CLI decode produced. Covers: inclusive input/output token decomposition +// (cache read+creation out of input, reasoning out of output), provider-prefix +// stripping for the model, tool mapping (file_read->Read, bash->Bash), +// extractBashCommands over `&&`, workspace-vs-subagent dedup keys derived from +// the source path, user-prompt pairing, and the model fallback for a sub-agent. +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/mux-parity') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'mux', + model: 'claude-opus-4-8', + inputTokens: 750, + outputTokens: 200, + cacheCreationInputTokens: 50, + cacheReadInputTokens: 200, + cachedInputTokens: 200, + reasoningTokens: 30, + webSearchRequests: 0, + costBasis: 'estimated', + tools: ['Read', 'Bash'], + bashCommands: ['git', 'bun'], + timestamp: new Date(1776023230000).toISOString(), + speed: 'standard', + deduplicationKey: 'mux:ws-abc:msg-1', + userMessage: 'implement the feature', + sessionId: 'ws-abc', + }, + { + provider: 'mux', + model: 'claude-sonnet-4-6', + inputTokens: 500, + outputTokens: 100, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: new Date(1776023240000).toISOString(), + speed: 'standard', + deduplicationKey: 'mux:child-a:c1', + userMessage: '', + sessionId: 'child-a', + }, +] + +async function collect(seen = new Set()): Promise { + const provider = createMuxProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + } + return calls +} + +describe('mux bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + raw.map(priceProviderCall).forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('the shared seenKeys set dedups a repeat scan', async () => { + const seen = new Set() + const first = await collect(seen) + const second = await collect(seen) + expect(first).toEqual(GOLDEN) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/open-design-bridge.test.ts b/packages/cli/tests/providers/open-design-bridge.test.ts new file mode 100644 index 00000000..e1a1f0b0 --- /dev/null +++ b/packages/cli/tests/providers/open-design-bridge.test.ts @@ -0,0 +1,123 @@ +import { dirname, join, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, it, expect } from 'vitest' + +import { createOpenDesignProvider } from '../../src/providers/open-design.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the open-design bridge migration (phase 8). +// Reuses the committed open-design fixture: the bridged provider (discovery + +// events.jsonl I/O CLI-side, pure decode delegated to +// @codeburn/core/providers/open-design) must reproduce exactly what the +// pre-migration in-CLI decode produced. Covers: the start-seeded model, model +// transitions via status events, uncached-input derivation, numeric-epoch +// timestamp normalization, per-event dedup that drops a repeated event id, the +// discovered project carried onto the call, and no-usage runs yielding nothing. +const here = dirname(fileURLToPath(import.meta.url)) +const DATA_DIR = resolve(here, '../fixtures/open-design/namespaces/release-stable/data') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'open-design', + model: 'openai-codex:gpt-5.5', + inputTokens: 950, + outputTokens: 200, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 50, + cachedInputTokens: 50, + reasoningTokens: 25, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-22T10:00:05.000Z', + speed: 'standard', + deduplicationKey: 'open-design:run-mixed:evt-codex-usage', + userMessage: '', + sessionId: 'run-mixed', + project: 'release-stable', + }, + { + provider: 'open-design', + model: 'glm-5.2', + inputTokens: 2900, + outputTokens: 400, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 100, + cachedInputTokens: 100, + reasoningTokens: 60, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-22T10:00:15.000Z', + speed: 'standard', + deduplicationKey: 'open-design:run-mixed:evt-glm-usage', + userMessage: '', + sessionId: 'run-mixed', + project: 'release-stable', + }, + { + provider: 'open-design', + model: 'glm-5.2', + inputTokens: 770, + outputTokens: 33, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 7, + cachedInputTokens: 7, + reasoningTokens: 3, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-22T12:00:05.000Z', + speed: 'standard', + deduplicationKey: 'open-design:run-start-seeded:evt-before-transition', + userMessage: '', + sessionId: 'run-start-seeded', + project: 'release-stable', + }, +] + +async function collect(seen = new Set()): Promise { + const provider = createOpenDesignProvider(DATA_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + } + return calls +} + +describe('open-design bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + raw.map(priceProviderCall).forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('the shared seenKeys set dedups a repeat scan', async () => { + const seen = new Set() + const first = await collect(seen) + const second = await collect(seen) + expect(first).toEqual(GOLDEN) + expect(second).toEqual([]) + }) + + it('the discovered run under fixtures resolves to the release-stable project', async () => { + const provider = createOpenDesignProvider(DATA_DIR) + const runs = (await provider.discoverSessions()).map(s => s.path) + expect(runs).toContain(join(DATA_DIR, 'runs', 'run-mixed', 'events.jsonl')) + }) +}) diff --git a/packages/cli/tests/providers/zerostack-bridge.test.ts b/packages/cli/tests/providers/zerostack-bridge.test.ts new file mode 100644 index 00000000..0c2a9c34 --- /dev/null +++ b/packages/cli/tests/providers/zerostack-bridge.test.ts @@ -0,0 +1,106 @@ +import { dirname, join, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, it, expect } from 'vitest' + +import { createZerostackProvider } from '../../src/providers/zerostack.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the zerostack bridge migration (phase 8). +// Zerostack is not in the frozen corpus, so a committed fixture golden is THE +// parity gate: the bridged provider (discovery + JSON I/O CLI-side, pure decode +// delegated to @codeburn/core/providers/zerostack) must reproduce exactly what +// the pre-migration in-CLI decode produced. The dedup key threads the absolute +// source path (`zerostack:::`), so it is built from +// the discovered source rather than hard-coded. Covers: cumulative session +// totals, the zero-token skip (elsewhere), the OpenRouter model passing through +// raw, the empty-model + string-array userMessage + `basename(path)` sessionId +// fallbacks, updated_at-then-created_at timestamp precedence, and the discovered +// project / recorded working_dir carried onto the call. +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/zerostack') + +function golden(dir: string): ParsedProviderCall[] { + const abcPath = join(dir, 'sess-abc.json') + const arrayPath = join(dir, 'sess-array.json') + return [ + { + provider: 'zerostack', + model: 'deepseek/deepseek-v4-pro', + inputTokens: 34119, + outputTokens: 961, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-19T11:34:14.140631+00:00', + speed: 'standard', + deduplicationKey: `zerostack:${abcPath}:2026-06-19T11:34:14.140631+00:00:sess-abc`, + userMessage: 'hello, what is this repo about?', + sessionId: 'sess-abc', + project: 'myproject', + projectPath: '/Users/test/myproject', + }, + { + provider: 'zerostack', + model: '', + inputTokens: 500, + outputTokens: 200, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + tools: [], + bashCommands: [], + timestamp: '2026-06-20T09:00:00.000000+00:00', + speed: 'standard', + deduplicationKey: `zerostack:${arrayPath}:2026-06-20T09:00:00.000000+00:00:sess-array`, + userMessage: 'part one part two', + sessionId: 'sess-array', + project: 'another', + projectPath: '/Users/test/another', + }, + ] +} + +async function collect(seen = new Set()): Promise { + const provider = createZerostackProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) calls.push(call) + } + return calls +} + +describe('zerostack bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(golden(FIXTURE_DIR)) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + raw.map(priceProviderCall).forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('the shared seenKeys set dedups a repeat scan', async () => { + const seen = new Set() + const first = await collect(seen) + const second = await collect(seen) + expect(first).toEqual(golden(FIXTURE_DIR)) + expect(second).toEqual([]) + }) +}) diff --git a/packages/core/package.json b/packages/core/package.json index 75c876cd..37fd5ebc 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -58,6 +58,26 @@ "./providers/qwen": { "types": "./dist/providers/qwen/index.d.ts", "import": "./dist/providers/qwen/index.js" + }, + "./providers/zerostack": { + "types": "./dist/providers/zerostack/index.d.ts", + "import": "./dist/providers/zerostack/index.js" + }, + "./providers/droid": { + "types": "./dist/providers/droid/index.d.ts", + "import": "./dist/providers/droid/index.js" + }, + "./providers/mux": { + "types": "./dist/providers/mux/index.d.ts", + "import": "./dist/providers/mux/index.js" + }, + "./providers/open-design": { + "types": "./dist/providers/open-design/index.d.ts", + "import": "./dist/providers/open-design/index.js" + }, + "./providers/lingtai-tui": { + "types": "./dist/providers/lingtai-tui/index.d.ts", + "import": "./dist/providers/lingtai-tui/index.js" } }, "files": [ diff --git a/packages/core/src/providers/droid/decode.ts b/packages/core/src/providers/droid/decode.ts new file mode 100644 index 00000000..3daef118 --- /dev/null +++ b/packages/core/src/providers/droid/decode.ts @@ -0,0 +1,220 @@ +// @codeburn/core Droid decoder: pure decode over supplied JSONL records + settings. +// No fs / env / clock — the host reads both the JSONL and settings files and hands them through. +// The rich output carries token buckets but NO pricing (cost leaves the decoder). + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { DroidDecodedCall, DroidJsonlEntry, DroidSettings, DroidContent } from './types.js' + +// Normalize content blocks: ensure array of block objects with minimal validation. +function normalizeContentBlocks( + content: T[] | string | null | undefined, +): T[] { + if (Array.isArray(content)) { + const isBlock = (b: T): boolean => b != null && typeof b === 'object' + return content.every(isBlock) ? content : content.filter(isBlock) + } + if (typeof content === 'string') return [{ type: 'text', text: content } as T] + return [] +} + +// Droid tool ids mapped to the canonical vocabulary. +export const droidToolNameMap: Record = { + Read: 'Read', + Create: 'Create', + Edit: 'Edit', + MultiEdit: 'MultiEdit', + LS: 'LS', + Glob: 'Glob', + Grep: 'Grep', + Execute: 'Bash', + AskUser: 'AskUser', + TodoWrite: 'TodoWrite', + Skill: 'Skill', + Task: 'Agent', + WebSearch: 'WebSearch', + FetchUrl: 'FetchUrl', + GenerateDroid: 'GenerateDroid', + ExitSpecMode: 'ExitSpecMode', +} + +// Strip Droid's model wrapper to the bare id (e.g. "custom:GLM-5.1-[Proxy]-0" +// -> "GLM-5.1"). Exported so the CLI's display-name path reuses the exact same +// reduction rather than duplicating it. +export function stripModelPrefix(raw: string): string { + return raw + .replace(/^custom:/, '') + .replace(/\[.*?\]/g, '') + .replace(/-\d+$/, '') + .replace(/-+$/, '') + .replace(/^-/, '') +} + +export type DroidDecodeInput = { + records: unknown[] + settings: DroidSettings + context: DecodeContext + seenKeys?: Set +} + +export type DroidDecodeResult = { + calls: DroidDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode Droid JSONL records into rich, cost-free calls. + * Droid stores token usage at session level; we distribute it evenly across assistant calls. + */ +export function decodeDroid({ records, settings, seenKeys: liveSeen }: DroidDecodeInput): DroidDecodeResult { + const seen = liveSeen ?? new Set() + const calls: DroidDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + let sessionId = '' + const sessionModelDisplay = settings.model ? stripModelPrefix(settings.model) : 'unknown' + let currentUserMessage = '' + + // Collect all assistant messages with their tools + const assistantCalls: Array<{ + id: string + timestamp: string + tools: string[] + bashCommands: string[] + }> = [] + + let pendingTools: string[] = [] + let pendingBashCommands: string[] = [] + + for (const rawRecord of records) { + let entry: DroidJsonlEntry + if (typeof rawRecord === 'string') { + try { + entry = JSON.parse(rawRecord) as DroidJsonlEntry + } catch { + continue + } + } else { + entry = rawRecord as DroidJsonlEntry + } + + if (entry.type === 'session_start') { + sessionId = entry.id ?? '' + continue + } + + if (entry.type !== 'message' || !entry.message) continue + + const msg = entry.message + + if (msg.role === 'user') { + const texts = normalizeContentBlocks(msg.content) + .filter(c => c.type === 'text' && c.text) + .map(c => c.text!) + .filter(Boolean) + // Skip system-reminder-only messages + const nonSystemTexts = texts.filter(t => !t.startsWith('')) + if (nonSystemTexts.length > 0) { + currentUserMessage = nonSystemTexts.join(' ').slice(0, 500) + } + continue + } + + if (msg.role === 'assistant') { + const toolUses = normalizeContentBlocks(msg.content).filter(c => c.type === 'tool_use') + + for (const tu of toolUses) { + const toolName = tu.name ?? '' + pendingTools.push(droidToolNameMap[toolName] ?? toolName) + + // Emit the RAW command string; the host runs Droid's base-name + // extraction (first-line + extractBashCommands, with its strip-ansi dep). + if (toolName === 'Execute' && tu.input && typeof tu.input['command'] === 'string') { + pendingBashCommands.push(tu.input['command'] as string) + } + } + + const hasText = normalizeContentBlocks(msg.content).some(c => c.type === 'text' && c.text) + + if (pendingTools.length > 0 || hasText) { + assistantCalls.push({ + id: entry.id ?? `msg-${assistantCalls.length}`, + timestamp: entry.timestamp ?? '', + tools: [...pendingTools], + bashCommands: [...pendingBashCommands], + }) + pendingTools = [] + pendingBashCommands = [] + } + continue + } + } + + if (assistantCalls.length === 0) return { calls, diagnostics } + + const totalTokens = settings.tokenUsage + if (!totalTokens) return { calls, diagnostics } + + const totalInput = totalTokens.inputTokens ?? 0 + const totalOutput = totalTokens.outputTokens ?? 0 + const totalCacheCreation = totalTokens.cacheCreationTokens ?? 0 + const totalCacheRead = totalTokens.cacheReadTokens ?? 0 + const totalThinking = totalTokens.thinkingTokens ?? 0 + const numCalls = assistantCalls.length + + // Distribute evenly across calls + const inputPerCall = Math.floor(totalInput / numCalls) + const outputPerCall = Math.floor(totalOutput / numCalls) + const cacheCreationPerCall = Math.floor(totalCacheCreation / numCalls) + const cacheReadPerCall = Math.floor(totalCacheRead / numCalls) + const thinkingPerCall = Math.floor(totalThinking / numCalls) + + for (let i = 0; i < assistantCalls.length; i++) { + const call = assistantCalls[i] + + // Assign remainder to the last call + const isLast = i === assistantCalls.length - 1 + const inputTokens = isLast + ? totalInput - inputPerCall * (numCalls - 1) + : inputPerCall + const outputTokens = isLast + ? totalOutput - outputPerCall * (numCalls - 1) + : outputPerCall + const cacheCreationTokens = isLast + ? totalCacheCreation - cacheCreationPerCall * (numCalls - 1) + : cacheCreationPerCall + const cacheReadTokens = isLast + ? totalCacheRead - cacheReadPerCall * (numCalls - 1) + : cacheReadPerCall + const thinkingTokens = isLast + ? totalThinking - thinkingPerCall * (numCalls - 1) + : thinkingPerCall + + const dedupKey = `droid:${sessionId}:${call.id}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const timestamp = call.timestamp || '' + + calls.push({ + provider: 'droid', + model: sessionModelDisplay, + inputTokens, + outputTokens, + cacheCreationInputTokens: cacheCreationTokens, + cacheReadInputTokens: cacheReadTokens, + cachedInputTokens: cacheReadTokens, + reasoningTokens: thinkingTokens, + webSearchRequests: 0, + tools: call.tools, + rawBashCommands: call.bashCommands, + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: i === 0 ? currentUserMessage : '', + sessionId, + }) + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/droid/index.ts b/packages/core/src/providers/droid/index.ts new file mode 100644 index 00000000..957fbf8e --- /dev/null +++ b/packages/core/src/providers/droid/index.ts @@ -0,0 +1,3 @@ +export { decodeDroid, droidToolNameMap, stripModelPrefix, type DroidDecodeInput, type DroidDecodeResult } from './decode.js' +export { toObservations, type RichDroidSessionDecode, type DroidToObservationsContext } from './observations.js' +export { type DroidDecodedCall, type DroidJsonlEntry, type DroidSettings } from './types.js' diff --git a/packages/core/src/providers/droid/observations.ts b/packages/core/src/providers/droid/observations.ts new file mode 100644 index 00000000..db24be90 --- /dev/null +++ b/packages/core/src/providers/droid/observations.ts @@ -0,0 +1,69 @@ +// Minimizing transform: rich Droid decode -> the strict observation envelope. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { DroidDecodedCall } from './types.js' + +export interface RichDroidSessionDecode { + sessionId: string + projectPath: string + calls: DroidDecodedCall[] +} + +export interface DroidToObservationsContext { + privacyKey: string + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: DroidDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichDroidSessionDecode, ctx: DroidToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'droid' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +export function toObservations( + decode: RichDroidSessionDecode | RichDroidSessionDecode[], + ctx: DroidToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/droid/types.ts b/packages/core/src/providers/droid/types.ts new file mode 100644 index 00000000..d3887ef2 --- /dev/null +++ b/packages/core/src/providers/droid/types.ts @@ -0,0 +1,54 @@ +// Raw record + rich-decode types for the Droid provider. + +export type DroidSettings = { + model?: string + tokenUsage?: { + inputTokens: number + outputTokens: number + cacheCreationTokens: number + cacheReadTokens: number + thinkingTokens: number + } +} + +export type DroidContent = { + type: string + text?: string + name?: string + input?: Record +} + +export type DroidMessage = { + role: string + content?: DroidContent[] +} + +export type DroidJsonlEntry = { + type: string + id?: string + timestamp?: string + message?: DroidMessage + title?: string + cwd?: string +} + +// The rich decode of one Droid assistant message. +// Mirrors the host's ParsedProviderCall minus cost fields. +export type DroidDecodedCall = { + provider: 'droid' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' | 'fast' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/src/providers/lingtai-tui/decode.ts b/packages/core/src/providers/lingtai-tui/decode.ts new file mode 100644 index 00000000..fee074dc --- /dev/null +++ b/packages/core/src/providers/lingtai-tui/decode.ts @@ -0,0 +1,190 @@ +// @codeburn/core LingTai TUI decoder: pure decode over supplied ledger JSONL records. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { LingTaiTuiDecodedCall, LingTaiLedgerEntry, JsonObject } from './types.js' + +export type LingTaiTuiDecodeInput = { + records: unknown[] + context: DecodeContext + agentId: string + fallbackModel: string + fallbackEndpoint: string + projectPath: string + // Discovered/manifest-derived source project (host-derived). Optional so a + // core unit test can decode without it. + project?: string + seenKeys?: Set +} + +export type LingTaiTuiDecodeResult = { + calls: LingTaiTuiDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +function asObject(value: unknown): JsonObject | null { + return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : null +} + +function stringField(obj: JsonObject | null, key: string): string | undefined { + const value = obj?.[key] + return typeof value === 'string' && value.trim() ? value : undefined +} + +function numericField(obj: JsonObject, key: keyof LingTaiLedgerEntry): number { + const raw = obj[key] + const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN + if (!Number.isFinite(n) || n <= 0) return 0 + return Math.trunc(n) +} + +function parseTimestamp(raw: unknown): string { + if (typeof raw === 'number' && Number.isFinite(raw)) { + const ms = raw < 1e12 ? raw * 1000 : raw + return new Date(ms).toISOString() + } + if (typeof raw !== 'string' || !raw.trim()) return '' + const d = new Date(raw) + return Number.isNaN(d.getTime()) ? '' : d.toISOString() +} + +function activityForSource(sourceLabel: string): { userMessage: string; tools: string[]; subagentTypes: string[] } { + const normalized = sourceLabel.trim().toLowerCase() + + if (normalized === 'tc_wake' || normalized.startsWith('tc_') || normalized.includes('wake')) { + return { + userMessage: 'LingTai task coordinator wake', + tools: ['Agent'], + subagentTypes: ['lingtai-task-coordinator'], + } + } + + if (normalized === 'daemon') { + return { + userMessage: 'LingTai daemon task', + tools: ['Agent'], + subagentTypes: ['lingtai-daemon'], + } + } + + if (normalized === 'summarize_apriori' || normalized.includes('summar')) { + return { + userMessage: 'LingTai planning summary', + tools: ['EnterPlanMode'], + subagentTypes: [], + } + } + + return { + userMessage: normalized === 'main' + ? 'LingTai main conversation' + : `LingTai ${sourceLabel || 'main'} conversation`, + tools: [], + subagentTypes: [], + } +} + +function parseLedgerLine(line: string | Buffer): LingTaiLedgerEntry | null { + const text = Buffer.isBuffer(line) ? line.toString('utf-8') : line + if (!text.trim()) return null + try { + const parsed = JSON.parse(text) as unknown + const obj = asObject(parsed) + return obj ? (obj as LingTaiLedgerEntry) : null + } catch { + return null + } +} + +/** + * Decode LingTai TUI ledger JSONL records into rich, cost-free calls. + */ +export function decodeLingTaiTui({ + records, + context, + agentId, + fallbackModel, + fallbackEndpoint, + projectPath, + project, + seenKeys: liveSeen, +}: LingTaiTuiDecodeInput): LingTaiTuiDecodeResult { + const seen = liveSeen ?? new Set() + const calls: LingTaiTuiDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + let lineNo = 0 + for (const rawRecord of records) { + lineNo += 1 + const entry = typeof rawRecord === 'string' + ? parseLedgerLine(rawRecord) + : (rawRecord as LingTaiLedgerEntry | null) + + if (!entry) continue + + const obj = entry as JsonObject + const inputTotal = numericField(obj, 'input') + const outputTokens = numericField(obj, 'output') + const reasoningTokens = numericField(obj, 'thinking') + const cachedInputTokens = numericField(obj, 'cached') + const totalTokens = inputTotal + outputTokens + reasoningTokens + cachedInputTokens + if (totalTokens === 0) continue + + // LingTai records provider-normalized input totals plus a separate cached count. + // Match CodeBurn's normal shape by billing cached tokens in cacheReadInputTokens. + const inputTokens = Math.max(0, inputTotal - cachedInputTokens) + const model = stringField(obj, 'model') ?? fallbackModel + const endpoint = stringField(obj, 'endpoint') ?? fallbackEndpoint + const timestamp = parseTimestamp(entry.ts) + const sourceLabel = stringField(obj, 'source') ?? 'main' + const emId = stringField(obj, 'em_id') ?? '' + const runId = stringField(obj, 'run_id') ?? '' + const sessionId = runId || `${agentId}:${sourceLabel}` + const activity = activityForSource(sourceLabel) + // The dedup key threads the source ref (host ledger path) exactly as the + // pre-migration decode did — NOT the agent-dir projectPath. + const dedupKey = [ + 'lingtai-tui', + context.sourceRef, + lineNo, + timestamp, + model, + endpoint, + sourceLabel, + emId, + runId, + inputTotal, + outputTokens, + reasoningTokens, + cachedInputTokens, + ].join(':') + + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + calls.push({ + provider: 'lingtai-tui', + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: cachedInputTokens, + cachedInputTokens, + reasoningTokens, + webSearchRequests: 0, + tools: activity.tools, + rawBashCommands: [], + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + turnId: `${sessionId}:line:${lineNo}`, + userMessage: activity.userMessage, + sessionId, + subagentTypes: activity.subagentTypes, + projectPath, + ...(project !== undefined ? { project } : {}), + }) + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/lingtai-tui/index.ts b/packages/core/src/providers/lingtai-tui/index.ts new file mode 100644 index 00000000..8e774535 --- /dev/null +++ b/packages/core/src/providers/lingtai-tui/index.ts @@ -0,0 +1,3 @@ +export { decodeLingTaiTui, type LingTaiTuiDecodeInput, type LingTaiTuiDecodeResult } from './decode.js' +export { toObservations, type RichLingTaiTuiSessionDecode, type LingTaiTuiToObservationsContext } from './observations.js' +export { type LingTaiTuiDecodedCall, type LingTaiLedgerEntry, type LingTaiAgentManifest } from './types.js' diff --git a/packages/core/src/providers/lingtai-tui/observations.ts b/packages/core/src/providers/lingtai-tui/observations.ts new file mode 100644 index 00000000..89364810 --- /dev/null +++ b/packages/core/src/providers/lingtai-tui/observations.ts @@ -0,0 +1,67 @@ +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { LingTaiTuiDecodedCall } from './types.js' + +export interface RichLingTaiTuiSessionDecode { + sessionId: string + projectPath: string + calls: LingTaiTuiDecodedCall[] +} + +export interface LingTaiTuiToObservationsContext { + privacyKey: string + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: LingTaiTuiDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichLingTaiTuiSessionDecode, ctx: LingTaiTuiToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'lingtai-tui' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +export function toObservations( + decode: RichLingTaiTuiSessionDecode | RichLingTaiTuiSessionDecode[], + ctx: LingTaiTuiToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/lingtai-tui/types.ts b/packages/core/src/providers/lingtai-tui/types.ts new file mode 100644 index 00000000..304b7ad2 --- /dev/null +++ b/packages/core/src/providers/lingtai-tui/types.ts @@ -0,0 +1,53 @@ +// Raw record + rich-decode types for the LingTai TUI provider. + +export type JsonObject = Record + +export type LingTaiAgentManifest = { + agent_id?: string + agent_name?: string + address?: string + nickname?: string | null + llm?: { + model?: string + base_url?: string + } +} + +export type LingTaiLedgerEntry = { + source?: string + em_id?: string + run_id?: string + ts?: string | number + input?: number | string + output?: number | string + thinking?: number | string + cached?: number | string + model?: string + endpoint?: string +} + +// The rich decode of one LingTai TUI ledger entry. +export type LingTaiTuiDecodedCall = { + provider: 'lingtai-tui' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' | 'fast' + deduplicationKey: string + userMessage: string + sessionId: string + subagentTypes: string[] + turnId: string + projectPath: string + // Discovered/manifest-derived source project, carried through so the bridge + // reproduces the pre-migration ParsedProviderCall byte-for-byte. + project?: string +} diff --git a/packages/core/src/providers/mux/decode.ts b/packages/core/src/providers/mux/decode.ts new file mode 100644 index 00000000..90cf528f --- /dev/null +++ b/packages/core/src/providers/mux/decode.ts @@ -0,0 +1,159 @@ +// @codeburn/core Mux decoder: pure decode over supplied chat JSONL records. +// No fs / env / clock — the host reads the file and hands lines straight through. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { MuxDecodedCall, MuxMessage, MuxPart } from './types.js' + +// Mux tool ids mapped to the canonical vocabulary. +export const muxToolNameMap: Record = { + bash: 'Bash', + file_read: 'Read', + file_edit_replace_string: 'Edit', + file_edit_replace_lines: 'Edit', + file_edit_insert: 'Edit', + file_edit_operation: 'Edit', + web_fetch: 'WebFetch', + web_search: 'WebSearch', + task: 'Agent', + todo: 'TodoWrite', +} + +function asRecord(v: unknown): Record | undefined { + return v !== null && typeof v === 'object' ? (v as Record) : undefined +} + +// Matches the host's canonical safeNumber (@codeburn/core claude decode): a +// number is kept only when finite and strictly positive, else 0. +function safeNumber(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0 +} + +// Guard against non-finite / out-of-range ms, which make toISOString() throw. +function toIsoTimestamp(ts: unknown, createdAt: unknown): string { + if (typeof ts === 'number' && Number.isFinite(ts)) { + const d = new Date(ts) + if (!Number.isNaN(d.getTime())) return d.toISOString() + } + return typeof createdAt === 'string' ? createdAt : '' +} + +function stripProvider(model: string): string { + const i = model.indexOf(':') + return i >= 0 ? model.slice(i + 1) : model +} + +export type MuxDecodeInput = { + records: unknown[] + context: DecodeContext + workspaceId: string + seenKeys?: Set +} + +export type MuxDecodeResult = { + calls: MuxDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode Mux JSONL records into rich, cost-free calls. + */ +export function decodeMux({ records, workspaceId, seenKeys: liveSeen }: MuxDecodeInput): MuxDecodeResult { + const seen = liveSeen ?? new Set() + const calls: MuxDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + let pendingUserMessage = '' + let lineIdx = 0 + + for (const rawRecord of records) { + lineIdx++ + let msg: MuxMessage + if (typeof rawRecord === 'string') { + try { + msg = JSON.parse(rawRecord) as MuxMessage + } catch { + continue + } + } else { + msg = rawRecord as MuxMessage + } + + if (!msg || typeof msg !== 'object') continue + + if (msg.role === 'user') { + const texts = (Array.isArray(msg.parts) ? msg.parts : []) + .filter(p => p?.type === 'text' && typeof p.text === 'string') + .map(p => p.text as string) + .filter(Boolean) + if (texts.length > 0) pendingUserMessage = texts.join(' ').slice(0, 500) + continue + } + + if (msg.role !== 'assistant') continue + const meta = msg.metadata + const usage = asRecord(meta?.usage) + if (!meta || !usage) continue + + const pm = meta.providerMetadata ?? {} + const anthropic = asRecord(pm['anthropic']) + + // mux reports inputTokens inclusive of cache read+creation and + // outputTokens inclusive of reasoning; decompose to codeburn's convention. + const cacheRead = safeNumber(usage['cachedInputTokens']) + const cacheCreate = safeNumber(anthropic?.['cacheCreationInputTokens']) + const reasoning = safeNumber(usage['reasoningTokens']) + const inputTokens = Math.max(0, safeNumber(usage['inputTokens']) - cacheRead - cacheCreate) + const outputTokens = Math.max(0, safeNumber(usage['outputTokens']) - reasoning) + + if (inputTokens === 0 && outputTokens === 0 && cacheRead === 0 && cacheCreate === 0 && reasoning === 0) { + continue + } + + const rawModel = typeof meta.model === 'string' && meta.model ? meta.model : 'unknown' + const model = stripProvider(rawModel) + const id = typeof msg.id === 'string' && msg.id ? msg.id : `L${lineIdx}` + const dedupKey = `mux:${workspaceId}:${id}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const toolParts = (Array.isArray(msg.parts) ? msg.parts : []).filter( + p => p?.type === 'dynamic-tool' && typeof p.toolName === 'string', + ) + const tools = toolParts.map(p => muxToolNameMap[p.toolName!] ?? p.toolName!) + // Emit the RAW shell scripts; the host runs extractBashCommands (with its + // strip-ansi dependency) to reduce them to base-name programs. + const rawBashCommands = toolParts + .filter(p => p.toolName === 'bash') + .flatMap(p => { + const input = asRecord(p.input) + const script = input?.['script'] ?? input?.['command'] + return typeof script === 'string' ? [script] : [] + }) + + const timestamp = toIsoTimestamp(meta.timestamp, msg.createdAt) + + calls.push({ + provider: 'mux', + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: cacheCreate, + cacheReadInputTokens: cacheRead, + cachedInputTokens: cacheRead, + reasoningTokens: reasoning, + webSearchRequests: 0, + tools, + rawBashCommands, + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: pendingUserMessage, + sessionId: workspaceId, + }) + + pendingUserMessage = '' + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/mux/index.ts b/packages/core/src/providers/mux/index.ts new file mode 100644 index 00000000..5f2b3f70 --- /dev/null +++ b/packages/core/src/providers/mux/index.ts @@ -0,0 +1,3 @@ +export { decodeMux, muxToolNameMap, type MuxDecodeInput, type MuxDecodeResult } from './decode.js' +export { toObservations, type RichMuxSessionDecode, type MuxToObservationsContext } from './observations.js' +export { type MuxDecodedCall, type MuxMessage } from './types.js' diff --git a/packages/core/src/providers/mux/observations.ts b/packages/core/src/providers/mux/observations.ts new file mode 100644 index 00000000..584c7330 --- /dev/null +++ b/packages/core/src/providers/mux/observations.ts @@ -0,0 +1,67 @@ +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { MuxDecodedCall } from './types.js' + +export interface RichMuxSessionDecode { + sessionId: string + projectPath: string + calls: MuxDecodedCall[] +} + +export interface MuxToObservationsContext { + privacyKey: string + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: MuxDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichMuxSessionDecode, ctx: MuxToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'mux' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +export function toObservations( + decode: RichMuxSessionDecode | RichMuxSessionDecode[], + ctx: MuxToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/mux/types.ts b/packages/core/src/providers/mux/types.ts new file mode 100644 index 00000000..595e4dfa --- /dev/null +++ b/packages/core/src/providers/mux/types.ts @@ -0,0 +1,42 @@ +// Raw record + rich-decode types for the Mux provider. + +export type MuxPart = { + type?: string + text?: string + toolName?: string + input?: unknown +} + +export type MuxMessage = { + id?: string + role?: string + parts?: MuxPart[] + createdAt?: string + metadata?: { + model?: string + timestamp?: number + historySequence?: number + usage?: unknown + providerMetadata?: Record + } +} + +// The rich decode of one Mux assistant message. +export type MuxDecodedCall = { + provider: 'mux' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' | 'fast' + deduplicationKey: string + userMessage: string + sessionId: string +} diff --git a/packages/core/src/providers/open-design/decode.ts b/packages/core/src/providers/open-design/decode.ts new file mode 100644 index 00000000..70453e6b --- /dev/null +++ b/packages/core/src/providers/open-design/decode.ts @@ -0,0 +1,138 @@ +// @codeburn/core Open Design decoder: pure decode over supplied event JSONL records. +// No fs / env / clock — the host reads the file and hands lines straight through. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { OpenDesignDecodedCall, OpenDesignEntry, TokenUsage } from './types.js' + +export type OpenDesignDecodeInput = { + records: unknown[] + context: DecodeContext + sessionId: string + // Discovered source project (host-derived). Optional so a core unit test can + // decode with only { records, context, sessionId }. + project?: string + seenKeys?: Set +} + +export type OpenDesignDecodeResult = { + calls: OpenDesignDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +function tokenValue(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0 +} + +function timestampValue(value: unknown): string { + const text = stringValue(value) + if (text) return text + if (typeof value !== 'number' || !Number.isFinite(value)) return '' + + const date = new Date(value) + return Number.isNaN(date.getTime()) ? '' : date.toISOString() +} + +function parseEvent(line: string | Buffer): OpenDesignEntry | null { + const text = (typeof line === 'string' ? line : line.toString('utf-8')).trim() + if (!text) return null + + try { + const parsed = JSON.parse(text) as unknown + return isRecord(parsed) ? parsed : null + } catch { + return null + } +} + +function parseUsage(data: unknown): TokenUsage | null { + if (!isRecord(data) || data['type'] !== 'usage') return null + const usage = data['usage'] + if (!isRecord(usage)) return null + + return { + inputTokens: tokenValue(usage['input_tokens']), + outputTokens: tokenValue(usage['output_tokens']), + cacheReadTokens: tokenValue(usage['cached_read_tokens']), + reasoningTokens: tokenValue(usage['thought_tokens']), + } +} + +/** + * Decode Open Design event JSONL records into rich, cost-free calls. + * Maintains state of currentModel as events indicate model transitions. + */ +export function decodeOpenDesign({ records, sessionId, project, seenKeys: liveSeen }: OpenDesignDecodeInput): OpenDesignDecodeResult { + const seen = liveSeen ?? new Set() + const calls: OpenDesignDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + let currentModel = '' + let fallbackEventCounter = 0 + + for (const rawRecord of records) { + let entry: OpenDesignEntry + if (typeof rawRecord === 'string') { + entry = parseEvent(rawRecord) ?? {} + } else { + entry = rawRecord as OpenDesignEntry + } + + const eventName = stringValue(entry.event) + const data = entry.data + + if (eventName === 'start' && isRecord(data)) { + const model = stringValue(data['model']) + if (model) currentModel = model + continue + } + + if (eventName !== 'agent' || !isRecord(data)) continue + + if (data['type'] === 'status') { + const model = stringValue(data['model']) + if (model) currentModel = model + continue + } + + const usage = parseUsage(data) + if (!usage || !currentModel) continue + + const eventId = stringValue(entry.id) ?? `line-${fallbackEventCounter++}` + const dedupKey = `open-design:${sessionId}:${eventId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const uncachedInputTokens = Math.max(0, usage.inputTokens - usage.cacheReadTokens) + + calls.push({ + provider: 'open-design', + model: currentModel, + inputTokens: uncachedInputTokens, + outputTokens: usage.outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: usage.cacheReadTokens, + cachedInputTokens: usage.cacheReadTokens, + reasoningTokens: usage.reasoningTokens, + webSearchRequests: 0, + tools: [], + rawBashCommands: [], + timestamp: timestampValue(entry.timestamp), + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: '', + sessionId, + ...(project !== undefined ? { project } : {}), + }) + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/open-design/index.ts b/packages/core/src/providers/open-design/index.ts new file mode 100644 index 00000000..e19eb8f2 --- /dev/null +++ b/packages/core/src/providers/open-design/index.ts @@ -0,0 +1,3 @@ +export { decodeOpenDesign, type OpenDesignDecodeInput, type OpenDesignDecodeResult } from './decode.js' +export { toObservations, type RichOpenDesignSessionDecode, type OpenDesignToObservationsContext } from './observations.js' +export { type OpenDesignDecodedCall, type OpenDesignEntry, type TokenUsage } from './types.js' diff --git a/packages/core/src/providers/open-design/observations.ts b/packages/core/src/providers/open-design/observations.ts new file mode 100644 index 00000000..ad7f1480 --- /dev/null +++ b/packages/core/src/providers/open-design/observations.ts @@ -0,0 +1,67 @@ +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { OpenDesignDecodedCall } from './types.js' + +export interface RichOpenDesignSessionDecode { + sessionId: string + projectPath: string + calls: OpenDesignDecodedCall[] +} + +export interface OpenDesignToObservationsContext { + privacyKey: string + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: OpenDesignDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichOpenDesignSessionDecode, ctx: OpenDesignToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'open-design' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +export function toObservations( + decode: RichOpenDesignSessionDecode | RichOpenDesignSessionDecode[], + ctx: OpenDesignToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/open-design/types.ts b/packages/core/src/providers/open-design/types.ts new file mode 100644 index 00000000..d53ee63a --- /dev/null +++ b/packages/core/src/providers/open-design/types.ts @@ -0,0 +1,38 @@ +// Raw record + rich-decode types for the Open Design provider. + +export type TokenUsage = { + inputTokens: number + outputTokens: number + cacheReadTokens: number + reasoningTokens: number +} + +export type OpenDesignEntry = { + id?: unknown + event?: unknown + data?: unknown + timestamp?: unknown +} + +// The rich decode of one Open Design usage event. +export type OpenDesignDecodedCall = { + provider: 'open-design' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' | 'fast' + deduplicationKey: string + userMessage: string + sessionId: string + // Discovered source project, carried through so the bridge reproduces the + // pre-migration ParsedProviderCall byte-for-byte. Omitted when not supplied. + project?: string +} diff --git a/packages/core/src/providers/zerostack/decode.ts b/packages/core/src/providers/zerostack/decode.ts new file mode 100644 index 00000000..b7f05101 --- /dev/null +++ b/packages/core/src/providers/zerostack/decode.ts @@ -0,0 +1,104 @@ +// @codeburn/core Zerostack decoder: pure decode over supplied session JSON records. +// No fs / env / clock — the host reads the file and hands JSON objects straight through. +// The rich output carries token buckets but NO pricing (cost leaves the decoder; +// the host prices via its estimated-cost seam). + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { ZerostackDecodedCall, ZerostackMessage, ZerostackSession } from './types.js' + +// Zerostack tool ids mapped to the canonical vocabulary. An id with +// no mapping passes through unchanged. +export const zerostackToolNameMap: Record = { + bash: 'Bash', + read: 'Read', + write: 'Write', + edit: 'Edit', + grep: 'Grep', + glob: 'Glob', + fetch: 'WebFetch', + search: 'WebSearch', + task: 'Agent', +} + +function firstUserMessage(messages: ZerostackMessage[]): string { + const msg = messages.find(m => m.role === 'user') + if (!msg) return '' + if (typeof msg.content === 'string') return msg.content + return (msg.content ?? []).map(c => c.text ?? '').filter(Boolean).join(' ') +} + +export type ZerostackDecodeInput = { + records: unknown[] + context: DecodeContext + // Host-derived per-source scalars (supplied by the CLI bridge). `project` is + // the discovered source project; `sessionIdFallback` is basename(path,'.json'), + // used when a session record carries no id. Both optional so a core unit test + // can call the decoder with only { records, context }. + project?: string + sessionIdFallback?: string + seenKeys?: Set +} + +export type ZerostackDecodeResult = { + calls: ZerostackDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode a Zerostack session file's record into a rich, cost-free call. + * Zerostack has one record per session with cumulative token totals. The dedup + * key threads the source ref (host path) exactly as the pre-migration decode did: + * `zerostack:::`. + */ +export function decodeZerostack({ + records, + context, + project, + sessionIdFallback, + seenKeys: liveSeen, +}: ZerostackDecodeInput): ZerostackDecodeResult { + const seen = liveSeen ?? new Set() + const calls: ZerostackDecodedCall[] = [] + const diagnostics: RecordDiagnostic[] = [] + + for (const rawRecord of records) { + const session = rawRecord as ZerostackSession | null + if (!session) continue + + const input = session.total_input_tokens ?? 0 + const output = session.total_output_tokens ?? 0 + if (input === 0 && output === 0) continue + + const timestamp = session.updated_at ?? session.created_at ?? '' + const sessionId = session.id ?? sessionIdFallback ?? '' + const dedupKey = `zerostack:${context.sourceRef}:${timestamp}:${sessionId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const model = session.model ?? '' + + calls.push({ + provider: 'zerostack', + model, + inputTokens: input, + outputTokens: output, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools: [], + rawBashCommands: [], + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: firstUserMessage(session.messages ?? []), + sessionId, + ...(project !== undefined ? { project } : {}), + ...(session.working_dir !== undefined ? { projectPath: session.working_dir } : {}), + }) + } + + return { calls, diagnostics } +} diff --git a/packages/core/src/providers/zerostack/index.ts b/packages/core/src/providers/zerostack/index.ts new file mode 100644 index 00000000..636f1e10 --- /dev/null +++ b/packages/core/src/providers/zerostack/index.ts @@ -0,0 +1,3 @@ +export { decodeZerostack, zerostackToolNameMap, type ZerostackDecodeInput, type ZerostackDecodeResult } from './decode.js' +export { toObservations, type RichZerostackSessionDecode, type ZerostackToObservationsContext } from './observations.js' +export { type ZerostackDecodedCall, type ZerostackMessage, type ZerostackSession } from './types.js' diff --git a/packages/core/src/providers/zerostack/observations.ts b/packages/core/src/providers/zerostack/observations.ts new file mode 100644 index 00000000..4955f65c --- /dev/null +++ b/packages/core/src/providers/zerostack/observations.ts @@ -0,0 +1,86 @@ +// Minimizing transform: rich Zerostack decode -> the strict observation envelope. +// This is where the content-smuggling guarantees bind. Only opaque ids, +// fingerprints, enums, numbers, timestamps, and CANONICAL tool names cross into +// the output — never the user message, project path, or shell command. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { ZerostackDecodedCall } from './types.js' + +/** One Zerostack session's rich decode, as the host holds it before minimization. */ +export interface RichZerostackSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (as decodeZerostack emits them). */ + calls: ZerostackDecodedCall[] +} + +export interface ZerostackToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +// Canonical tool-name charset, mirroring core's CanonicalToolName schema. +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: ZerostackDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichZerostackSessionDecode, ctx: ZerostackToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'zerostack' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Zerostack decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + * + * Content-smuggling guarantee: no free text (user message, project path, + * command) is ever copied into the result. Only fingerprints, enums, numbers, + * timestamps, dedup keys, and canonical tool names cross the boundary. + */ +export function toObservations( + decode: RichZerostackSessionDecode | RichZerostackSessionDecode[], + ctx: ZerostackToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/zerostack/types.ts b/packages/core/src/providers/zerostack/types.ts new file mode 100644 index 00000000..2ba9edf2 --- /dev/null +++ b/packages/core/src/providers/zerostack/types.ts @@ -0,0 +1,44 @@ +// Raw record + rich-decode types for the Zerostack provider. + +export type ZerostackMessage = { + role?: string + content?: string | Array<{ text?: string }> +} + +export type ZerostackSession = { + id?: string + messages?: ZerostackMessage[] + created_at?: string + updated_at?: string + total_input_tokens?: number + total_output_tokens?: number + model?: string + provider?: string + working_dir?: string +} + +// The rich decode of one Zerostack session (one cumulative call). +// Mirrors the host's ParsedProviderCall minus cost fields. +export type ZerostackDecodedCall = { + provider: 'zerostack' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' | 'fast' + deduplicationKey: string + userMessage: string + sessionId: string + // Host-derived, carried through so the bridge can reproduce the pre-migration + // ParsedProviderCall byte-for-byte. `project` is the discovered source project; + // `projectPath` is the session's recorded working_dir. Both omitted when absent. + project?: string + projectPath?: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index 71a7aa47..aacfe072 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -128,6 +128,16 @@ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/kimi/types.ts', 'src/providers/qwen/decode.ts', 'src/providers/qwen/types.ts', + 'src/providers/zerostack/decode.ts', + 'src/providers/zerostack/types.ts', + 'src/providers/droid/decode.ts', + 'src/providers/droid/types.ts', + 'src/providers/mux/decode.ts', + 'src/providers/mux/types.ts', + 'src/providers/open-design/decode.ts', + 'src/providers/open-design/types.ts', + 'src/providers/lingtai-tui/decode.ts', + 'src/providers/lingtai-tui/types.ts', ]) describe('architecture gate: no classification or free text in @codeburn/core source', () => { diff --git a/packages/core/tests/providers/zerostack-decode.test.ts b/packages/core/tests/providers/zerostack-decode.test.ts new file mode 100644 index 00000000..a5e8691f --- /dev/null +++ b/packages/core/tests/providers/zerostack-decode.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' + +import { decodeZerostack, toObservations } from '../../src/providers/zerostack/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'zerostack', sourceRef: 'ref' } + +describe('zerostack rich decode (moved to @codeburn/core)', () => { + it('decodes one session per JSON file into a cost-free rich call', () => { + const records = [ + { + id: 'sess-1', + messages: [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'world' }, + ], + total_input_tokens: 100, + total_output_tokens: 50, + model: 'deepseek/deepseek-v4-pro', + created_at: '2026-06-01T10:00:00Z', + updated_at: '2026-06-01T10:01:00Z', + }, + ] + + const { calls } = decodeZerostack({ records, context }) + expect(calls).toHaveLength(1) + const call = calls[0]! + expect(call.model).toBe('deepseek/deepseek-v4-pro') + expect(call.inputTokens).toBe(100) + expect(call.outputTokens).toBe(50) + expect(call.userMessage).toBe('hello') + expect(call.deduplicationKey).toContain('zerostack:') + }) + + it('skips sessions with zero tokens', () => { + const records = [ + { + id: 'empty', + total_input_tokens: 0, + total_output_tokens: 0, + model: 'unknown', + }, + ] + + const { calls } = decodeZerostack({ records, context }) + expect(calls).toHaveLength(0) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const records = [ + { + id: 'sess-a', + messages: [{ role: 'user', content: 'test' }], + total_input_tokens: 10, + total_output_tokens: 5, + model: 'test-model', + created_at: '2026-06-01T10:00:00Z', + working_dir: '/Users/t/project', + }, + ] + const { calls } = decodeZerostack({ records, context }) + const { sessions } = toObservations( + { sessionId: 'sess-a', projectPath: '/Users/t/project', calls }, + { privacyKey: 'test-key', provider: 'zerostack' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 7cf6f241..a1886dd4 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -17,6 +17,11 @@ export default defineConfig({ 'src/providers/grok/index.ts', 'src/providers/kimi/index.ts', 'src/providers/qwen/index.ts', + 'src/providers/zerostack/index.ts', + 'src/providers/droid/index.ts', + 'src/providers/mux/index.ts', + 'src/providers/open-design/index.ts', + 'src/providers/lingtai-tui/index.ts', ], format: ['esm'], target: 'node20',