diff --git a/packages/cli/src/providers/copilot.ts b/packages/cli/src/providers/copilot.ts index 6c8eb1fe..84a0a850 100644 --- a/packages/cli/src/providers/copilot.ts +++ b/packages/cli/src/providers/copilot.ts @@ -61,15 +61,18 @@ import { readdir, stat } from 'fs/promises' import { homedir, platform } from 'os' import { join, basename, dirname, posix, win32 } from 'path' import { existsSync } from 'fs' -import { createHash } from 'crypto' import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import { estimateTokens } from '../context-tree.js' +import { + collectJetBrainsRepoDirCandidates, decodeCopilot, normalizeCopilotTool, +} from '@codeburn/core/providers/copilot' +import type { CopilotDecodedCall, CopilotOtelConversationRecord, CopilotOtelSpanRecord, + SpanAttributes } from '@codeburn/core/providers/copilot' +import { createBridgedProvider } from './bridge.js' import type { Provider, SessionSource, - SessionParser, ParsedProviderCall, } from './types.js' @@ -90,165 +93,10 @@ const modelDisplayNames: Record = { 'copilot-anthropic-auto': 'Copilot (Anthropic auto)', } -// --------------------------------------------------------------------------- -// Tool name normalisation (unchanged from original, plus OTel tool names) -// --------------------------------------------------------------------------- -const toolNameMap: Record = { - // JSONL session-state tool names - bash: 'Bash', - skill: 'Skill', - read_file: 'Read', - write_file: 'Edit', - edit_file: 'Edit', - delete_file: 'Delete', - github_repo: 'GitHub', - web_search: 'WebSearch', - run_in_terminal: 'Shell', - // JetBrains Copilot agent tool names (snake_case) - insert_edit_into_file: 'Edit', - create_file: 'Edit', - get_errors: 'Diagnostics', - file_search: 'Search', - grep_search: 'Search', - semantic_search: 'Search', - list_dir: 'Search', - fetch_webpage: 'Web', - // OTel execute_tool span names from Copilot Chat: - readFile: 'Read', - writeFile: 'Edit', - editFile: 'Edit', - runCommand: 'Shell', - runInTerminal: 'Shell', - findFiles: 'Search', - grepSearch: 'Search', - codebaseSearch: 'Search', - getErrors: 'Diagnostics', - listCodeUsages: 'Search', - createFile: 'Edit', - deleteFile: 'Delete', - renameOrMoveFile: 'Edit', - fetchWebpage: 'Web', -} - -/** - * Normalise a raw tool name to its display form. - * - Known tools are mapped via toolNameMap. - * - MCP tools (containing both '-' and '_') are formatted as - * mcp__server_name__tool_name. - * - Everything else is returned unchanged. - */ -function normalizeTool(rawTool: string): string { - const mapped = toolNameMap[rawTool] - if (mapped) return mapped - // MCP tool names follow the pattern: server-name-tool_operand - // e.g. github-mcp-server-list_issues → mcp__github_mcp_server__list_issues - const dashIdx = rawTool.lastIndexOf('-') - if (dashIdx > 0 && rawTool.includes('_')) { - const server = rawTool.slice(0, dashIdx).replace(/-/g, '_') - const tool = rawTool.slice(dashIdx + 1) - return `mcp__${server}__${tool}` - } - return rawTool -} - const modelDisplayEntries = Object.entries(modelDisplayNames).sort( (a, b) => b[0].length - a[0].length ) -// Tool names that represent shell/bash execution. When the AI calls one of -// these, we extract the `arguments.command` string into bashCommands[]. -const BASH_TOOL_NAMES = new Set(['bash', 'run_in_terminal', 'runInTerminal', 'runCommand']) - -// --------------------------------------------------------------------------- -// Types for JSONL session state events (unchanged from original) -// --------------------------------------------------------------------------- -type ToolRequest = { - toolName?: string // older format - name?: string // newer format (copilot-agent) - arguments?: Record -} - -type SessionStartData = { - selectedModel?: string -} - -type ModelChangeData = { - newModel: string - previousModel?: string -} - -type UserMessageData = { - content: string - interactionId?: string -} - -type AssistantMessageData = { - messageId: string - model?: string // present in newer copilot-agent format - outputTokens: number - interactionId?: string - toolRequests?: ToolRequest[] -} - -type SubagentSelectedData = { - agentName: string - agentDisplayName?: string - tools?: string[] -} - -// Per-model usage rollup the CLI writes into session.shutdown. inputTokens is -// cache-INCLUSIVE (input + cache_read + cache_write); see the shutdown handler. -type ShutdownModelUsage = { - inputTokens?: number - outputTokens?: number - cacheReadTokens?: number - cacheWriteTokens?: number - reasoningTokens?: number -} - -type SessionShutdownData = { - modelMetrics?: Record - sessionStartTime?: number -} - -type CopilotEvent = - | { type: 'session.start'; data: SessionStartData; timestamp?: string } - | { type: 'session.model_change'; data: ModelChangeData; timestamp?: string } - | { type: 'user.message'; data: UserMessageData; timestamp?: string } - | { type: 'assistant.message'; data: AssistantMessageData; timestamp?: string } - | { type: 'subagent.selected'; data: SubagentSelectedData; timestamp?: string } - | { type: 'session.shutdown'; data: SessionShutdownData; timestamp?: string } - -type ChatJournalPathSegment = string | number -type ChatSessionRequest = Record - -// --------------------------------------------------------------------------- -// Types for OTel span rows from agent-traces.db -// --------------------------------------------------------------------------- - -// The OTel SQLite store schema uses a spans table where attributes are stored -// either as a JSON blob or as individual columns. We handle both patterns. -// The Copilot Budget extension reads from this same DB and uses per-span -// token counts, confirming this schema is stable enough to depend on. - -// Parsed attribute bag from a span -interface SpanAttributes { - 'gen_ai.operation.name'?: string - 'gen_ai.response.model'?: string - 'gen_ai.request.model'?: string - 'gen_ai.usage.input_tokens'?: number - 'gen_ai.usage.output_tokens'?: number - 'gen_ai.usage.cache_read.input_tokens'?: number - 'gen_ai.usage.cache_creation.input_tokens'?: number - 'gen_ai.conversation.id'?: string - 'gen_ai.agent.name'?: string - 'gen_ai.tool.name'?: string - 'gen_ai.tool.call.arguments'?: string - 'copilot_chat.parent_chat_session_id'?: string - 'github.copilot.chat.turn.id'?: string - [key: string]: unknown -} - // --------------------------------------------------------------------------- // Paths // --------------------------------------------------------------------------- @@ -389,236 +237,6 @@ function loadSpanAttributesFromTable( } } -/** - * Convert nanosecond or millisecond epoch to ISO timestamp. - * The OTel spec uses nanoseconds, but some implementations use milliseconds. - */ -function epochToISO(epoch: number): string { - // Guard malformed rows: new Date(NaN).toISOString() throws. Fall back to the - // epoch (1970) so a bad timestamp is excluded from period totals, not crashing. - if (!Number.isFinite(epoch) || epoch <= 0) return new Date(0).toISOString() - // If the value looks like nanoseconds (> 1e15), convert to ms - const ms = epoch > 1e15 ? Math.floor(epoch / 1e6) : epoch > 1e12 ? epoch : epoch * 1000 - return new Date(ms).toISOString() -} - -function timestampToISO(raw: unknown): string { - if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { - return epochToISO(raw) - } - if (typeof raw !== 'string') return '' - const trimmed = raw.trim() - if (!trimmed) return '' - if (/^\d+(\.\d+)?$/.test(trimmed)) { - return epochToISO(Number(trimmed)) - } - const parsed = Date.parse(trimmed) - return Number.isNaN(parsed) ? '' : new Date(parsed).toISOString() -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function isReplayContainer(value: unknown): value is object { - return typeof value === 'object' && value !== null -} - -function createReplayObject(): Record { - return Object.create(null) as Record -} - -const FORBIDDEN_CHAT_JOURNAL_KEYS = new Set(['__proto__', 'prototype', 'constructor']) - -function parseChatJournalPath(rawPath: unknown, fallback?: ChatJournalPathSegment[]): ChatJournalPathSegment[] | null { - const value = rawPath === undefined ? fallback : rawPath - if (!Array.isArray(value)) return null - - const path: ChatJournalPathSegment[] = [] - for (const segment of value) { - if (typeof segment === 'number') { - if (!Number.isInteger(segment) || segment < 0) return null - path.push(segment) - continue - } - if (typeof segment === 'string') { - if (FORBIDDEN_CHAT_JOURNAL_KEYS.has(segment)) return null - path.push(segment) - continue - } - return null - } - return path -} - -function getReplayValue(container: object, segment: ChatJournalPathSegment): unknown { - return (container as Record)[String(segment)] -} - -function setReplayValue(container: object, segment: ChatJournalPathSegment, value: unknown): void { - ;(container as Record)[String(segment)] = value -} - -function createContainerForNext(segment: ChatJournalPathSegment): unknown[] | Record { - return typeof segment === 'number' ? [] : createReplayObject() -} - -function ensureReplayParent(root: object, path: ChatJournalPathSegment[]): object | null { - let current: object = root - for (let i = 0; i < path.length - 1; i++) { - const segment = path[i]! - const nextSegment = path[i + 1]! - let child = getReplayValue(current, segment) - if (!isReplayContainer(child)) { - const created = createContainerForNext(nextSegment) - setReplayValue(current, segment, created) - current = created - continue - } - current = child - } - return current -} - -function applyChatJournalSet(root: unknown, path: ChatJournalPathSegment[], value: unknown): unknown { - if (path.length === 0) return value - - const workingRoot = isReplayContainer(root) ? root : createReplayObject() - const parent = ensureReplayParent(workingRoot, path) - if (!parent) return workingRoot - setReplayValue(parent, path[path.length - 1]!, value) - return workingRoot -} - -function applyChatJournalAppend(root: unknown, path: ChatJournalPathSegment[], items: unknown[]): unknown { - const workingRoot = isReplayContainer(root) ? root : createReplayObject() - - if (path.length === 0) { - if (Array.isArray(workingRoot)) { - for (const item of items) workingRoot.push(item) - } - return workingRoot - } - - const parent = ensureReplayParent(workingRoot, path) - if (!parent) return workingRoot - - const last = path[path.length - 1]! - let target = getReplayValue(parent, last) - const targetArray: unknown[] = Array.isArray(target) ? target : [] - if (target !== targetArray) { - setReplayValue(parent, last, targetArray) - } - for (const item of items) targetArray.push(item) - return workingRoot -} - -function replayChatSessionJournal(content: string): unknown { - let root: unknown = createReplayObject() - const lines = content.split('\n').filter((l) => l.trim()) - - for (const line of lines) { - let entry: unknown - try { - entry = JSON.parse(line) as unknown - } catch { - continue - } - if (!isRecord(entry)) continue - - const kind = entry['kind'] - if (kind === 0) { - root = entry['v'] - continue - } - - if (kind === 1) { - const path = parseChatJournalPath(entry['k']) - if (!path) continue - root = applyChatJournalSet(root, path, entry['v']) - continue - } - - if (kind === 2) { - const hasPath = Object.prototype.hasOwnProperty.call(entry, 'k') - const path = parseChatJournalPath(hasPath ? entry['k'] : undefined, ['requests']) - const items = Array.isArray(entry['v']) ? entry['v'] : [] - if (!path) continue - root = applyChatJournalAppend(root, path, items) - } - } - - return root -} - -function numberOrZero(raw: unknown): number { - return typeof raw === 'number' && Number.isFinite(raw) && raw > 0 ? raw : 0 -} - -function readString(raw: unknown): string { - return typeof raw === 'string' ? raw : '' -} - -function modelFromChatSessionRequest(req: ChatSessionRequest, metadata: Record): string { - const resolved = readString(metadata['resolvedModel']) - if (resolved) return resolved - - const modelId = readString(req['modelId']).replace(/^copilot\//, '') - return modelId || 'unknown' -} - -function extractChatSessionTools(metadata: Record): string[] { - const rounds = metadata['toolCallRounds'] - if (!Array.isArray(rounds)) return [] - - const names = new Set() - const addName = (raw: unknown): void => { - if (typeof raw === 'string' && raw.trim()) names.add(normalizeTool(raw)) - } - const addFromRecord = (record: Record): void => { - addName(record['toolName']) - addName(record['name']) - addName(record['tool']) - } - - for (const round of rounds) { - if (!isRecord(round)) continue - addFromRecord(round) - - for (const key of ['tools', 'toolCalls', 'toolRequests']) { - const entries = round[key] - if (!Array.isArray(entries)) continue - for (const entry of entries) { - if (typeof entry === 'string') { - addName(entry) - } else if (isRecord(entry)) { - addFromRecord(entry) - } - } - } - } - - return [...names] -} - -/** - * Extract a shell command string from an OTel execute_tool span's - * `gen_ai.tool.call.arguments` attribute. The attribute is a JSON-encoded - * argument object (e.g. `{"command":"ls -la"}`); we pull out the `command` - * field. Returns null when the attribute is absent or doesn't carry a command, - * so callers can skip shell-command extraction cleanly. - */ -function parseToolCommand(raw: unknown): string | null { - if (typeof raw !== 'string' || !raw.trim()) return null - try { - const parsed = JSON.parse(raw) as Record - const command = parsed['command'] - return typeof command === 'string' ? command : null - } catch { - return null - } -} - // Shell control-flow keywords. These lead a statement but are not commands, so // they must never be reported as bash commands. const OTEL_SHELL_KEYWORDS = new Set([ @@ -646,464 +264,6 @@ function extractOtelBashCommands(command: string): string[] { return extractBashCommands(normalized).filter(c => !OTEL_SHELL_KEYWORDS.has(c)) } -// --------------------------------------------------------------------------- -// Helpers for JSONL / transcript parsing -// --------------------------------------------------------------------------- - -/** - * Safely coerce a raw toolRequests value to an array of ToolRequest. - * Non-array values (string, null, undefined) are treated as empty arrays - * so that a corrupt event.data doesn't abort the whole file parse loop. - */ -function coerceToolRequests(raw: unknown): ToolRequest[] { - return Array.isArray(raw) ? (raw as ToolRequest[]) : [] -} - -/** - * Infer the model bucket for a VS Code transcript file by counting the - * toolCallId prefixes across all assistant messages: - * call_* → OpenAI - * tooluse_* / toolu_* → Anthropic - * The dominant prefix determines the model for the whole session. - * Returns '' if no toolCallIds are present. - */ -function inferTranscriptModel(lines: string[]): string { - let openaiCount = 0 - let anthropicCount = 0 - - for (const line of lines) { - try { - const event = JSON.parse(line) as CopilotEvent - if (event.type !== 'assistant.message') continue - const data = event.data as AssistantMessageData & { toolRequests?: Array<{ toolCallId?: string }> } - const reqs = coerceToolRequests(data.toolRequests) - for (const req of reqs) { - const id = (req as { toolCallId?: unknown }).toolCallId - if (typeof id !== 'string') continue - if (id.startsWith('call_')) openaiCount++ - else if (/^tooluse_|^toolu_/.test(id)) anthropicCount++ - } - } catch { - continue - } - } - - if (openaiCount === 0 && anthropicCount === 0) return '' - return openaiCount >= anthropicCount ? 'copilot-openai-auto' : 'copilot-anthropic-auto' -} - -// --------------------------------------------------------------------------- -// JSONL parser (handles both regular session-state events and VS Code -// transcript format via session.start { producer: 'copilot-agent' }) -// --------------------------------------------------------------------------- - -function createJsonlParser( - source: SessionSource, - seenKeys: Set -): SessionParser { - return { - async *parse(): AsyncGenerator { - const content = await readSessionFile(source.path) - if (!content) return - const sessionId = basename(dirname(source.path)) - const lines = content.split('\n').filter((l) => l.trim()) - - // Detect VS Code transcript format: the first session.start event has - // { producer: 'copilot-agent' } and no outputTokens in messages. - let isTranscript = false - let currentModel = '' - let pendingUserMessage = '' - // Track the active subagent for this session (from subagent.selected events). - // Resets when a new subagent is selected. - let currentSubagentType: string | undefined - - // First pass: detect format and infer transcript model if needed. - for (const line of lines) { - try { - const ev = JSON.parse(line) as CopilotEvent - if (ev.type === 'session.start') { - const data = ev.data as SessionStartData & { producer?: string } - if (data.producer === 'copilot-agent') { - isTranscript = true - } - break - } - if (ev.type === 'session.model_change') break // regular format - } catch { - continue - } - } - - if (isTranscript) { - currentModel = inferTranscriptModel(lines) - if (!currentModel) return // no toolCallIds to infer model from - } - - // Shutdown rollups may lack their own timestamp; remember the last - // stamped event so the supplementary call is never left with an empty - // timestamp, which the date-range filters silently drop. - let lastEventTimestamp = '' - - for (const line of lines) { - let event: CopilotEvent - try { - event = JSON.parse(line) as CopilotEvent - } catch { - continue - } - if (typeof event.timestamp === 'string' && event.timestamp) lastEventTimestamp = event.timestamp - - if (event.type === 'session.start') { - if (!isTranscript) { - currentModel = (event.data as SessionStartData).selectedModel ?? currentModel - } - continue - } - - if (event.type === 'session.model_change') { - currentModel = (event.data as ModelChangeData).newModel ?? currentModel - continue - } - - if (event.type === 'subagent.selected') { - currentSubagentType = (event.data as SubagentSelectedData).agentName - continue - } - - if (event.type === 'user.message') { - pendingUserMessage = (event.data as UserMessageData).content ?? '' - continue - } - - if (event.type === 'session.shutdown') { - // The Copilot CLI writes a per-model token/cost rollup here at - // shutdown: the only place a CLI session records input, cache-read - // and cache-write tokens (assistant.message events carry output - // only). VS Code transcripts never carry this rollup, so this path - // is gated to the CLI (non-transcript) format, leaving VS Code, - // JetBrains and OTel sources untouched. - // - // We emit one supplementary call per model carrying ONLY the - // input/cache tokens the per-turn events lack; output is excluded so - // the assistant.message output (and its cost) is not double-counted. - // Combined with the per-turn output cost, this yields the full, - // CLI-measured session cost. - if (isTranscript) continue - const shutdownData = event.data as SessionShutdownData - const modelMetrics = shutdownData.modelMetrics - if (!isRecord(modelMetrics)) continue - - const shutdownTimestamp = - (event.timestamp ?? '') || timestampToISO(shutdownData.sessionStartTime) || lastEventTimestamp - - for (const [model, metrics] of Object.entries(modelMetrics)) { - if (!model || !isRecord(metrics)) continue - const usage = metrics['usage'] - if (!isRecord(usage)) continue - - const cacheReadTokens = numberOrZero(usage['cacheReadTokens']) - const cacheWriteTokens = numberOrZero(usage['cacheWriteTokens']) - const reasoningTokens = numberOrZero(usage['reasoningTokens']) - // usage.inputTokens is cache-INCLUSIVE (input + cache_read + - // cache_write). calculateCost expects the uncached input alone with - // cache tokens billed separately, so subtract the cache components. - // Clamp at 0 in case a future schema reports input non-inclusively. - const inputTokens = Math.max( - 0, - numberOrZero(usage['inputTokens']) - cacheReadTokens - cacheWriteTokens - ) - - // Nothing this call would add over the per-turn events, so skip it - // to avoid an empty $0 row (output is intentionally excluded). - if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0) continue - - const dedupKey = `copilot:${sessionId}:shutdown:${model}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - // Tokens are real counts written by the CLI, so this cost is - // measured, not char-estimated: costIsEstimated is false. - // - // NOT lifted to the pricing pass (Phase 0 residual; revisit Phase 8): - // this call carries reasoningTokens for reporting but deliberately - // prices output as 0 (the per-turn assistant.message events own the - // output cost; billing it here would double-count). The generic - // 'estimated' path bills outputTokens + reasoningTokens, so it cannot - // reproduce this reasoning-excluded figure when reasoningTokens > 0. - // Keep the in-decoder price call; the pass leaves it untouched (no - // costBasis). copilot is non-whitelisted, so the cache-read recompute - // already bills reasoning here — a pre-existing cold/warm divergence - // left exactly as-is. - const costUSD = calculateCost(model, inputTokens, 0, cacheWriteTokens, cacheReadTokens, 0) - - yield { - provider: 'copilot', - sessionId, - model, - inputTokens, - outputTokens: 0, - cacheCreationInputTokens: cacheWriteTokens, - cacheReadInputTokens: cacheReadTokens, - cachedInputTokens: 0, - reasoningTokens, - webSearchRequests: 0, - costUSD, - costIsEstimated: false, - tools: [], - bashCommands: [], - timestamp: shutdownTimestamp, - speed: 'standard' as const, - deduplicationKey: dedupKey, - userMessage: '', - } - } - continue - } - - if (event.type === 'assistant.message') { - const msgData = event.data as AssistantMessageData - const { messageId, model: msgModel, outputTokens = 0 } = msgData - const rawRequests = (msgData as { toolRequests?: unknown }).toolRequests - const toolRequests = coerceToolRequests(rawRequests) - - // model may be carried per-message in newer copilot-agent format - if (msgModel) currentModel = msgModel - // Regular JSONL: skip zero-token messages; transcripts don't have tokens - if (!isTranscript && outputTokens === 0) continue - if (!currentModel) continue - - const dedupKey = `copilot:${sessionId}:${messageId}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const tools = toolRequests - .map((t) => { - const raw = typeof t === 'object' && t !== null - ? ((t as { name?: unknown; toolName?: unknown }).name ?? (t as { name?: unknown; toolName?: unknown }).toolName) - : null - return typeof raw === 'string' ? normalizeTool(raw) : null - }) - .filter((t): t is string => t !== null) - - const skills = toolRequests.flatMap((t) => { - if (typeof t !== 'object' || t === null) return [] - const name = (t.name ?? t.toolName) ?? '' - if (name !== 'skill') return [] - const skill = t.arguments?.['skill'] - return typeof skill === 'string' && skill.trim().length > 0 ? [skill.trim()] : [] - }) - - // Extract base command names from bash-type tool requests, routing the - // raw command through the shared extractBashCommands helper so chained - // commands are normalised the same way as every other provider - // (see bash-utils.ts, parser.ts, forge.ts, grok.ts, etc.). - const bashCommands = toolRequests.flatMap((t) => { - if (typeof t !== 'object' || t === null) return [] - const name = (t.name ?? t.toolName) ?? '' - if (!BASH_TOOL_NAMES.has(name)) return [] - const cmd = t.arguments?.['command'] - return typeof cmd === 'string' ? extractBashCommands(cmd) : [] - }) - - // Copilot JSONL only logs outputTokens; inputTokens are NOT available. - // Cost will be lower than actual API cost. This is the original - // behaviour — OTel data (below) replaces it when available. - - yield { - provider: 'copilot', - sessionId, - model: currentModel, - inputTokens: 0, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated' as const, - tools, - bashCommands, - skills: skills.length > 0 ? skills : undefined, - subagentTypes: currentSubagentType ? [currentSubagentType] : undefined, - timestamp: event.timestamp ?? '', - speed: 'standard' as const, - deduplicationKey: dedupKey, - userMessage: pendingUserMessage, - } - pendingUserMessage = '' - } - } - }, - } -} - -function createChatSessionParser( - source: SessionSource, - seenKeys: Set -): SessionParser { - return { - async *parse(): AsyncGenerator { - const content = await readSessionFile(source.path) - if (!content) return - - const root = replayChatSessionJournal(content) - if (!isRecord(root)) return - - const sessionId = readString(root['sessionId']) || basename(source.path, '.jsonl') - const sessionCreatedAt = timestampToISO(root['creationDate']) - const requests = Array.isArray(root['requests']) ? root['requests'] : [] - - for (let index = 0; index < requests.length; index++) { - const rawReq = requests[index] - if (!isRecord(rawReq)) continue - - const result = rawReq['result'] - const resultRecord = isRecord(result) ? result : null - const rawMetadata = resultRecord?.['metadata'] - const metadata = isRecord(rawMetadata) ? rawMetadata : createReplayObject() - - const inputTokens = numberOrZero(metadata['promptTokens']) - const metadataOutputTokens = numberOrZero(metadata['outputTokens']) - const outputTokens = metadataOutputTokens || numberOrZero(rawReq['completionTokens']) - - if (inputTokens === 0 && outputTokens === 0) continue - - const requestId = readString(rawReq['requestId']) || `request-${index}` - const dedupKey = `copilot-chatsession:${sessionId}:${requestId}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const model = modelFromChatSessionRequest(rawReq, metadata) - const timestamp = timestampToISO(rawReq['timestamp']) || sessionCreatedAt - - yield { - provider: 'copilot', - sessionId, - project: source.project, - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated' as const, - tools: extractChatSessionTools(metadata), - bashCommands: [], - timestamp, - speed: 'standard' as const, - deduplicationKey: dedupKey, - userMessage: '', - } - } - }, - } -} - -// --------------------------------------------------------------------------- -// JetBrains parser (Nitrite .db from ~/.config/github-copilot) -// --------------------------------------------------------------------------- -// -// The JetBrains Copilot plugin stores each chat/agent session in a Nitrite -// (H2 MVStore) .db of Java-serialized documents. There is NO token accounting -// anywhere in the store, so we estimate output tokens from the assistant reply -// text (the same char-count approach CodeBurn already uses for Cursor and -// legacy Copilot JSONL). Cost is therefore marked costIsEstimated. -// -// The model (e.g. "claude-opus-4.5", "gpt-4.1") is not always tagged on each -// turn, so we recover it by scanning the raw buffer for a known model token. - -// Known JetBrains Copilot model tokens, longest-first so we match the most -// specific name (e.g. "gpt-4.1-mini" before "gpt-4.1"). -const JETBRAINS_MODEL_TOKENS = [ - 'claude-opus-4.5', - 'claude-opus-4.1', - 'claude-opus-4', - 'claude-sonnet-4.5', - 'claude-sonnet-4', - 'gpt-5.3-codex', - 'gpt-5.3', - 'gpt-5.2', - 'gpt-5.1', - 'gpt-5-mini', - 'gpt-5', - 'gpt-4.1-mini', - 'gpt-4.1-nano', - 'gpt-4.1', - 'gpt-4o-mini', - 'gpt-4o', - 'gemini-2.5-pro', - 'gemini-2.0-flash', - 'o3-mini', - 'o4-mini', - 'o3', -] - -/** - * Normalise a raw JetBrains model token to CodeBurn's canonical model id. - * Claude names use dots on disk (claude-opus-4.5) but dashes in the pricing - * tables (claude-opus-4-5); GPT/Gemini names are kept verbatim. - */ -function normalizeJetBrainsModelName(raw: string): string { - const t = raw.trim() - if (!t) return '' - if (t.startsWith('claude-')) return t.replace(/\./g, '-') - return t -} - -/** Match a known model token at an alnum boundary anywhere in a string. */ -function findJetBrainsModelToken(s: string): string { - for (const token of JETBRAINS_MODEL_TOKENS) { - const escaped = token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - // "o3" etc. must not match inside words like "iso3166". - if (new RegExp(`(?() - let m: RegExpExecArray | null - while ((m = re.exec(raw))) { - // Decode %20 etc. and strip a trailing .rej/.orig suffix noise; keep the dir. - let p = m[1] - try { p = decodeURIComponent(p) } catch { /* leave as-is */ } - const dir = p.slice(0, p.lastIndexOf('/')) - if (dir.startsWith('/')) seen.add(dir) - } - if (seen.size === 0) return undefined - - for (const dir of seen) { - const repo = findGitRepoRoot(dir) - if (repo) return repo - } - return undefined -} - /** Walk up from `dir` to the nearest ancestor containing `.git`; return its basename. */ function findGitRepoRoot(dir: string): string | undefined { let cur = dir @@ -1160,675 +320,6 @@ function extractJetBrainsProjectName(raw: string): string | undefined { return undefined } -// --------------------------------------------------------------------------- -// Nitrite .db (H2 MVStore) extraction -// --------------------------------------------------------------------------- -// -// JetBrains Copilot sessions store their conversation in the Nitrite .db -// (copilot-*-nitrite.db). One .db holds many conversations. Assistant replies -// are stored as a distinct blob shape: -// -// {"__first__":{"type":"Subgraph","value":"..."}, ...} -// -// which is more deeply escaped than the user-message value-maps. The reply text -// is recovered by progressive unescaping and collecting "text":"..." fields. -// Failed turns ("Sorry, an error occurred …") carry an error status and no reply -// text — they are detected and billed as $0. - -// One assistant turn recovered from a .db. -type JBDbTurn = { - replyText: string - model: string - errored: boolean - // The owning conversation (chat tab): its internal GUID and title. One .db - // holds many conversations; turns are grouped back to their tab by this id. - conversationId: string - conversationTitle: string - // The file path this conversation referenced (home-relative common dir), or - // '' if the chat touched no files. Used as the project label. - conversationProject: string -} - -// A conversation (chat tab) recovered from a .db: internal GUID → title. -type JBConversation = { id: string; title: string } - -/** - * Recover the conversation (chat-tab) records from a raw .db buffer. Each is - * stored as `$ … name … value … source copilot`. Returns the - * GUID→title map so turns can be grouped back to the tab the user sees. - */ -function extractJetBrainsConversations(raw: string): JBConversation[] { - // A conversation's title EVOLVES as the user chats: it starts as "New Agent - // Session", may pass through an auto-generated name, and ends at the final - // title shown in the UI. The same `$<GUID> … name … value <title> … source` - // record is rewritten each time, so we collect every occurrence per GUID and - // keep the LAST meaningful (non-default) one. - const DEFAULT_TITLES = new Set(['New Agent Session', 'New Session', 'New Chat']) - const byId = new Map<string, string>() - const re = /\$([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})[\s\S]{0,8}name/g - let m: RegExpExecArray | null - while ((m = re.exec(raw))) { - const id = m[1] - const window = raw.slice(m.index, m.index + 400) - // The title is the Java-UTF string between the `value` marker and `source`. - const tm = window.match(/value.{1,6}?([\x20-\x7e]{3,80}?)t\x00\x06source/) - if (!tm) continue - const title = Buffer.from(tm[1].replace(/^[^A-Za-z0-9]*/, ''), 'latin1').toString('utf8').trim() - if (!title) continue - // Keep the latest non-default title; only fall back to a default if no - // meaningful title has been seen for this conversation yet. - const existing = byId.get(id) - if (existing && !DEFAULT_TITLES.has(existing) && DEFAULT_TITLES.has(title)) continue - byId.set(id, title) - } - return [...byId.entries()].map(([id, title]) => ({ id, title })) -} - -/** Brace-match a JSON object starting at `start`, tolerating escaped quotes. */ -function matchJsonObject(raw: string, start: number): { chunk: string; end: number } { - let depth = 0 - let inStr = false - let esc = false - let i = start - for (; i < raw.length; i++) { - const c = raw[i] - if (esc) { esc = false; continue } - if (c === '\\') { esc = true; continue } - if (c === '"') { inStr = !inStr; continue } - if (inStr) continue - if (c === '{') depth++ - else if (c === '}') { depth--; if (depth === 0) { i++; break } } - } - return { chunk: raw.slice(start, i), end: i } -} - -/** - * Recover the assistant reply text from a `__first__`/Subgraph response blob. - * - * JetBrains Copilot has two turn shapes, both handled here: - * - * - **Ask mode:** the reply is a `Markdown` record whose `data` is an escaped - * JSON document `{"text":"…","annotations":…}`. - * - **Agent mode** (e.g. PyCharm agent sessions): the reply is the `reply` - * field of an `AgentRound` record `{"roundId":N,"reply":"…","toolCalls":[…]}`. - * In agent mode the `Markdown` records hold the USER's prompts, not the - * reply, so we must NOT read them — the assistant output is the AgentRound - * reply. - * - * Both are read STRUCTURALLY rather than by fully unescaping the blob (which - * would strip the reply's own quotes and make regex extraction ambiguous): we - * locate each `data`/`reply` value, read it as a properly-delimited JSON-string - * literal (honouring escaping), unescape one level, and `JSON.parse` to reach - * the text. We unescape the blob one level at a time and extract at the first - * depth that yields text, never accumulating across depths (which would union a - * quote-truncated half-unescaped capture with the full one and garble the - * reply, inflating the token/cost estimate). - * - * Steps/error/progress-only blobs (no Markdown text and no AgentRound reply) - * yield '' and are billed as $0 upstream. - */ -function extractResponseText(blob: string): string { - let s = blob - for (let depth = 0; depth < 8; depth++) { - // Decide the mode by the PRESENCE of an AgentRound record, not by whether it - // yielded a reply. In agent mode the Markdown record holds the USER prompt, - // so an agent blob whose reply is empty (a failed turn, or a pure tool-call - // round) must NOT fall back to Markdown — that would bill the user's prompt - // as the assistant's output. Ask-mode blobs have no AgentRound record and - // use Markdown. (Verified across every observed store: the two reply shapes - // never coexist in one blob, so this mode split is unambiguous.) - const isAgentMode = /"type":"AgentRound"/.test(s) - if (isAgentMode || /"type":"Markdown"/.test(s)) { - const decoded = isAgentMode ? extractAgentRoundReplies(s) : extractMarkdownTexts(s) - // The .db is read as latin1 (byte-stable), so multibyte UTF-8 characters - // are split into separate code units. Re-interpret as UTF-8 so the char - // count (→ token estimate) reflects real content length, not byte count. - // decoded may be empty (failed/tool-only agent turn) → '' (billed $0). - return Buffer.from(decoded.join('\n').trim(), 'latin1').toString('utf8') - } - // Not yet at the depth where record markers appear bare — unescape one level - // in a single left-to-right pass so `\\` and `\"` resolve together (a - // two-pass replace would turn `\\"` into `\"` not `\\` + `"`). - const next = s.replace(/\\([\\"])/g, '$1') - if (next === s) break - s = next - } - return '' -} - -/** - * Collect the `text` of every `Markdown` record in `s`, treating each record's - * `data` value as a one-level-escaped JSON string parsed structurally (so the - * reply's own quotes never truncate it). Returns [] if `s` is not yet at the - * right unescape depth (no bare `"type":"Markdown"` with a parseable `data`). - * Scoping to Markdown skips `Error` (`message`) and `Steps` records — not - * billable output. Revisions repeat a reply, so identical texts are de-duped. - */ -function extractMarkdownTexts(s: string): string[] { - return extractRecordStrings(s, '"type":"Markdown"', '"data":"', 'text') -} - -/** - * Collect the non-empty `reply` of every `AgentRound` record (agent mode). A - * single blob can hold several rounds (a multi-turn agent session); each round's - * `reply` is the assistant's text for that step (empty on pure tool-call rounds). - * Deduped in order. - */ -function extractAgentRoundReplies(s: string): string[] { - return extractRecordStrings(s, '"type":"AgentRound"', '"data":"', 'reply') -} - -/** - * Shared structural reader: for every `<marker>` in `s`, find the following - * `<dataKey>` string literal (a one-level-escaped JSON document), parse it, and - * collect `doc[field]` when it is a non-empty string. Reading the value as a - * delimited literal — not a greedy regex — means the payload's own quotes never - * truncate it. Returns [] when `s` is not yet at the depth where the marker - * appears bare with a parseable payload. De-dupes in order (the store keeps - * byte-copies/revisions of each reply). - */ -function extractRecordStrings(s: string, marker: string, dataKey: string, field: string): string[] { - const texts: string[] = [] - const seen = new Set<string>() - const re = new RegExp(marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g') - let m: RegExpExecArray | null - while ((m = re.exec(s))) { - const dk = s.indexOf(dataKey, m.index) - if (dk === -1 || dk - m.index > 200) continue - // The value runs from after `<dataKey>` to the first UNescaped quote (an odd - // run of preceding backslashes escapes it). - const start = dk + dataKey.length - let i = start - for (; i < s.length; i++) { - if (s[i] !== '"') continue - let bs = 0 - for (let j = i - 1; j >= start && s[j] === '\\'; j--) bs++ - if (bs % 2 === 0) break - } - const literal = s.slice(start, i) - try { - // Wrapping in quotes + parsing unescapes exactly one level → the inner - // JSON document as a string; parsing THAT reaches { <field>, … }. - const doc = JSON.parse(JSON.parse('"' + literal + '"') as string) as Record<string, unknown> - const text = typeof doc[field] === 'string' ? (doc[field] as string) : '' - if (text && !seen.has(text)) { - seen.add(text) - texts.push(text) - } - } catch { - // Not the right depth (or not a matching record) — skip. - } - } - return texts -} - -/** - * Extract assistant turns from a raw (latin1) Nitrite .db buffer. Each turn is - * one `{"__first__":{"type":"Subgraph"…}` blob; the per-turn model is recovered - * from inside the blob when present, else the whole-store default. Each turn is - * grouped back to its owning conversation (chat tab) by the nearest preceding - * conversation GUID. Duplicate byte-copies of the same reply (the store keeps - * several) are de-duplicated by content, per conversation. - */ -function extractJetBrainsDbTurns(raw: string): JBDbTurn[] { - const conversations = extractJetBrainsConversations(raw) - // Precompute the byte offset of each conversation GUID's full form so a turn - // can be attributed to the conversation whose id most recently precedes it. - const convById = new Map(conversations.map((c) => [c.id, c])) - - const turns: JBDbTurn[] = [] - const seenReplies = new Set<string>() // keyed by `${conversationId}::${reply}` - const re = /\{"__first__":\{"type":"Subgraph"/g - let m: RegExpExecArray | null - while ((m = re.exec(raw))) { - const { chunk, end } = matchJsonObject(raw, m.index) - re.lastIndex = end - - // Attribute this turn to the conversation whose GUID last appears before it. - let conversationId = '' - let conversationTitle = '' - let bestPos = -1 - for (const c of convById.values()) { - const p = raw.lastIndexOf(c.id, m.index) - if (p > bestPos) { - bestPos = p - conversationId = c.id - conversationTitle = c.title - } - } - - const replyText = extractResponseText(chunk) - // The files this turn referenced (home-relative common dir) → project label. - const conversationProject = inferJetBrainsProject(chunk) ?? '' - // A per-turn model token sometimes appears inside the blob. - const model = findJetBrainsModelToken(chunk) - // A failed turn carries an error status / phrase AND produces no reply text. - // Requiring empty text avoids misclassifying a genuine reply that merely - // *discusses* an error (e.g. explaining a stack trace) as a failed turn. - const hasErrorMarker = /error occurred|"isError":true|\\+"status\\+":\\+"(?:error|failed)\\+"/i.test(chunk) - if (hasErrorMarker && !replyText) { - turns.push({ replyText: '', model, errored: true, conversationId, conversationTitle, conversationProject }) - continue - } - if (!replyText) continue // Steps/progress-only blob — no billable output - const dedupeKey = `${conversationId}::${replyText}` - if (seenReplies.has(dedupeKey)) continue - seenReplies.add(dedupeKey) - turns.push({ replyText, model, errored: false, conversationId, conversationTitle, conversationProject }) - } - - // --------------------------------------------------------------------------- - // Fallback: old JetBrains Copilot plugin format (≤1.5.x, e.g. 1.5.59-243) - // --------------------------------------------------------------------------- - // In this format ALL session turns are stored inside ONE large outer Nitrite - // document — a binary-framed JSON object with UUID-keyed Value entries — rather - // than the per-turn {"__first__":{"type":"Subgraph",...}} blobs used by newer - // plugins (≥1.12.x). The AgentRound entries sit one escaping level deeper - // inside the outer document's string values, so `extractResponseText`'s - // depth-unescape loop handles extraction correctly once we feed it the right - // chunk. MVStore keeps two identical copies of the collection; `seenReplies` - // deduplicates them automatically. - // - // Detection heuristic: the __first__/Subgraph path produced no turns AND the - // raw file contains bare 'AgentRound' text (meaning old-format data is present). - if (turns.length === 0 && raw.includes('AgentRound')) { - // The outer Nitrite document is preceded by a single binary framing byte - // (0x81 in practice, but any non-printable/non-ASCII byte in MVStore). - // It starts with a UUID-keyed Value entry: {"<uuid>":{"type":"Value",...}}. - // Hex is matched case-insensitively — an uppercase UUID must not cause the - // whole session to fall through to $0 (the exact bug this path fixes). - const outerDocRe = /[\x00-\x1f\x7f-\xff]\{"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}":\{"type":"Value"/g - let dm: RegExpExecArray | null - while ((dm = outerDocRe.exec(raw))) { - // Skip the leading binary byte; matchJsonObject starts at the '{'. - const docStart = dm.index + 1 - const { chunk, end } = matchJsonObject(raw, docStart) - outerDocRe.lastIndex = end - - // Skip documents that contain no AgentRound data (e.g. empty sessions). - if (!chunk.includes('AgentRound')) continue - - // Attribute to the conversation whose GUID most recently precedes this doc. - let conversationId = '' - let conversationTitle = '' - let bestPos = -1 - for (const c of convById.values()) { - const p = raw.lastIndexOf(c.id, docStart) - if (p > bestPos) { - bestPos = p - conversationId = c.id - conversationTitle = c.title - } - } - - // extractResponseText handles the depth-1 unescape needed to surface the - // AgentRound records, then calls extractAgentRoundReplies for each turn. - // Because the outer document holds ALL turns in one blob we get back a - // single joined string; split it on the '\n' join to yield per-turn texts. - const allReplies = extractResponseText(chunk) - if (!allReplies) continue - - const conversationProject = inferJetBrainsProject(chunk) ?? '' - const storeModel = findJetBrainsModelToken(chunk) - - // extractResponseText joins multiple replies with '\n'. Since individual - // replies can themselves span multiple lines we cannot cleanly split here — - // instead we emit one ParsedProviderCall per outer document (one session). - const dedupeKey = `${conversationId}::${allReplies}` - if (seenReplies.has(dedupeKey)) continue - seenReplies.add(dedupeKey) - - turns.push({ - replyText: allReplies, - model: storeModel, - errored: false, - conversationId, - conversationTitle, - conversationProject, - }) - } - } - - // A project derived from ANY turn of a conversation applies to all its turns - // (the files are usually referenced in the first substantive turn only). - const projByConv = new Map<string, string>() - for (const t of turns) { - if (t.conversationProject && !projByConv.has(t.conversationId)) { - projByConv.set(t.conversationId, t.conversationProject) - } - } - for (const t of turns) { - if (!t.conversationProject) t.conversationProject = projByConv.get(t.conversationId) ?? '' - } - - return turns -} - -// --------------------------------------------------------------------------- -// JetBrains parser: one ParsedProviderCall per assistant turn in the .db -// --------------------------------------------------------------------------- - -function createJetBrainsParser( - source: JetBrainsSessionSource, - seenKeys: Set<string> -): SessionParser { - return { - async *parse(): AsyncGenerator<ParsedProviderCall> { - const sessionId = source.sessionId - - // Nitrite .db (the store's authoritative session content). Read as latin1 - // so byte offsets are stable through the binary MVStore framing. - if (source.dbPath) { - let dbRaw: string | null = null - try { - dbRaw = await readSessionFile(source.dbPath, 'latin1') - } catch { - dbRaw = null - } - if (dbRaw) { - const storeModel = inferJetBrainsModel(dbRaw) - const turns = extractJetBrainsDbTurns(dbRaw) - // Dedup keys derive from the reply CONTENT, not the scan position: - // copilot is a durable provider (cached turns are never deleted and a - // re-parse appends any key it hasn't seen), while MVStore compaction - // can rewrite the file with blobs in a different byte order. With - // positional keys, a rewrite that puts a new blob ahead of an old one - // hands the new turn the old turn's key (skipped as seen) and re-emits - // the old turn under a fresh index — double-billing it. The per-hash - // counter keeps genuinely repeated replies and errored turns (which - // share replyText '') distinct within a conversation. - const perContentIndex = new Map<string, number>() - for (const turn of turns) { - // One .db holds many chat tabs; group each turn under its own - // conversation so the user sees one session per tab, not per file. - const convId = turn.conversationId || sessionId - const contentHash = createHash('sha256').update(turn.replyText).digest('hex').slice(0, 12) - const nth = (perContentIndex.get(`${convId}:${contentHash}`) ?? 0) + 1 - perContentIndex.set(`${convId}:${contentHash}`, nth) - const dedupKey = `copilot:jb:${convId}:${contentHash}:${nth}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - // Prefer the per-turn model, else the store default, else a generic - // Copilot bucket so a real reply is never mis-priced as free. - const model = turn.model || storeModel || 'copilot-anthropic-auto' - // Errored turns (failed generation) contribute no billable output. - const outputTokens = turn.errored ? 0 : estimateTokens(turn.replyText) - // Project resolution precedence: - // 1. projectName — the plugin's own recorded label (1.12+), - // joined across kind dirs by store id. Authoritative. - // 2. the git repo root of a file:// path the chat referenced - // (older plugins / when projectName is absent). - // 3. one honest bucket when neither signal exists. - // The conversation TITLE is a chat-thread name, NOT a project, and is - // kept out of `project` (it would otherwise pollute By-Project). - const project = - source.projectName || turn.conversationProject || 'copilot-jetbrains' - - yield { - provider: 'copilot', - sessionId: convId, - project, - model, - inputTokens: 0, - outputTokens, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - // Tokens are char-estimated (costIsEstimated) but the dollar - // amount is still table-priced from those tokens: costBasis - // 'estimated'. Errored turns have outputTokens 0 → priced to $0. - costBasis: 'estimated' as const, - costIsEstimated: true, - tools: [], - bashCommands: [], - timestamp: source.mtime, - speed: 'standard' as const, - deduplicationKey: dedupKey, - // Surface the chat-thread name here (it is the session's label, not - // a project) so it remains visible in session-level views. - userMessage: turn.conversationTitle, - } - } - } - } - - }, - } -} - -// --------------------------------------------------------------------------- -// OTel SQLite parser — reads agent-traces.db for FULL token data -// --------------------------------------------------------------------------- - -function createOtelParser( - source: SessionSource, - seenKeys: Set<string> -): SessionParser { - return { - async *parse(): AsyncGenerator<ParsedProviderCall> { - // Lazy-load the SQLite module (same pattern as Cursor/OpenCode providers) - const { openDatabase } = await import('../sqlite.js') - - // One DB open handles ALL conversations — avoids N opens for N conversations. - const db = openDatabase(source.path) - - try { - // --------------------------------------------------------------- - // Get all distinct conversations in the DB with their project names. - // --------------------------------------------------------------- - const conversationRows = db.query<{ - conversation_id: string - project: string | null - min_start: number - }>( - `SELECT DISTINCT - sa_conv.value AS conversation_id, - COALESCE(sa_repo.value, 'copilot-chat') AS project, - MIN(s.start_time_ms) AS min_start - FROM spans s - LEFT JOIN span_attributes sa_conv - ON s.span_id = sa_conv.span_id AND sa_conv.key = 'gen_ai.conversation.id' - LEFT JOIN span_attributes sa_repo - ON s.span_id = sa_repo.span_id AND sa_repo.key = 'github.copilot.git.repository' - WHERE sa_conv.value IS NOT NULL - GROUP BY sa_conv.value - ORDER BY min_start DESC` - ) - - for (const convRow of conversationRows) { - const conversationId = convRow.conversation_id - if (!conversationId) continue - - let project = convRow.project ?? 'copilot-chat' - if (project.includes('/')) { - project = basename(project.replace(/\.git$/, '')) - } - - // ----------------------------------------------------------- - // Query all 'chat' spans for this conversation. - // ----------------------------------------------------------- - - const spanIdRows = db.query<{ span_id: string; trace_id: string }>( - `SELECT DISTINCT s.span_id, s.trace_id - FROM spans s - INNER JOIN span_attributes sa - ON s.span_id = sa.span_id AND sa.key = 'gen_ai.conversation.id' AND sa.value = ? - ORDER BY s.start_time_ms ASC`, - [conversationId] - ) - - // Collect trace IDs and span IDs belonging to this conversation - const traceIds = new Set<string>() - for (const row of spanIdRows) { - traceIds.add(row.trace_id) - } - - if (traceIds.size === 0) { - continue - } - - // Now query all spans within those traces to find chat and tool spans. - // Pull the metadata columns in the same query so we don't re-query the - // spans table once per chat span below (avoids an N+1). - const traceIdArr = [...traceIds] - const tracePlaceholders = traceIdArr.map(() => '?').join(',') - const traceSpans = db.query<{ - span_id: string - trace_id: string - operation_name: string | null - start_time_ms: number - response_model: string | null - }>( - `SELECT span_id, trace_id, operation_name, start_time_ms, response_model FROM spans WHERE trace_id IN (${tracePlaceholders})`, - traceIdArr - ) - - // Collect tool names, shell commands and subagent names from the - // execute_tool / invoke_agent spans for each trace. These mirror the - // metadata the JSONL path captures, so the OTel source stays - // equivalent (tools + bashCommands + subagentTypes are all first-class - // call metadata per types.ts). - // - // Subagent attribution: VS Code records a subagent run as an - // invoke_agent span carrying copilot_chat.parent_chat_session_id. The - // root turn agent (gen_ai.agent.name = 'GitHub Copilot Chat') has NO - // parent session and is intentionally excluded, otherwise it would - // surface as a bogus 'GitHub Copilot Chat' entry in the agents view. - // A subagent's invoke_agent span lives in the same trace as that - // subagent's own chat spans, so attributing the agent name per-trace - // labels exactly the subagent's calls. - const toolsByTrace = new Map<string, string[]>() - const bashByTrace = new Map<string, string[]>() - const subagentsByTrace = new Map<string, string[]>() - const chatSpanIds: string[] = [] - const spanMetaById = new Map<string, { trace_id: string; start_time_ms: number; response_model: string | null }>() - - for (const span of traceSpans) { - const opName = span.operation_name || '' - spanMetaById.set(span.span_id, span) - - if (opName === 'chat') { - chatSpanIds.push(span.span_id) - continue - } - - if (opName === 'execute_tool') { - // Load tool name from attributes and normalise to display form - const attrs = loadSpanAttributesFromTable(db, span.span_id) - const rawToolName = attrs['gen_ai.tool.name'] as string | undefined - if (rawToolName) { - const existing = toolsByTrace.get(span.trace_id) ?? [] - existing.push(normalizeTool(rawToolName)) - toolsByTrace.set(span.trace_id, existing) - - // For shell tools, extract command names via the OTEL-specific - // normaliser (handles the full multi-line scripts the OTEL store - // records; see extractOtelBashCommands). - if (BASH_TOOL_NAMES.has(rawToolName)) { - const command = parseToolCommand(attrs['gen_ai.tool.call.arguments']) - if (command) { - const bash = bashByTrace.get(span.trace_id) ?? [] - bash.push(...extractOtelBashCommands(command)) - bashByTrace.set(span.trace_id, bash) - } - } - } - continue - } - - // Genuine subagent invocation: an invoke_agent span with a parent - // chat session. The root turn agent ('GitHub Copilot Chat') has no - // parent session and is skipped to avoid a bogus agents-view entry. - if (opName === 'invoke_agent') { - const attrs = loadSpanAttributesFromTable(db, span.span_id) - const parentSession = attrs['copilot_chat.parent_chat_session_id'] - const agentName = attrs['gen_ai.agent.name'] as string | undefined - if (parentSession && agentName) { - const subs = subagentsByTrace.get(span.trace_id) ?? [] - subs.push(agentName) - subagentsByTrace.set(span.trace_id, subs) - } - } - } - - // Yield one ParsedProviderCall per chat span - for (const spanId of chatSpanIds) { - const attrs = loadSpanAttributesFromTable(db, spanId) - - const spanMetadata = spanMetaById.get(spanId) - if (!spanMetadata) continue - - const model = - (attrs['gen_ai.response.model'] as string | undefined) ?? - (attrs['gen_ai.request.model'] as string | undefined) ?? - spanMetadata.response_model ?? - 'unknown' - - const inputTokens = Number(attrs['gen_ai.usage.input_tokens'] ?? 0) - const outputTokens = Number(attrs['gen_ai.usage.output_tokens'] ?? 0) - const cacheReadTokens = Number(attrs['gen_ai.usage.cache_read.input_tokens'] ?? 0) - const cacheCreationTokens = Number(attrs['gen_ai.usage.cache_creation.input_tokens'] ?? 0) - - if (inputTokens === 0 && outputTokens === 0) { - continue - } - - // Dedup key uses span_id which is globally unique - const dedupKey = `copilot-otel:${spanId}` - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - // Also add a JSONL-style dedupKey pattern so that if the same - // interaction appears in both OTel and JSONL, we don't double-count. - // We use the turn ID from Copilot attributes if available. - const turnId = attrs['github.copilot.chat.turn.id'] as string | undefined - if (turnId) { - const jsonlDedupKey = `copilot:${conversationId}:${turnId}` - seenKeys.add(jsonlDedupKey) - } - - const tools = toolsByTrace.get(spanMetadata.trace_id) ?? [] - const bashCommands = bashByTrace.get(spanMetadata.trace_id) ?? [] - const subagentTypes = subagentsByTrace.get(spanMetadata.trace_id) - const timestamp = epochToISO(spanMetadata.start_time_ms) - - // FULL token data (input + cache) — the key improvement — priced by - // the pass. reasoningTokens is 0 (not in the OTel schema), so the - // output basis is exactly outputTokens. - - yield { - provider: 'copilot', - sessionId: conversationId, - project, - model, - inputTokens, - outputTokens, - cacheCreationInputTokens: cacheCreationTokens, - cacheReadInputTokens: cacheReadTokens, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated' as const, - tools, - bashCommands, - subagentTypes: subagentTypes && subagentTypes.length > 0 ? subagentTypes : undefined, - timestamp, - speed: 'standard' as const, - deduplicationKey: dedupKey, - userMessage: '', // Not available in OTel spans by default - } - } - } - } finally { - db.close() - } - }, - } -} - // --------------------------------------------------------------------------- // Extended SessionSource for OTel sessions // --------------------------------------------------------------------------- @@ -2066,10 +557,6 @@ async function resolveJetBrainsProjectNames( } } -// --------------------------------------------------------------------------- -// Provider factory -// --------------------------------------------------------------------------- - /** * Returns the VS Code workspaceStorage directories for all VS Code variants * (Code, Code Insiders, VSCodium) on the given platform. Used to discover @@ -2289,6 +776,211 @@ async function discoverTranscriptSessions( return sources } +// --------------------------------------------------------------------------- +// Bridged provider I/O adapter +// --------------------------------------------------------------------------- + +async function readJsonlRecords(source: SessionSource): Promise<unknown[] | null> { + const content = await readSessionFile(source.path) + if (!content) return null + return [{ kind: 'jsonl', sessionId: basename(dirname(source.path)), + lines: content.split('\n').filter((l) => l.trim()) }] +} + +async function readChatSessionRecords(source: SessionSource): Promise<unknown[] | null> { + const content = await readSessionFile(source.path) + if (!content) return null + return [{ kind: 'chatsession', content, project: source.project, + fallbackSessionId: basename(source.path, '.jsonl') }] +} + +async function readJetBrainsRecords(source: SessionSource): Promise<unknown[] | null> { + const jbSource = source as JetBrainsSessionSource + if (!jbSource.dbPath) return null + let raw: string | null = null + try { raw = await readSessionFile(jbSource.dbPath, 'latin1') } catch { raw = null } + if (!raw) return null + // FS probe stays host-side: resolve every candidate dir the decoder could ask for. + const memo = new Map<string, string | undefined>() + const repoRootByDir = new Map<string, string>() + for (const dir of collectJetBrainsRepoDirCandidates(raw)) { + if (!memo.has(dir)) memo.set(dir, findGitRepoRoot(dir)) + const repo = memo.get(dir) + if (repo) repoRootByDir.set(dir, repo) + } + return [{ kind: 'jetbrains', raw, repoRootByDir, sessionId: jbSource.sessionId, + mtime: jbSource.mtime, ...(jbSource.projectName !== undefined ? { projectName: jbSource.projectName } : {}) }] +} + +async function readOtelRecords(source: SessionSource): Promise<unknown[] | null> { + // Lazy-load the SQLite module (same pattern as Cursor/OpenCode providers) + const { openDatabase } = await import('../sqlite.js') + + // One DB open handles ALL conversations — avoids N opens for N conversations. + const db = openDatabase(source.path) + + try { + // --------------------------------------------------------------- + // Get all distinct conversations in the DB with their project names. + // --------------------------------------------------------------- + const conversationRows = db.query<{ + conversation_id: string + project: string | null + min_start: number + }>( + `SELECT DISTINCT + sa_conv.value AS conversation_id, + COALESCE(sa_repo.value, 'copilot-chat') AS project, + MIN(s.start_time_ms) AS min_start + FROM spans s + LEFT JOIN span_attributes sa_conv + ON s.span_id = sa_conv.span_id AND sa_conv.key = 'gen_ai.conversation.id' + LEFT JOIN span_attributes sa_repo + ON s.span_id = sa_repo.span_id AND sa_repo.key = 'github.copilot.git.repository' + WHERE sa_conv.value IS NOT NULL + GROUP BY sa_conv.value + ORDER BY min_start DESC` + ) + + const conversations: CopilotOtelConversationRecord[] = [] + + for (const convRow of conversationRows) { + const conversationId = convRow.conversation_id + if (!conversationId) continue + + let project = convRow.project ?? 'copilot-chat' + if (project.includes('/')) { + project = basename(project.replace(/\.git$/, '')) + } + + const spanIdRows = db.query<{ span_id: string; trace_id: string }>( + `SELECT DISTINCT s.span_id, s.trace_id + FROM spans s + INNER JOIN span_attributes sa + ON s.span_id = sa.span_id AND sa.key = 'gen_ai.conversation.id' AND sa.value = ? + ORDER BY s.start_time_ms ASC`, + [conversationId] + ) + + const traceIds = new Set<string>() + for (const row of spanIdRows) { + traceIds.add(row.trace_id) + } + + if (traceIds.size === 0) { + continue + } + + const traceIdArr = [...traceIds] + const tracePlaceholders = traceIdArr.map(() => '?').join(',') + const traceSpans = db.query<{ + span_id: string + trace_id: string + operation_name: string | null + start_time_ms: number + response_model: string | null + }>( + `SELECT span_id, trace_id, operation_name, start_time_ms, response_model FROM spans WHERE trace_id IN (${tracePlaceholders})`, + traceIdArr + ) + + const spans: CopilotOtelSpanRecord[] = [] + for (const span of traceSpans) { + const operationName = span.operation_name || '' + const attrs: SpanAttributes | null = + operationName === 'chat' || operationName === 'execute_tool' || operationName === 'invoke_agent' + ? loadSpanAttributesFromTable(db, span.span_id) + : null + spans.push({ + spanId: span.span_id, + traceId: span.trace_id, + operationName, + startTimeMs: span.start_time_ms, + responseModel: span.response_model, + attrs, + }) + } + + conversations.push({ conversationId, project, spans }) + } + + return [{ kind: 'otel', conversations }] + } finally { + db.close() + } +} + +async function readRecords(source: SessionSource): Promise<unknown[] | null> { + if (isOtelSource(source)) return readOtelRecords(source) + if (isChatSessionSource(source)) return readChatSessionRecords(source) + if (isJetBrainsSource(source)) return readJetBrainsRecords(source) + return readJsonlRecords(source) +} + +function toProviderCall(rich: CopilotDecodedCall): ParsedProviderCall { + const base = { + provider: 'copilot' as const, + sessionId: rich.sessionId, + ...(rich.project !== undefined ? { project: rich.project } : {}), + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + tools: rich.tools, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + } + + if (rich.arm === 'jsonl-shutdown') { + // Tokens are real counts written by the CLI, so this cost is + // measured, not char-estimated: costIsEstimated is false. + // + // NOT lifted to the pricing pass (Phase 0 residual; revisit Phase 8): + // this call carries reasoningTokens for reporting but deliberately + // prices output as 0 (the per-turn assistant.message events own the + // output cost; billing it here would double-count). The generic + // 'estimated' path bills outputTokens + reasoningTokens, so it cannot + // reproduce this reasoning-excluded figure when reasoningTokens > 0. + // Keep the in-decoder price call; the pass leaves it untouched (no + // costBasis). copilot is non-whitelisted, so the cache-read recompute + // already bills reasoning here — a pre-existing cold/warm divergence + // left exactly as-is. + const costUSD = calculateCost( + rich.model, rich.inputTokens, 0, rich.cacheCreationInputTokens, rich.cacheReadInputTokens, 0, + ) + return { ...base, costUSD, costIsEstimated: false, bashCommands: [] } + } + + if (rich.arm === 'otel') { + return { ...base, costBasis: 'estimated', + bashCommands: rich.rawBashCommands.flatMap((c) => extractOtelBashCommands(c)), + subagentTypes: rich.subagentTypes } + } + + if (rich.arm === 'jsonl-turn') { + return { ...base, costBasis: 'estimated', + bashCommands: rich.rawBashCommands.flatMap((c) => extractBashCommands(c)), + skills: rich.skills, subagentTypes: rich.subagentTypes } + } + + if (rich.arm === 'jetbrains') { + return { ...base, costBasis: 'estimated', costIsEstimated: true, bashCommands: [] } + } + + // chatsession + return { ...base, costBasis: 'estimated', bashCommands: [] } +} + +// --------------------------------------------------------------------------- +// Provider factory +// --------------------------------------------------------------------------- + export function createCopilotProvider( sessionStateDir?: string, workspaceStorageDir?: string, @@ -2319,7 +1011,7 @@ export function createCopilotProvider( return getVSCodeGlobalStorageDirs(homedir(), platform()) } - return { + return createBridgedProvider<CopilotDecodedCall>({ name: 'copilot', displayName: 'Copilot', durableSources: true, @@ -2332,7 +1024,7 @@ export function createCopilotProvider( }, toolDisplayName(rawTool: string): string { - return normalizeTool(rawTool) + return normalizeCopilotTool(rawTool) }, async discoverSessions(): Promise<SessionSource[]> { @@ -2406,26 +1098,10 @@ export function createCopilotProvider( return sources }, - createSessionParser( - source: SessionSource, - seenKeys: Set<string> - ): SessionParser { - // Route to the correct parser based on source type. - // The dedup key set (seenKeys) is shared across both parsers, - // so if OTel already yielded a span, the JSONL parser will skip - // the matching assistant.message (and vice versa). - if (isOtelSource(source)) { - return createOtelParser(source, seenKeys) - } - if (isChatSessionSource(source)) { - return createChatSessionParser(source, seenKeys) - } - if (isJetBrainsSource(source)) { - return createJetBrainsParser(source, seenKeys) - } - return createJsonlParser(source, seenKeys) - }, - } + readRecords, + decode: decodeCopilot, + toProviderCall, + }) } // Default export for the provider registry diff --git a/packages/cli/tests/providers/copilot-bridge.test.ts b/packages/cli/tests/providers/copilot-bridge.test.ts new file mode 100644 index 00000000..e304b0cf --- /dev/null +++ b/packages/cli/tests/providers/copilot-bridge.test.ts @@ -0,0 +1,1809 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { mkdtemp, mkdir, writeFile, rm } from 'fs/promises' +import { join, basename, dirname } from 'path' +import { tmpdir } from 'os' +import { createRequire } from 'node:module' + +import { createCopilotProvider } from '../../src/providers/copilot.js' +import { isSqliteAvailable } from '../../src/sqlite.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall } from '../../src/providers/types.js' + +const requireForTest = createRequire(import.meta.url) + +let tmpDir: string + +beforeEach(async () => { + tmpDir = await mkdtemp(join(tmpdir(), 'copilot-bridge-')) + // Disable OTel discovery by default; OTel scenarios override this. + vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1') +}) + +afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }) + vi.unstubAllEnvs() +}) + +// ----------------------------------------------------------------------------- +// JSONL / CLI session-state fixture builders (copied from copilot.test.ts) +// ----------------------------------------------------------------------------- + +async function createSessionDir(sessionId: string, lines: string[], cwd = '/home/user/myproject') { + const sessionDir = join(tmpDir, sessionId) + await mkdir(sessionDir, { recursive: true }) + await writeFile(join(sessionDir, 'workspace.yaml'), `id: ${sessionId}\ncwd: ${cwd}\n`) + await writeFile(join(sessionDir, 'events.jsonl'), lines.join('\n') + '\n') + return join(sessionDir, 'events.jsonl') +} + +function modelChange(newModel: string, previousModel?: string) { + return JSON.stringify({ type: 'session.model_change', timestamp: '2026-04-15T10:00:01Z', data: { newModel, previousModel } }) +} + +function userMessage(content: string) { + return JSON.stringify({ type: 'user.message', timestamp: '2026-04-15T10:00:10Z', data: { content, interactionId: 'int-1' } }) +} + +function subagentSelected(agentName: string) { + return JSON.stringify({ type: 'subagent.selected', timestamp: '2026-04-15T10:00:12Z', data: { agentName } }) +} + +function assistantMessage(opts: { + messageId: string + outputTokens: number + tools?: string[] + toolArgs?: Record<string, unknown>[] + timestamp?: string +}) { + const toolRequests = (opts.tools ?? []).map((name, i) => ({ + name, + toolCallId: `call-${name}`, + type: 'function', + ...(opts.toolArgs?.[i] ? { arguments: opts.toolArgs[i] } : {}), + })) + return JSON.stringify({ + type: 'assistant.message', + timestamp: opts.timestamp ?? '2026-04-15T10:00:15Z', + data: { + messageId: opts.messageId, + outputTokens: opts.outputTokens, + interactionId: 'int-1', + toolRequests, + }, + }) +} + +function shutdownEvent(opts: { + modelMetrics: Record<string, { + inputTokens: number + outputTokens: number + cacheReadTokens: number + cacheWriteTokens: number + reasoningTokens?: number + }> + timestamp?: string +}) { + const modelMetrics: Record<string, unknown> = {} + for (const [model, u] of Object.entries(opts.modelMetrics)) { + modelMetrics[model] = { + requests: { count: 1, cost: 1 }, + usage: { + inputTokens: u.inputTokens, + outputTokens: u.outputTokens, + cacheReadTokens: u.cacheReadTokens, + cacheWriteTokens: u.cacheWriteTokens, + reasoningTokens: u.reasoningTokens ?? 0, + }, + } + } + return JSON.stringify({ + type: 'session.shutdown', + timestamp: opts.timestamp ?? '2026-04-15T10:05:00Z', + data: { shutdownType: 'routine', sessionStartTime: 1784102040274, modelMetrics }, + }) +} + +function transcriptSessionStart(sessionId: string) { + return JSON.stringify({ type: 'session.start', data: { sessionId, producer: 'copilot-agent' } }) +} + +function transcriptUserMessage(content: string) { + return JSON.stringify({ type: 'user.message', data: { content, attachments: [] } }) +} + +function transcriptAssistantMessage(opts: { + messageId: string + content?: string + reasoningText?: string + toolCallIds?: string[] + toolNames?: string[] +}) { + return JSON.stringify({ + type: 'assistant.message', + data: { + messageId: opts.messageId, + content: opts.content ?? '', + reasoningText: opts.reasoningText ?? '', + toolRequests: (opts.toolCallIds ?? []).map((id, i) => ({ + toolCallId: id, + name: opts.toolNames?.[i] ?? (i === 0 ? 'read_file' : 'run_in_terminal'), + type: 'function', + })), + }, + }) +} + +// ----------------------------------------------------------------------------- +// chatSessions fixture builders (copied from copilot.test.ts) +// ----------------------------------------------------------------------------- + +function chatSessionSampleRequest(overrides: Record<string, unknown> = {}) { + return { + requestId: 'request_8c8ce017-6e3f-460a-9931-5a16825d231a', + modelId: 'copilot/claude-sonnet-4.6', + completionTokens: 490, + result: { + metadata: { + promptTokens: 32543, + outputTokens: 60, + resolvedModel: 'claude-sonnet-4-6', + toolCallRounds: [{ thinking: { tokens: 0 }, modelId: 'claude-sonnet-4.6' }], + agentId: 'github.copilot.editsAgent', + }, + }, + ...overrides, + } +} + +async function createChatSessionFile(filePath: string, entries: unknown[]) { + await writeFile(filePath, entries.map(entry => JSON.stringify(entry)).join('\n') + '\n') +} + +// ----------------------------------------------------------------------------- +// OTel fixture builders (copied from copilot.test.ts) +// ----------------------------------------------------------------------------- + +interface SpanDef { + spanId: string + traceId: string + operationName: string + startTimeMs?: number + responseModel?: string + attrs: Record<string, string | number> +} + +function createOtelDb(dbPath: string): void { + const { DatabaseSync } = requireForTest('node:sqlite') as { DatabaseSync: new (path: string) => TestDb } + const db = new DatabaseSync(dbPath) + db.exec(` + CREATE TABLE spans ( + span_id TEXT PRIMARY KEY NOT NULL, + trace_id TEXT NOT NULL, + operation_name TEXT, + start_time_ms INTEGER NOT NULL DEFAULT 0, + response_model TEXT + ); + CREATE TABLE span_attributes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + span_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT + ); + `) + db.close() +} + +interface TestDb { + exec(sql: string): void + prepare(sql: string): { run(...params: unknown[]): void } + close(): void +} + +function insertSpan(dbPath: string, span: SpanDef): void { + const { DatabaseSync } = requireForTest('node:sqlite') as { DatabaseSync: new (path: string) => TestDb } + const db = new DatabaseSync(dbPath) + db.prepare( + `INSERT INTO spans (span_id, trace_id, operation_name, start_time_ms, response_model) + VALUES (?, ?, ?, ?, ?)` + ).run(span.spanId, span.traceId, span.operationName, span.startTimeMs ?? 0, span.responseModel ?? null) + const attrStmt = db.prepare( + `INSERT INTO span_attributes (span_id, key, value) VALUES (?, ?, ?)` + ) + for (const [key, value] of Object.entries(span.attrs)) { + attrStmt.run(span.spanId, key, String(value)) + } + db.close() +} + +// ----------------------------------------------------------------------------- +// JetBrains fixture builders (copied from copilot.test.ts) +// ----------------------------------------------------------------------------- + +function jbDbSource(path: string, sessionId: string, mtime = '2026-07-03T12:00:00.000Z') { + return { + path, + project: 'copilot-jetbrains', + provider: 'copilot', + sourceType: 'jetbrains', + sessionId, + storeId: sessionId, + dbPath: path, + mtime, + } as unknown as { path: string; project: string; provider: string; sourceType?: string } +} + +function jbAssistantBlob(text: string, opts: { model?: string; errored?: boolean; files?: string[] } = {}) { + const innerMd = { type: 'Markdown', data: JSON.stringify({ text, annotations: [] }) } + const valueMap: Record<string, unknown> = { + 'a1b2c3d4-0000-0000-0000-000000000001': { type: 'Value', value: JSON.stringify(innerMd) }, + } + if (opts.model) valueMap['__model__'] = { type: 'Value', value: `{"model":"${opts.model}"}` } + if (opts.files) { + valueMap['__refs__'] = { + type: 'Value', + value: JSON.stringify({ type: 'References', data: opts.files.map((f) => `file://${f}`).join(' ') }), + } + } + const outer: Record<string, unknown> = { + __first__: { type: 'Subgraph', value: JSON.stringify(valueMap) }, + } + if (opts.errored) { + outer['__err__'] = { + type: 'Value', + value: JSON.stringify({ type: 'Error', message: 'Sorry, an error occurred while generating a response' }), + } + } + return JSON.stringify(outer) +} + +function jbAgentBlob(rounds: string[], opts: { model?: string; userPrompt?: string; errored?: boolean } = {}) { + const valueMap: Record<string, unknown> = {} + let n = 0 + if (opts.userPrompt !== undefined) { + const md = { type: 'Markdown', data: JSON.stringify({ text: opts.userPrompt, annotations: [] }) } + valueMap[`u0000000-0000-0000-0000-00000000000${n++}`] = { type: 'Value', value: JSON.stringify(md) } + } + for (const reply of rounds) { + const ar = { type: 'AgentRound', data: JSON.stringify({ roundId: n, reply, toolCalls: [] }) } + valueMap[`a0000000-0000-0000-0000-00000000000${n++}`] = { type: 'Value', value: JSON.stringify(ar) } + } + if (opts.model) valueMap['__model__'] = { type: 'Value', value: `{"model":"${opts.model}"}` } + const outer: Record<string, unknown> = { __first__: { type: 'Subgraph', value: JSON.stringify(valueMap) } } + if (opts.errored) { + outer['__err__'] = { + type: 'Value', + value: JSON.stringify({ type: 'Error', message: 'Sorry, an error occurred while generating a response' }), + } + } + return JSON.stringify(outer) +} + +function jbConversationRecord(guid: string, title: string) { + return `$${guid}t\x00\x04namesq\x00\x01?@\x00\x00w\x00\x00t\x00value t\x00${title}t\x00\x06sourcet\x00copilotx` +} + +function jbDbContent(blobs: string[], conversations: string[] = []) { + return ( + 'H:2,block:9,blockSize:1000,format:3\n' + + 'com.github.copilot.agent.session.persistence.nitrite.entity.NtAgentTurn\n' + + conversations.join('\n') + '\n' + + blobs.join('\nt\x00\x00model\n') + + '\n' + ) +} + +async function createJetBrainsDb(root: string, ide: string, kind: string, storeId: string, content: string) { + const dir = join(root, ide, kind, storeId) + await mkdir(dir, { recursive: true }) + const dbName = + kind === 'chat-agent-sessions' + ? 'copilot-agent-sessions-nitrite.db' + : kind === 'chat-edit-sessions' + ? 'copilot-edit-sessions-nitrite.db' + : 'copilot-chat-nitrite.db' + await writeFile(join(dir, dbName), content) + return join(dir, dbName) +} + +function jbProjectNameField(name: string) { + const len = Buffer.byteLength(name, 'utf8') + const hi = String.fromCharCode((len >> 8) & 0xff) + const lo = String.fromCharCode(len & 0xff) + return `t\x00\x0bprojectName\x74${hi}${lo}${name}t\x00\x04usert\x00\x08dev-user` +} + +function jbOldFormatDoc(rounds: Array<{ reply: string; model?: string }>, opts: { upperUuid?: boolean } = {}) { + const cased = (u: string) => (opts.upperUuid ? u.toUpperCase() : u) + const entries: Record<string, unknown> = {} + entries[cased('0f383f5c-f169-4fee-9115-c06d4dd8985f')] = { + type: 'Value', + value: JSON.stringify({ type: 'References', data: '[]' }), + } + rounds.forEach((r, i) => { + const uuid = cased(`ccadf30b-fa34-4387-9f14-0a5f63457d${String(i).padStart(2, '0')}`) + const agentRoundData = JSON.stringify({ roundId: i + 1, reply: r.reply, toolCalls: [] }) + const agentRoundValue = JSON.stringify({ type: 'AgentRound', data: agentRoundData }) + entries[uuid] = { type: 'Value', value: agentRoundValue } + if (r.model) { + const modelUuid = cased(`bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbb${String(i).padStart(4, '0')}`) + entries[modelUuid] = { type: 'Value', value: `{"model":"${r.model}"}` } + } + }) + return '\x81' + JSON.stringify(entries) +} + +// ----------------------------------------------------------------------------- +// Capture helper +// ----------------------------------------------------------------------------- + +async function capture(source: Record<string, unknown>) { + const provider = createCopilotProvider('/nonexistent/jsonl', '/nonexistent/ws') + const seen = new Set<string>() + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(source as never, seen).parse()) { + calls.push(call) + } + return { calls, seen } +} + +// ----------------------------------------------------------------------------- +// Captured goldens (from the unmodified provider) +// ----------------------------------------------------------------------------- + +const G1_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-g1", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 150, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Read", + "mcp__github_mcp_server__list_issues" + ], + "bashCommands": [], + "subagentTypes": [ + "github.copilot.editsAgent" + ], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g1:msg-1", + "userMessage": "run the migration" + }, + { + "provider": "copilot", + "sessionId": "sess-g1", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 80, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Bash", + "Skill" + ], + "bashCommands": [ + "ls" + ], + "skills": [ + "refactor" + ], + "subagentTypes": [ + "github.copilot.editsAgent" + ], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g1:msg-2", + "userMessage": "" + } +] + +const G1_KEYS = [ + "copilot:sess-g1:msg-1", + "copilot:sess-g1:msg-2" +] + +const G2_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-g2", + "model": "copilot-openai-auto", + "inputTokens": 0, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Read" + ], + "bashCommands": [], + "timestamp": "", + "speed": "standard", + "deduplicationKey": "copilot:sess-g2:msg-1", + "userMessage": "mixed transcript" + }, + { + "provider": "copilot", + "sessionId": "sess-g2", + "model": "copilot-openai-auto", + "inputTokens": 0, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Read" + ], + "bashCommands": [], + "timestamp": "", + "speed": "standard", + "deduplicationKey": "copilot:sess-g2:msg-2", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-g2", + "model": "copilot-openai-auto", + "inputTokens": 0, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Read", + "Shell" + ], + "bashCommands": [], + "timestamp": "", + "speed": "standard", + "deduplicationKey": "copilot:sess-g2:msg-3", + "userMessage": "" + } +] + +const G2_KEYS = [ + "copilot:sess-g2:msg-1", + "copilot:sess-g2:msg-2", + "copilot:sess-g2:msg-3" +] + +const G3_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "claude-sonnet-4-5", + "inputTokens": 0, + "outputTokens": 100, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:msg-1", + "userMessage": "first" + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "gpt-5", + "inputTokens": 0, + "outputTokens": 200, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:msg-2", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "claude-sonnet-4-5", + "inputTokens": 100, + "outputTokens": 0, + "cacheCreationInputTokens": 2000, + "cacheReadInputTokens": 8000, + "cachedInputTokens": 0, + "reasoningTokens": 15, + "webSearchRequests": 0, + "costUSD": 0.0102, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:05:00Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:shutdown:claude-sonnet-4-5", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "gpt-5", + "inputTokens": 50, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 5000, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costUSD": 0.0006875, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:05:00Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:shutdown:gpt-5", + "userMessage": "" + } +] + +const G3_KEYS = [ + "copilot:sess-g3:msg-1", + "copilot:sess-g3:msg-2", + "copilot:sess-g3:shutdown:claude-sonnet-4-5", + "copilot:sess-g3:shutdown:gpt-5" +] + +const G3_PRICED_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "claude-sonnet-4-5", + "inputTokens": 0, + "outputTokens": 100, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:msg-1", + "userMessage": "first", + "costUSD": 0.0015 + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "gpt-5", + "inputTokens": 0, + "outputTokens": 200, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:msg-2", + "userMessage": "", + "costUSD": 0.002 + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "claude-sonnet-4-5", + "inputTokens": 100, + "outputTokens": 0, + "cacheCreationInputTokens": 2000, + "cacheReadInputTokens": 8000, + "cachedInputTokens": 0, + "reasoningTokens": 15, + "webSearchRequests": 0, + "costUSD": 0.0102, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:05:00Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:shutdown:claude-sonnet-4-5", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-g3", + "model": "gpt-5", + "inputTokens": 50, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 5000, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costUSD": 0.0006875, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:05:00Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-g3:shutdown:gpt-5", + "userMessage": "" + } +] + +const G4_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "g4-session", + "project": "myproject", + "model": "claude-sonnet-4-6", + "inputTokens": 32543, + "outputTokens": 60, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-05-30T16:05:13.100Z", + "speed": "standard", + "deduplicationKey": "copilot-chatsession:g4-session:req-resolved", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "g4-session", + "project": "myproject", + "model": "gpt-4.1", + "inputTokens": 1200, + "outputTokens": 90, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-05-30T16:05:13.200Z", + "speed": "standard", + "deduplicationKey": "copilot-chatsession:g4-session:req-fallback", + "userMessage": "" + } +] + +const G4_KEYS = [ + "copilot-chatsession:g4-session:req-fallback", + "copilot-chatsession:g4-session:req-resolved" +] + +const G5_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "shared-utils", + "model": "claude-opus-4-5", + "inputTokens": 0, + "outputTokens": 7, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:29c75429dae1:1", + "userMessage": "Conversation B" + }, + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "shared-utils", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 10, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:55e5aea23f97:1", + "userMessage": "Conversation B" + }, + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "shared-utils", + "model": "claude-opus-4-5", + "inputTokens": 0, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:e3b0c44298fc:1", + "userMessage": "Conversation B" + }, + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "shared-utils", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 9, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:eff208336025:1", + "userMessage": "Conversation B" + } +] + +const G5_KEYS = [ + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:29c75429dae1:1", + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:55e5aea23f97:1", + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:e3b0c44298fc:1", + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:eff208336025:1" +] + +const G6_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "17a5d71b-27f7-4937-8803-7fc2cbb705cb", + "project": "copilot-jetbrains", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 29, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:17a5d71b-27f7-4937-8803-7fc2cbb705cb:a4d4d9a6916b:1", + "userMessage": "Understanding HBase Architecture" + } +] + +const G6_KEYS = [ + "copilot:jb:17a5d71b-27f7-4937-8803-7fc2cbb705cb:a4d4d9a6916b:1" +] + +const G7_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "conv-g7", + "project": "copilot-chat", + "model": "gpt-4.1", + "inputTokens": 1200, + "outputTokens": 150, + "cacheCreationInputTokens": 300, + "cacheReadInputTokens": 30000, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Shell" + ], + "bashCommands": [ + "echo", + "git", + "npm" + ], + "timestamp": "1970-01-01T00:16:40.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-g7-chat-1", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "conv-g7", + "project": "copilot-chat", + "model": "claude-haiku-4.5", + "inputTokens": 400, + "outputTokens": 50, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "subagentTypes": [ + "Explore" + ], + "timestamp": "1970-01-01T00:35:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-g7-sub-chat", + "userMessage": "" + } +] + +const G7_KEYS = [ + "copilot-otel:span-g7-chat-1", + "copilot-otel:span-g7-sub-chat", + "copilot:conv-g7:turn-g7-1" +] + +const G8_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "conv-g8-b", + "project": "copilot-chat", + "model": "claude-sonnet-4", + "inputTokens": 600, + "outputTokens": 120, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "1970-01-01T00:33:20.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-g8-b-chat", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "conv-g8-a", + "project": "repo", + "model": "gpt-4.1", + "inputTokens": 500, + "outputTokens": 100, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "1970-01-01T00:16:40.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-g8-a-chat", + "userMessage": "" + } +] + +const G8_KEYS = [ + "copilot-otel:span-g8-a-chat", + "copilot-otel:span-g8-b-chat" +] + +const G9_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-x1", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 42, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x1:msg-emit", + "userMessage": "keep me pending" + }, + { + "provider": "copilot", + "sessionId": "sess-x1", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 7, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x1:msg-after", + "userMessage": "" + } +] + +const G9_KEYS = [ + "copilot:sess-x1:msg-after", + "copilot:sess-x1:msg-emit" +] + +const G10_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-x2", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 10, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x2:msg-1", + "userMessage": "" + } +] + +const G10_KEYS = [ + "copilot:sess-x2:msg-1" +] + +const G11_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "sess-x4", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 5, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-04-15T10:00:15Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x4:msg-1", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-x4", + "model": "clamped", + "inputTokens": 0, + "outputTokens": 0, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 50, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costUSD": 0, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-15T07:54:00.274Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x4:shutdown:clamped", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "sess-x4", + "model": "gpt-4.1", + "inputTokens": 700, + "outputTokens": 0, + "cacheCreationInputTokens": 100, + "cacheReadInputTokens": 200, + "cachedInputTokens": 0, + "reasoningTokens": 7, + "webSearchRequests": 0, + "costUSD": 0.00175, + "costIsEstimated": false, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-15T07:54:00.274Z", + "speed": "standard", + "deduplicationKey": "copilot:sess-x4:shutdown:gpt-4.1", + "userMessage": "" + } +] + +const G11_KEYS = [ + "copilot:sess-x4:msg-1", + "copilot:sess-x4:shutdown:clamped", + "copilot:sess-x4:shutdown:gpt-4.1" +] + +const G12_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "x16-fallback", + "project": "myproject", + "model": "unknown", + "inputTokens": 10, + "outputTokens": 490, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [ + "Read", + "Bash", + "mcp__github_mcp_server__list_issues", + "Shell", + "Skill" + ], + "bashCommands": [], + "timestamp": "2026-05-01T08:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot-chatsession:x16-fallback:req-tools", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "x16-fallback", + "project": "myproject", + "model": "unknown", + "inputTokens": 3, + "outputTokens": 4, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-05-01T08:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot-chatsession:x16-fallback:req-unknown-model", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "x16-fallback", + "project": "myproject", + "model": "gpt-4.1", + "inputTokens": 5, + "outputTokens": 6, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "2026-05-01T08:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot-chatsession:x16-fallback:request-2", + "userMessage": "" + } +] + +const G12_KEYS = [ + "copilot-chatsession:x16-fallback:req-tools", + "copilot-chatsession:x16-fallback:req-unknown-model", + "copilot-chatsession:x16-fallback:request-2" +] + +const G13_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "web-api", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 9, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:eff208336025:1", + "userMessage": "Conversation X7" + }, + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "web-api", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 9, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:e0110dcd5a4e:1", + "userMessage": "Conversation X7" + } +] + +const G13_KEYS = [ + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:e0110dcd5a4e:1", + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:eff208336025:1" +] + +const G14_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "485825c0-3331-46a7-acb2-c71875ad6640", + "project": "pipe|repo", + "model": "gpt-4.1", + "inputTokens": 0, + "outputTokens": 8, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:03e1bde7d7c0:1", + "userMessage": "Conversation X8" + } +] + +const G14_KEYS = [ + "copilot:jb:485825c0-3331-46a7-acb2-c71875ad6640:03e1bde7d7c0:1" +] + +const G15_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "x17-store", + "project": "copilot-jetbrains", + "model": "copilot-anthropic-auto", + "inputTokens": 0, + "outputTokens": 5, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:x17-store:9829e901954d:1", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "x17-store", + "project": "copilot-jetbrains", + "model": "copilot-anthropic-auto", + "inputTokens": 0, + "outputTokens": 6, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "costIsEstimated": true, + "tools": [], + "bashCommands": [], + "timestamp": "2026-07-03T12:00:00.000Z", + "speed": "standard", + "deduplicationKey": "copilot:jb:x17-store:95b22f4dffb0:1", + "userMessage": "" + } +] + +const G15_KEYS = [ + "copilot:jb:x17-store:95b22f4dffb0:1", + "copilot:jb:x17-store:9829e901954d:1" +] + +const G16_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "conv-x11", + "project": "copilot-chat", + "model": "", + "inputTokens": 10, + "outputTokens": 5, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "1970-01-01T00:16:40.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-x11", + "userMessage": "" + } +] + +const G16_KEYS = [ + "copilot-otel:span-x11" +] + +const G17_GOLDEN: ParsedProviderCall[] = [ + { + "provider": "copilot", + "sessionId": "conv-x12-b", + "project": "copilot-chat", + "model": "gpt-4.1", + "inputTokens": 100, + "outputTokens": 20, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "1970-01-01T00:16:40.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-x12-shared", + "userMessage": "" + }, + { + "provider": "copilot", + "sessionId": "conv-x12-b", + "project": "copilot-chat", + "model": "gpt-5", + "inputTokens": 7, + "outputTokens": 3, + "cacheCreationInputTokens": 0, + "cacheReadInputTokens": 0, + "cachedInputTokens": 0, + "reasoningTokens": 0, + "webSearchRequests": 0, + "costBasis": "estimated", + "tools": [], + "bashCommands": [], + "timestamp": "1970-01-01T00:33:20.000Z", + "speed": "standard", + "deduplicationKey": "copilot-otel:span-x12-b-own", + "userMessage": "" + } +] + +const G17_KEYS = [ + "copilot-otel:span-x12-b-own", + "copilot-otel:span-x12-shared", + "copilot:conv-x12-b:turn-x12" +] + +// ----------------------------------------------------------------------------- +// Golden scenarios +// ----------------------------------------------------------------------------- + +describe('copilot bridge — fixture parity', () => { + it('G1 jsonl CLI format: model_change, user message, tools, bash chain, skill, subagent', async () => { + const eventsPath = await createSessionDir('sess-g1', [ + modelChange('gpt-4.1'), + userMessage('run the migration'), + subagentSelected('github.copilot.editsAgent'), + assistantMessage({ + messageId: 'msg-1', + outputTokens: 150, + tools: ['read_file', 'github-mcp-server-list_issues'], + }), + assistantMessage({ + messageId: 'msg-2', + outputTokens: 80, + tools: ['bash', 'skill'], + toolArgs: [ + { command: 'cd x && ls -la' }, + { skill: 'refactor' }, + ], + }), + ]) + const source = { path: eventsPath, project: 'myproject', provider: 'copilot' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G1_GOLDEN) + expect([...seen].sort()).toEqual(G1_KEYS) + }) + + it('G2 jsonl transcript: copilot-agent producer with mixed toolCallIds', async () => { + const eventsPath = await createSessionDir('sess-g2', [ + transcriptSessionStart('sess-g2'), + transcriptUserMessage('mixed transcript'), + transcriptAssistantMessage({ messageId: 'msg-1', content: 'one', toolCallIds: ['call_a'] }), + transcriptAssistantMessage({ messageId: 'msg-2', content: 'two', toolCallIds: ['tooluse_XY'] }), + transcriptAssistantMessage({ messageId: 'msg-3', content: 'three', toolCallIds: ['call_b', 'call_c'] }), + ]) + const source = { path: eventsPath, project: 'myproject', provider: 'copilot' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G2_GOLDEN) + expect([...seen].sort()).toEqual(G2_KEYS) + }) + + it('G3 jsonl shutdown rollup: two models, reasoning, cache-inclusive input', async () => { + const eventsPath = await createSessionDir('sess-g3', [ + modelChange('claude-sonnet-4-5'), + userMessage('first'), + assistantMessage({ messageId: 'msg-1', outputTokens: 100 }), + modelChange('gpt-5', 'claude-sonnet-4-5'), + assistantMessage({ messageId: 'msg-2', outputTokens: 200 }), + shutdownEvent({ + modelMetrics: { + 'claude-sonnet-4-5': { + inputTokens: 10100, + outputTokens: 100, + cacheReadTokens: 8000, + cacheWriteTokens: 2000, + reasoningTokens: 15, + }, + 'gpt-5': { + inputTokens: 5050, + outputTokens: 200, + cacheReadTokens: 5000, + cacheWriteTokens: 0, + reasoningTokens: 0, + }, + }, + }), + ]) + const source = { path: eventsPath, project: 'myproject', provider: 'copilot' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G3_GOLDEN) + expect([...seen].sort()).toEqual(G3_KEYS) + expect(calls.map(priceProviderCall)).toEqual(G3_PRICED_GOLDEN) + }) + + it('G4 chatsession: kind 0/1/2 journal, resolvedModel, fallback modelId, zero-token skip', async () => { + const filePath = join(tmpDir, 'g4.jsonl') + await createChatSessionFile(filePath, [ + { kind: 0, v: { version: 3, creationDate: 1780157113020, sessionId: 'g4-session', requests: [] } }, + { + kind: 1, + k: ['requests'], + v: [ + chatSessionSampleRequest({ + requestId: 'req-resolved', + timestamp: 1780157113100, + }), + ], + }, + { + kind: 2, + k: ['requests'], + v: [ + chatSessionSampleRequest({ + requestId: 'req-fallback', + modelId: 'copilot/gpt-4.1', + timestamp: 1780157113200, + result: { metadata: { promptTokens: 1200, outputTokens: 90 } }, + }), + chatSessionSampleRequest({ + requestId: 'req-zero', + modelId: 'copilot/gpt-4.1', + timestamp: 1780157113300, + completionTokens: 0, + result: { metadata: { promptTokens: 0, outputTokens: 0 } }, + }), + ], + }, + ]) + const source = { path: filePath, project: 'myproject', provider: 'copilot', sourceType: 'chatsession' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G4_GOLDEN) + expect([...seen].sort()).toEqual(G4_KEYS) + }) + + it('G5 jetbrains new format: ask, agent decoy, errored, two conversations, file:// repo, projectName', async () => { + const repoDir = join(tmpDir, 'container', 'web-api') + await mkdir(join(repoDir, '.git'), { recursive: true }) + const repoFile = join(repoDir, 'src', 'Main.java') + + const guidA = '6acf5299-f9f7-404f-812d-dbe8300e1e5b' + const guidB = '485825c0-3331-46a7-acb2-c71875ad6640' + + const content = jbDbContent( + [ + jbProjectNameField('shared-utils'), + jbAssistantBlob('Ask-mode answer in markdown.', { model: 'claude-opus-4.5' }), + jbAgentBlob(['AgentRound reply, not the user prompt.'], { model: 'gpt-4.1', userPrompt: 'summarise this repo' }), + jbAssistantBlob('', { errored: true }), + jbAssistantBlob('Answer referencing a real repo file.', { model: 'gpt-4.1', files: [repoFile] }), + ], + [ + jbConversationRecord(guidA, 'Conversation A'), + jbConversationRecord(guidB, 'Conversation B'), + ] + ) + const dbPath = await createJetBrainsDb(tmpDir, 'iu', 'chat-agent-sessions', 'g5-store', content) + const { calls, seen } = await capture({ ...jbDbSource(dbPath, 'g5-store'), projectName: 'shared-utils' }) + expect(calls).toEqual(G5_GOLDEN) + expect([...seen].sort()).toEqual(G5_KEYS) + }) + + it('G6 jetbrains old format (<=1.5.x): outer UUID-keyed Value document with AgentRound, uppercase UUIDs', async () => { + const convGuid = '17a5d71b-27f7-4937-8803-7fc2cbb705cb' + const oldFormatContent = + 'H:2,block:8,blockSize:1000,format:3\n' + + 'com.github.copilot.agent.session.persistence.nitrite.entity.NtAgentTurn\n' + + jbConversationRecord(convGuid, 'Understanding HBase Architecture') + '\n' + + jbOldFormatDoc( + [ + { reply: "I'll scan the repository to find the top-level project structure.", model: 'gpt-4.1' }, + { reply: "Now I'll open the README to explain architecture." }, + { reply: '' }, + ], + { upperUuid: true } + ) + const dbPath = await createJetBrainsDb(tmpDir, 'iu', 'chat-agent-sessions', 'g6-store', oldFormatContent) + const { calls, seen } = await capture(jbDbSource(dbPath, 'g6-store')) + expect(calls).toEqual(G6_GOLDEN) + expect([...seen].sort()).toEqual(G6_KEYS) + }) + + it('G7 otel single conversation: tokens, multi-line shell, subagent, root agent, zero-token skip', async () => { + if (!isSqliteAvailable()) return + vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '') + const dbPath = join(tmpDir, 'agent-traces.db') + vi.stubEnv('CODEBURN_COPILOT_OTEL_DB', dbPath) + createOtelDb(dbPath) + + insertSpan(dbPath, { + spanId: 'span-g7-chat-1', + traceId: 'trace-g7', + operationName: 'chat', + startTimeMs: 1000, + responseModel: 'gpt-4.1', + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.response.model': 'gpt-4.1', + 'gen_ai.usage.input_tokens': 1200, + 'gen_ai.usage.output_tokens': 150, + 'gen_ai.usage.cache_read.input_tokens': 30000, + 'gen_ai.usage.cache_creation.input_tokens': 300, + 'github.copilot.chat.turn.id': 'turn-g7-1', + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g7-tool', + traceId: 'trace-g7', + operationName: 'execute_tool', + startTimeMs: 1500, + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.tool.name': 'runInTerminal', + 'gen_ai.tool.call.arguments': JSON.stringify({ + command: 'for f in *.ts; do\n echo "$f"\ndone\ngit status\nnpm test', + }), + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g7-subagent', + traceId: 'trace-g7-sub', + operationName: 'invoke_agent', + startTimeMs: 2000, + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.agent.name': 'Explore', + 'copilot_chat.parent_chat_session_id': 'conv-g7', + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g7-sub-chat', + traceId: 'trace-g7-sub', + operationName: 'chat', + startTimeMs: 2100, + responseModel: 'claude-haiku-4.5', + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.response.model': 'claude-haiku-4.5', + 'gen_ai.usage.input_tokens': 400, + 'gen_ai.usage.output_tokens': 50, + 'gen_ai.usage.cache_read.input_tokens': 0, + 'gen_ai.usage.cache_creation.input_tokens': 0, + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g7-root-agent', + traceId: 'trace-g7-root', + operationName: 'invoke_agent', + startTimeMs: 3000, + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.agent.name': 'GitHub Copilot Chat', + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g7-zero', + traceId: 'trace-g7-zero', + operationName: 'chat', + startTimeMs: 4000, + responseModel: 'gpt-4.1', + attrs: { + 'gen_ai.conversation.id': 'conv-g7', + 'gen_ai.response.model': 'gpt-4.1', + 'gen_ai.usage.input_tokens': 0, + 'gen_ai.usage.output_tokens': 0, + 'gen_ai.usage.cache_read.input_tokens': 0, + 'gen_ai.usage.cache_creation.input_tokens': 0, + }, + }) + + const source = { path: dbPath, project: 'copilot-chat', provider: 'copilot', sourceType: 'otel' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G7_GOLDEN) + expect([...seen].sort()).toEqual(G7_KEYS) + }) + + it('G8 otel two conversations in one DB: one with github.copilot.git.repository, one without', async () => { + if (!isSqliteAvailable()) return + vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '') + const dbPath = join(tmpDir, 'agent-traces.db') + vi.stubEnv('CODEBURN_COPILOT_OTEL_DB', dbPath) + createOtelDb(dbPath) + + insertSpan(dbPath, { + spanId: 'span-g8-a-chat', + traceId: 'trace-g8-a', + operationName: 'chat', + startTimeMs: 1000, + responseModel: 'gpt-4.1', + attrs: { + 'gen_ai.conversation.id': 'conv-g8-a', + 'gen_ai.response.model': 'gpt-4.1', + 'gen_ai.usage.input_tokens': 500, + 'gen_ai.usage.output_tokens': 100, + 'github.copilot.git.repository': 'file:///Users/dev/projects/web-api/repo.git', + }, + }) + + insertSpan(dbPath, { + spanId: 'span-g8-b-chat', + traceId: 'trace-g8-b', + operationName: 'chat', + startTimeMs: 2000, + responseModel: 'claude-sonnet-4', + attrs: { + 'gen_ai.conversation.id': 'conv-g8-b', + 'gen_ai.response.model': 'claude-sonnet-4', + 'gen_ai.usage.input_tokens': 600, + 'gen_ai.usage.output_tokens': 120, + }, + }) + + const source = { path: dbPath, project: 'copilot-chat', provider: 'copilot', sourceType: 'otel' } + const { calls, seen } = await capture(source) + expect(calls).toEqual(G8_GOLDEN) + expect([...seen].sort()).toEqual(G8_KEYS) + }) + + it('G9 jsonl: pendingUserMessage survives a skipped assistant message (J31)', async () => { + const eventsPath = await createSessionDir('sess-x1', [ + modelChange('gpt-4.1'), + userMessage('keep me pending'), + assistantMessage({ messageId: 'msg-skip', outputTokens: 0 }), + assistantMessage({ messageId: 'msg-emit', outputTokens: 42 }), + assistantMessage({ messageId: 'msg-after', outputTokens: 7 }), + ]) + const { calls, seen } = await capture({ path: eventsPath, project: 'myproject', provider: 'copilot' }) + expect(calls).toEqual(G9_GOLDEN) + expect([...seen].sort()).toEqual(G9_KEYS) + }) + + it('G10 jsonl: an empty-string newModel wins under ?? and blanks the model (J10/J23)', async () => { + const eventsPath = await createSessionDir('sess-x2', [ + modelChange('gpt-4.1'), + assistantMessage({ messageId: 'msg-1', outputTokens: 10 }), + JSON.stringify({ type: 'session.model_change', timestamp: '2026-04-15T10:00:02Z', data: { newModel: '' } }), + assistantMessage({ messageId: 'msg-2', outputTokens: 20 }), + JSON.stringify({ type: 'session.model_change', timestamp: '2026-04-15T10:00:03Z', data: {} }), + assistantMessage({ messageId: 'msg-3', outputTokens: 30 }), + ]) + const { calls, seen } = await capture({ path: eventsPath, project: 'myproject', provider: 'copilot' }) + expect(calls).toEqual(G10_GOLDEN) + expect([...seen].sort()).toEqual(G10_KEYS) + }) + + it('G11 jsonl shutdown: skip arms, cache clamp, sessionStartTime timestamp (J14-J18)', async () => { + const eventsPath = await createSessionDir('sess-x4', [ + modelChange('gpt-4.1'), + assistantMessage({ messageId: 'msg-1', outputTokens: 5 }), + JSON.stringify({ type: 'session.shutdown', timestamp: '', data: { sessionStartTime: 1784102040274, modelMetrics: { + '': { usage: { inputTokens: 10 } }, + 'no-usage': {}, + 'bad-usage': { usage: 'nope' }, + 'all-zero': { usage: { inputTokens: 0, outputTokens: 900 } }, + 'clamped': { usage: { inputTokens: 10, cacheReadTokens: 50, cacheWriteTokens: 0 } }, + 'gpt-4.1': { usage: { inputTokens: 1000, cacheReadTokens: 200, cacheWriteTokens: 100, reasoningTokens: 7 } }, + } } }), + ]) + const { calls, seen } = await capture({ path: eventsPath, project: 'myproject', provider: 'copilot' }) + expect(calls).toEqual(G11_GOLDEN) + expect([...seen].sort()).toEqual(G11_KEYS) + // The shutdown arm prices itself; the pricing pass must never see a costBasis. + for (const call of calls.filter(c => c.deduplicationKey.includes(':shutdown:'))) { + expect(Object.hasOwn(call, 'costBasis')).toBe(false) + expect(call.costIsEstimated).toBe(false) + } + }) + + it('G12 chatsession: id/model/timestamp fallbacks, completionTokens fallback, tool rounds', async () => { + const filePath = join(tmpDir, 'x16-fallback.jsonl') + await createChatSessionFile(filePath, [ + { kind: 0, v: { version: 3, creationDate: '2026-05-01T08:00:00.000Z', requests: [] } }, + { kind: 2, k: ['requests'], v: [ + { requestId: 'req-tools', completionTokens: 490, result: { metadata: { promptTokens: 10, outputTokens: 0, toolCallRounds: [ + { toolName: 'read_file', tools: ['bash', 'read_file'], toolCalls: [{ name: 'github-mcp-server-list_issues' }, 7], toolRequests: [{ tool: ' ' }, { toolName: 'runInTerminal' }] }, + 'not-a-round', + { name: 'skill' }, + ] } } }, + { requestId: 'req-unknown-model', result: { metadata: { promptTokens: 3, outputTokens: 4 } } }, + { modelId: 'copilot/gpt-4.1', timestamp: 'not-a-date', result: { metadata: { promptTokens: 5, outputTokens: 6 } } }, + ] }, + ]) + const { calls, seen } = await capture({ path: filePath, project: 'myproject', provider: 'copilot', sourceType: 'chatsession' }) + expect(calls).toEqual(G12_GOLDEN) + expect([...seen].sort()).toEqual(G12_KEYS) + }) + + it('G13 jetbrains: no projectName, project comes from the file:// git repo root', async () => { + const repoDir = join(tmpDir, 'container', 'web-api') + await mkdir(join(repoDir, '.git'), { recursive: true }) + const repoFile = join(repoDir, 'src', 'Main.java') + const guid = '485825c0-3331-46a7-acb2-c71875ad6640' + const content = jbDbContent([ + jbAssistantBlob('Answer referencing a real repo file.', { model: 'gpt-4.1', files: [repoFile] }), + jbAssistantBlob('Second turn with no file references.', { model: 'gpt-4.1' }), + ], [jbConversationRecord(guid, 'Conversation X7')]) + const dbPath = await createJetBrainsDb(tmpDir, 'iu', 'chat-agent-sessions', 'x7-store', content) + const { calls, seen } = await capture(jbDbSource(dbPath, 'x7-store')) + expect(calls).toEqual(G13_GOLDEN) + expect([...seen].sort()).toEqual(G13_KEYS) + // Back-filled from the first turn (B12) via the host-resolved repo-root map. + expect(calls.map(c => c.project)).toEqual(['web-api', 'web-api']) + }) + + it('G14 jetbrains: a pipe character inside a file:// path still resolves (B22)', async () => { + const repoDir = join(tmpDir, 'pipe|repo') + await mkdir(join(repoDir, '.git'), { recursive: true }) + const repoFile = join(repoDir, 'src', 'Main.java') + const guid = '485825c0-3331-46a7-acb2-c71875ad6640' + const content = jbDbContent([ + jbAssistantBlob('Answer referencing a piped path.', { model: 'gpt-4.1', files: [repoFile] }), + ], [jbConversationRecord(guid, 'Conversation X8')]) + const dbPath = await createJetBrainsDb(tmpDir, 'iu', 'chat-agent-sessions', 'x8-store', content) + const { calls, seen } = await capture(jbDbSource(dbPath, 'x8-store')) + expect(calls).toEqual(G14_GOLDEN) + expect([...seen].sort()).toEqual(G14_KEYS) + }) + + it('G15 jetbrains: no conversation record, repeated reply, empty blob, model fallback', async () => { + const content = jbDbContent([ + jbAssistantBlob('Repeated reply body.'), + jbAssistantBlob('Repeated reply body.'), + jbAssistantBlob(''), + jbAssistantBlob('Distinct second reply.'), + ]) + const dbPath = await createJetBrainsDb(tmpDir, 'iu', 'chat-agent-sessions', 'x17-store', content) + const { calls, seen } = await capture(jbDbSource(dbPath, 'x17-store')) + expect(calls).toEqual(G15_GOLDEN) + expect([...seen].sort()).toEqual(G15_KEYS) + }) + + it('G16 otel: an empty-string response_model wins under ?? (O13)', async () => { + if (!isSqliteAvailable()) return + const dbPath = join(tmpDir, 'agent-traces.db') + createOtelDb(dbPath) + insertSpan(dbPath, { + spanId: 'span-x11', traceId: 'trace-x11', operationName: 'chat', startTimeMs: 1000, responseModel: '', + attrs: { 'gen_ai.conversation.id': 'conv-x11', 'gen_ai.usage.input_tokens': 10, 'gen_ai.usage.output_tokens': 5 }, + }) + const { calls, seen } = await capture({ path: dbPath, project: 'copilot-chat', provider: 'copilot', sourceType: 'otel' }) + expect(calls).toEqual(G16_GOLDEN) + expect([...seen].sort()).toEqual(G16_KEYS) + expect(calls[0]!.model).toBe('') + }) + + it('G17 otel: one span shared by two conversations is emitted once (O21)', async () => { + if (!isSqliteAvailable()) return + const dbPath = join(tmpDir, 'agent-traces.db') + createOtelDb(dbPath) + insertSpan(dbPath, { + spanId: 'span-x12-shared', traceId: 'trace-x12', operationName: 'chat', startTimeMs: 1000, responseModel: 'gpt-4.1', + attrs: { 'gen_ai.conversation.id': 'conv-x12-a', 'gen_ai.usage.input_tokens': 100, 'gen_ai.usage.output_tokens': 20, 'github.copilot.chat.turn.id': 'turn-x12' }, + }) + const { DatabaseSync } = requireForTest('node:sqlite') as { DatabaseSync: new (p: string) => TestDb } + const raw = new DatabaseSync(dbPath) + raw.prepare(`INSERT INTO span_attributes (span_id, key, value) VALUES (?, ?, ?)`).run('span-x12-shared', 'gen_ai.conversation.id', 'conv-x12-b') + raw.close() + insertSpan(dbPath, { + spanId: 'span-x12-b-own', traceId: 'trace-x12', operationName: 'chat', startTimeMs: 2000, responseModel: 'gpt-5', + attrs: { 'gen_ai.conversation.id': 'conv-x12-b', 'gen_ai.usage.input_tokens': 7, 'gen_ai.usage.output_tokens': 3 }, + }) + const { calls, seen } = await capture({ path: dbPath, project: 'copilot-chat', provider: 'copilot', sourceType: 'otel' }) + expect(calls).toEqual(G17_GOLDEN) + expect([...seen].sort()).toEqual(G17_KEYS) + }) + +}) diff --git a/packages/core/package.json b/packages/core/package.json index 650ce289..9d8719a3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -138,6 +138,10 @@ "./providers/devin": { "types": "./dist/providers/devin/index.d.ts", "import": "./dist/providers/devin/index.js" + }, + "./providers/copilot": { + "types": "./dist/providers/copilot/index.d.ts", + "import": "./dist/providers/copilot/index.js" } }, "files": [ diff --git a/packages/core/src/providers/copilot/decode.ts b/packages/core/src/providers/copilot/decode.ts new file mode 100644 index 00000000..be889de0 --- /dev/null +++ b/packages/core/src/providers/copilot/decode.ts @@ -0,0 +1,1301 @@ +import { createHash } from 'crypto' +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { + AssistantMessageData, + ChatJournalPathSegment, + ChatSessionRequest, + CopilotCallArm, + CopilotDecodedCall, + CopilotEvent, + CopilotOtelConversationRecord, + CopilotOtelSpanRecord, + CopilotRecordEnvelope, + JBConversation, + JBDbTurn, + ModelChangeData, + SessionShutdownData, + SessionStartData, + SpanAttributes, + SubagentSelectedData, + ToolRequest, + UserMessageData, +} from './types.js' + +// --------------------------------------------------------------------------- +// Tool name normalisation (unchanged from original, plus OTel tool names) +// --------------------------------------------------------------------------- +export const copilotToolNameMap: Record<string, string> = { + // JSONL session-state tool names + bash: 'Bash', + skill: 'Skill', + read_file: 'Read', + write_file: 'Edit', + edit_file: 'Edit', + delete_file: 'Delete', + github_repo: 'GitHub', + web_search: 'WebSearch', + run_in_terminal: 'Shell', + // JetBrains Copilot agent tool names (snake_case) + insert_edit_into_file: 'Edit', + create_file: 'Edit', + get_errors: 'Diagnostics', + file_search: 'Search', + grep_search: 'Search', + semantic_search: 'Search', + list_dir: 'Search', + fetch_webpage: 'Web', + // OTel execute_tool span names from Copilot Chat: + readFile: 'Read', + writeFile: 'Edit', + editFile: 'Edit', + runCommand: 'Shell', + runInTerminal: 'Shell', + findFiles: 'Search', + grepSearch: 'Search', + codebaseSearch: 'Search', + getErrors: 'Diagnostics', + listCodeUsages: 'Search', + createFile: 'Edit', + deleteFile: 'Delete', + renameOrMoveFile: 'Edit', + fetchWebpage: 'Web', +} + +/** + * Normalise a raw tool name to its display form. + * - Known tools are mapped via toolNameMap. + * - MCP tools (containing both '-' and '_') are formatted as + * mcp__server_name__tool_name. + * - Everything else is returned unchanged. + */ +export function normalizeCopilotTool(rawTool: string): string { + const mapped = copilotToolNameMap[rawTool] + if (mapped) return mapped + // MCP tool names follow the pattern: server-name-tool_operand + // e.g. github-mcp-server-list_issues → mcp__github_mcp_server__list_issues + const dashIdx = rawTool.lastIndexOf('-') + if (dashIdx > 0 && rawTool.includes('_')) { + const server = rawTool.slice(0, dashIdx).replace(/-/g, '_') + const tool = rawTool.slice(dashIdx + 1) + return `mcp__${server}__${tool}` + } + return rawTool +} + +// Tool names that represent shell/bash execution. When the AI calls one of +// these, we extract the `arguments.command` string into rawBashCommands[]. +const BASH_TOOL_NAMES = new Set(['bash', 'run_in_terminal', 'runInTerminal', 'runCommand']) + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Convert nanosecond or millisecond epoch to ISO timestamp. + * The OTel spec uses nanoseconds, but some implementations use milliseconds. + */ +function epochToISO(epoch: number): string { + // Guard malformed rows: new Date(NaN).toISOString() throws. Fall back to the + // epoch (1970) so a bad timestamp is excluded from period totals, not crashing. + if (!Number.isFinite(epoch) || epoch <= 0) return new Date(0).toISOString() + // If the value looks like nanoseconds (> 1e15), convert to ms + const ms = epoch > 1e15 ? Math.floor(epoch / 1e6) : epoch > 1e12 ? epoch : epoch * 1000 + return new Date(ms).toISOString() +} + +function timestampToISO(raw: unknown): string { + if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { + return epochToISO(raw) + } + if (typeof raw !== 'string') return '' + const trimmed = raw.trim() + if (!trimmed) return '' + if (/^\d+(\.\d+)?$/.test(trimmed)) { + return epochToISO(Number(trimmed)) + } + const parsed = Date.parse(trimmed) + return Number.isNaN(parsed) ? '' : new Date(parsed).toISOString() +} + +function isRecord(value: unknown): value is Record<string, unknown> { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isReplayContainer(value: unknown): value is object { + return typeof value === 'object' && value !== null +} + +function createReplayObject(): Record<string, unknown> { + return Object.create(null) as Record<string, unknown> +} + +const FORBIDDEN_CHAT_JOURNAL_KEYS = new Set(['__proto__', 'prototype', 'constructor']) + +function parseChatJournalPath(rawPath: unknown, fallback?: ChatJournalPathSegment[]): ChatJournalPathSegment[] | null { + const value = rawPath === undefined ? fallback : rawPath + if (!Array.isArray(value)) return null + + const path: ChatJournalPathSegment[] = [] + for (const segment of value) { + if (typeof segment === 'number') { + if (!Number.isInteger(segment) || segment < 0) return null + path.push(segment) + continue + } + if (typeof segment === 'string') { + if (FORBIDDEN_CHAT_JOURNAL_KEYS.has(segment)) return null + path.push(segment) + continue + } + return null + } + return path +} + +function getReplayValue(container: object, segment: ChatJournalPathSegment): unknown { + return (container as Record<string, unknown>)[String(segment)] +} + +function setReplayValue(container: object, segment: ChatJournalPathSegment, value: unknown): void { + ;(container as Record<string, unknown>)[String(segment)] = value +} + +function createContainerForNext(segment: ChatJournalPathSegment): unknown[] | Record<string, unknown> { + return typeof segment === 'number' ? [] : createReplayObject() +} + +function ensureReplayParent(root: object, path: ChatJournalPathSegment[]): object | null { + let current: object = root + for (let i = 0; i < path.length - 1; i++) { + const segment = path[i]! + const nextSegment = path[i + 1]! + let child = getReplayValue(current, segment) + if (!isReplayContainer(child)) { + const created = createContainerForNext(nextSegment) + setReplayValue(current, segment, created) + current = created + continue + } + current = child + } + return current +} + +function applyChatJournalSet(root: unknown, path: ChatJournalPathSegment[], value: unknown): unknown { + if (path.length === 0) return value + + const workingRoot = isReplayContainer(root) ? root : createReplayObject() + const parent = ensureReplayParent(workingRoot, path) + if (!parent) return workingRoot + setReplayValue(parent, path[path.length - 1]!, value) + return workingRoot +} + +function applyChatJournalAppend(root: unknown, path: ChatJournalPathSegment[], items: unknown[]): unknown { + const workingRoot = isReplayContainer(root) ? root : createReplayObject() + + if (path.length === 0) { + if (Array.isArray(workingRoot)) { + for (const item of items) workingRoot.push(item) + } + return workingRoot + } + + const parent = ensureReplayParent(workingRoot, path) + if (!parent) return workingRoot + + const last = path[path.length - 1]! + let target = getReplayValue(parent, last) + const targetArray: unknown[] = Array.isArray(target) ? target : [] + if (target !== targetArray) { + setReplayValue(parent, last, targetArray) + } + for (const item of items) targetArray.push(item) + return workingRoot +} + +function replayChatSessionJournal(content: string): unknown { + let root: unknown = createReplayObject() + const lines = content.split('\n').filter((l) => l.trim()) + + for (const line of lines) { + let entry: unknown + try { + entry = JSON.parse(line) as unknown + } catch { + continue + } + if (!isRecord(entry)) continue + + const kind = entry['kind'] + if (kind === 0) { + root = entry['v'] + continue + } + + if (kind === 1) { + const path = parseChatJournalPath(entry['k']) + if (!path) continue + root = applyChatJournalSet(root, path, entry['v']) + continue + } + + if (kind === 2) { + const hasPath = Object.prototype.hasOwnProperty.call(entry, 'k') + const path = parseChatJournalPath(hasPath ? entry['k'] : undefined, ['requests']) + const items = Array.isArray(entry['v']) ? entry['v'] : [] + if (!path) continue + root = applyChatJournalAppend(root, path, items) + } + } + + return root +} + +function numberOrZero(raw: unknown): number { + return typeof raw === 'number' && Number.isFinite(raw) && raw > 0 ? raw : 0 +} + +function readString(raw: unknown): string { + return typeof raw === 'string' ? raw : '' +} + +function modelFromChatSessionRequest(req: ChatSessionRequest, metadata: Record<string, unknown>): string { + const resolved = readString(metadata['resolvedModel']) + if (resolved) return resolved + + const modelId = readString(req['modelId']).replace(/^copilot\//, '') + return modelId || 'unknown' +} + +function extractChatSessionTools(metadata: Record<string, unknown>): string[] { + const rounds = metadata['toolCallRounds'] + if (!Array.isArray(rounds)) return [] + + const names = new Set<string>() + const addName = (raw: unknown): void => { + if (typeof raw === 'string' && raw.trim()) names.add(normalizeCopilotTool(raw)) + } + const addFromRecord = (record: Record<string, unknown>): void => { + addName(record['toolName']) + addName(record['name']) + addName(record['tool']) + } + + for (const round of rounds) { + if (!isRecord(round)) continue + addFromRecord(round) + + for (const key of ['tools', 'toolCalls', 'toolRequests']) { + const entries = round[key] + if (!Array.isArray(entries)) continue + for (const entry of entries) { + if (typeof entry === 'string') { + addName(entry) + } else if (isRecord(entry)) { + addFromRecord(entry) + } + } + } + } + + return [...names] +} + +/** + * Extract a shell command string from an OTel execute_tool span's + * `gen_ai.tool.call.arguments` attribute. The attribute is a JSON-encoded + * argument object (e.g. `{"command":"ls -la"}`); we pull out the `command` + * field. Returns null when the attribute is absent or doesn't carry a command, + * so callers can skip shell-command extraction cleanly. + */ +function parseToolCommand(raw: unknown): string | null { + if (typeof raw !== 'string' || !raw.trim()) return null + try { + const parsed = JSON.parse(raw) as Record<string, unknown> + const command = parsed['command'] + return typeof command === 'string' ? command : null + } catch { + return null + } +} + +/** + * Safely coerce a raw toolRequests value to an array of ToolRequest. + * Non-array values (string, null, undefined) are treated as empty arrays + * so that a corrupt event.data doesn't abort the whole file parse loop. + */ +function coerceToolRequests(raw: unknown): ToolRequest[] { + return Array.isArray(raw) ? (raw as ToolRequest[]) : [] +} + +/** + * Infer the model bucket for a VS Code transcript file by counting the + * toolCallId prefixes across all assistant messages: + * call_* → OpenAI + * tooluse_* / toolu_* → Anthropic + * The dominant prefix determines the model for the whole session. + * Returns '' if no toolCallIds are present. + */ +function inferTranscriptModel(lines: string[]): string { + let openaiCount = 0 + let anthropicCount = 0 + + for (const line of lines) { + try { + const event = JSON.parse(line) as CopilotEvent + if (event.type !== 'assistant.message') continue + const data = event.data as AssistantMessageData & { toolRequests?: Array<{ toolCallId?: string }> } + const reqs = coerceToolRequests(data.toolRequests) + for (const req of reqs) { + const id = (req as { toolCallId?: unknown }).toolCallId + if (typeof id !== 'string') continue + if (id.startsWith('call_')) openaiCount++ + else if (/^tooluse_|^toolu_/.test(id)) anthropicCount++ + } + } catch { + continue + } + } + + if (openaiCount === 0 && anthropicCount === 0) return '' + return openaiCount >= anthropicCount ? 'copilot-openai-auto' : 'copilot-anthropic-auto' +} + +// --------------------------------------------------------------------------- +// JetBrains helpers +// --------------------------------------------------------------------------- + +// Known JetBrains Copilot model tokens, longest-first so we match the most +// specific name (e.g. "gpt-4.1-mini" before "gpt-4.1"). +const JETBRAINS_MODEL_TOKENS = [ + 'claude-opus-4.5', + 'claude-opus-4.1', + 'claude-opus-4', + 'claude-sonnet-4.5', + 'claude-sonnet-4', + 'gpt-5.3-codex', + 'gpt-5.3', + 'gpt-5.2', + 'gpt-5.1', + 'gpt-5-mini', + 'gpt-5', + 'gpt-4.1-mini', + 'gpt-4.1-nano', + 'gpt-4.1', + 'gpt-4o-mini', + 'gpt-4o', + 'gemini-2.5-pro', + 'gemini-2.0-flash', + 'o3-mini', + 'o4-mini', + 'o3', +] + +/** + * Normalise a raw JetBrains model token to CodeBurn's canonical model id. + * Claude names use dots on disk (claude-opus-4.5) but dashes in the pricing + * tables (claude-opus-4-5); GPT/Gemini names are kept verbatim. + */ +function normalizeJetBrainsModelName(raw: string): string { + const t = raw.trim() + if (!t) return '' + if (t.startsWith('claude-')) return t.replace(/\./g, '-') + return t +} + +/** Match a known model token at an alnum boundary anywhere in a string. */ +function findJetBrainsModelToken(s: string): string { + for (const token of JETBRAINS_MODEL_TOKENS) { + const escaped = token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // "o3" etc. must not match inside words like "iso3166". + if (new RegExp(`(?<![A-Za-z0-9])${escaped}(?![A-Za-z0-9])`).test(s)) { + return normalizeJetBrainsModelName(token) + } + } + return '' +} + +/** Recover a model from a raw buffer by scanning for a known token. */ +function inferJetBrainsModel(raw: string): string { + return findJetBrainsModelToken(raw) +} + +/** + * Every directory a `file://` reference in this store could resolve to. The host + * resolves each to a git repo root and hands the map back to the decoder, which + * consults it instead of touching the filesystem. + */ +export function collectJetBrainsRepoDirCandidates(raw: string): string[] { + const dirs = new Set<string>() + const TAIL = /^\/[^"\\]+?(?=\\|")/ + let from = 0 + for (;;) { + const at = raw.indexOf('file://', from) + if (at === -1) break + from = at + 7 + const m = TAIL.exec(raw.slice(from)) + if (!m) continue + let p = m[0] + try { p = decodeURIComponent(p) } catch { /* leave as-is */ } + const dir = p.slice(0, p.lastIndexOf('/')) + if (dir.startsWith('/')) dirs.add(dir) + } + return [...dirs] +} + +/** + * Infer the project (repository name) from the file:// URIs a chat referenced. + * + * The JetBrains store has no workspace/cwd record, and there is no reliable + * marker inside a path for where the repo root sits (users nest repos under + * arbitrary container dirs). So for each referenced file we walk UP the real + * filesystem to the nearest ancestor containing a `.git` entry and use that + * directory's basename — the true repo root. This is the one approach that + * yields a clean, consistent name (e.g. `my-service`) instead of a deep subdir + * or an inconsistent prose-scraped guess. + * + * Returns undefined when the chat referenced no files or none resolve to a repo + * that still exists on disk (caller then falls back to a generic bucket). + */ +function inferJetBrainsProject(raw: string, repoRootByDir: ReadonlyMap<string, string>): string | undefined { + // Capture referenced absolute paths (original case — we hit the real FS). + const re = /file:\/\/(\/[^"\\]+?)(?:\\|")/g + const seen = new Set<string>() + let m: RegExpExecArray | null + while ((m = re.exec(raw))) { + // Decode %20 etc. and strip a trailing .rej/.orig suffix noise; keep the dir. + let p = m[1] + try { p = decodeURIComponent(p) } catch { /* leave as-is */ } + const dir = p.slice(0, p.lastIndexOf('/')) + if (dir.startsWith('/')) seen.add(dir) + } + if (seen.size === 0) return undefined + + for (const dir of seen) { + const repo = repoRootByDir.get(dir) + if (repo) return repo + } + return undefined +} + +/** + * Recover the conversation (chat-tab) records from a raw .db buffer. Each is + * stored as `$<GUID> … name … value <title> … source copilot`. Returns the + * GUID→title map so turns can be grouped back to the tab the user sees. + */ +function extractJetBrainsConversations(raw: string): JBConversation[] { + // A conversation's title EVOLVES as the user chats: it starts as "New Agent + // Session", may pass through an auto-generated name, and ends at the final + // title shown in the UI. The same `$<GUID> … name … value <title> … source` + // record is rewritten each time, so we collect every occurrence per GUID and + // keep the LAST meaningful (non-default) one. + const DEFAULT_TITLES = new Set(['New Agent Session', 'New Session', 'New Chat']) + const byId = new Map<string, string>() + const re = /\$([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})[\s\S]{0,8}name/g + let m: RegExpExecArray | null + while ((m = re.exec(raw))) { + const id = m[1] + const window = raw.slice(m.index, m.index + 400) + // The title is the Java-UTF string between the `value` marker and `source`. + const tm = window.match(/value.{1,6}?([\x20-\x7e]{3,80}?)t\x00\x06source/) + if (!tm) continue + const title = Buffer.from(tm[1].replace(/^[^A-Za-z0-9]*/, ''), 'latin1').toString('utf8').trim() + if (!title) continue + // Keep the latest non-default title; only fall back to a default if no + // meaningful title has been seen for this conversation yet. + const existing = byId.get(id) + if (existing && !DEFAULT_TITLES.has(existing) && DEFAULT_TITLES.has(title)) continue + byId.set(id, title) + } + return [...byId.entries()].map(([id, title]) => ({ id, title })) +} + +/** Brace-match a JSON object starting at `start`, tolerating escaped quotes. */ +function matchJsonObject(raw: string, start: number): { chunk: string; end: number } { + let depth = 0 + let inStr = false + let esc = false + let i = start + for (; i < raw.length; i++) { + const c = raw[i] + if (esc) { esc = false; continue } + if (c === '\\') { esc = true; continue } + if (c === '"') { inStr = !inStr; continue } + if (inStr) continue + if (c === '{') depth++ + else if (c === '}') { depth--; if (depth === 0) { i++; break } } + } + return { chunk: raw.slice(start, i), end: i } +} + +/** + * Recover the assistant reply text from a `__first__`/Subgraph response blob. + * + * JetBrains Copilot has two turn shapes, both handled here: + * + * - **Ask mode:** the reply is a `Markdown` record whose `data` is an escaped + * JSON document `{"text":"…","annotations":…}`. + * - **Agent mode** (e.g. PyCharm agent sessions): the reply is the `reply` + * field of an `AgentRound` record `{"roundId":N,"reply":"…","toolCalls":[…]}`. + * In agent mode the `Markdown` records hold the USER's prompts, not the + * reply, so we must NOT read them — the assistant output is the AgentRound + * reply. + * + * Both are read STRUCTURALLY rather than by fully unescaping the blob (which + * would strip the reply's own quotes and make regex extraction ambiguous): we + * locate each `data`/`reply` value, read it as a properly-delimited JSON-string + * literal (honouring escaping), unescape one level, and `JSON.parse` to reach + * the text. We unescape the blob one level at a time and extract at the first + * depth that yields text, never accumulating across depths (which would union a + * quote-truncated half-unescaped capture with the full one and garble the + * reply, inflating the token/cost estimate). + * + * Steps/error/progress-only blobs (no Markdown text and no AgentRound reply) + * yield '' and are billed as $0 upstream. + */ +function extractResponseText(blob: string): string { + let s = blob + for (let depth = 0; depth < 8; depth++) { + // Decide the mode by the PRESENCE of an AgentRound record, not by whether it + // yielded a reply. In agent mode the Markdown record holds the USER prompt, + // so an agent blob whose reply is empty (a failed turn, or a pure tool-call + // round) must NOT fall back to Markdown — that would bill the user's prompt + // as the assistant's output. Ask-mode blobs have no AgentRound record and + // use Markdown. (Verified across every observed store: the two reply shapes + // never coexist in one blob, so this mode split is unambiguous.) + const isAgentMode = /"type":"AgentRound"/.test(s) + if (isAgentMode || /"type":"Markdown"/.test(s)) { + const decoded = isAgentMode ? extractAgentRoundReplies(s) : extractMarkdownTexts(s) + // The .db is read as latin1 (byte-stable), so multibyte UTF-8 characters + // are split into separate code units. Re-interpret as UTF-8 so the char + // count (→ token estimate) reflects real content length, not byte count. + // decoded may be empty (failed/tool-only agent turn) → '' (billed $0). + return Buffer.from(decoded.join('\n').trim(), 'latin1').toString('utf8') + } + // Not yet at the depth where record markers appear bare — unescape one level + // in a single left-to-right pass so `\\` and `\"` resolve together (a + // two-pass replace would turn `\\"` into `\"` not `\\` + `"`). + const next = s.replace(/\\([\\"])/g, '$1') + if (next === s) break + s = next + } + return '' +} + +/** + * Collect the `text` of every `Markdown` record in `s`, treating each record's + * `data` value as a one-level-escaped JSON string parsed structurally (so the + * reply's own quotes never truncate it). Returns [] if `s` is not yet at the + * right unescape depth (no bare `"type":"Markdown"` with a parseable `data`). + * Scoping to Markdown skips `Error` (`message`) and `Steps` records — not + * billable output. Revisions repeat a reply, so identical texts are de-duped. + */ +function extractMarkdownTexts(s: string): string[] { + return extractRecordStrings(s, '"type":"Markdown"', '"data":"', 'text') +} + +/** + * Collect the non-empty `reply` of every `AgentRound` record (agent mode). A + * single blob can hold several rounds (a multi-turn agent session); each round's + * `reply` is the assistant's text for that step (empty on pure tool-call rounds). + * Deduped in order. + */ +function extractAgentRoundReplies(s: string): string[] { + return extractRecordStrings(s, '"type":"AgentRound"', '"data":"', 'reply') +} + +/** + * Shared structural reader: for every `<marker>` in `s`, find the following + * `<dataKey>` string literal (a one-level-escaped JSON document), parse it, and + * collect `doc[field]` when it is a non-empty string. Reading the value as a + * delimited literal — not a greedy regex — means the payload's own quotes never + * truncate it. Returns [] when `s` is not yet at the depth where the marker + * appears bare with a parseable payload. De-dupes in order (the store keeps + * byte-copies/revisions of each reply). + */ +function extractRecordStrings(s: string, marker: string, dataKey: string, field: string): string[] { + const texts: string[] = [] + const seen = new Set<string>() + const re = new RegExp(marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g') + let m: RegExpExecArray | null + while ((m = re.exec(s))) { + const dk = s.indexOf(dataKey, m.index) + if (dk === -1 || dk - m.index > 200) continue + // The value runs from after `<dataKey>` to the first UNescaped quote (an odd + // run of preceding backslashes escapes it). + const start = dk + dataKey.length + let i = start + for (; i < s.length; i++) { + if (s[i] !== '"') continue + let bs = 0 + for (let j = i - 1; j >= start && s[j] === '\\'; j--) bs++ + if (bs % 2 === 0) break + } + const literal = s.slice(start, i) + try { + // Wrapping in quotes + parsing unescapes exactly one level → the inner + // JSON document as a string; parsing THAT reaches { <field>, … }. + const doc = JSON.parse(JSON.parse('"' + literal + '"') as string) as Record<string, unknown> + const text = typeof doc[field] === 'string' ? (doc[field] as string) : '' + if (text && !seen.has(text)) { + seen.add(text) + texts.push(text) + } + } catch { + // Not the right depth (or not a matching record) — skip. + } + } + return texts +} + +const CHARS_PER_TOKEN = 4 +function estimateTokensFromChars(chars: number): number { + return Math.ceil(chars / CHARS_PER_TOKEN) +} + +/** + * Extract assistant turns from a raw (latin1) Nitrite .db buffer. Each turn is + * one `{"__first__":{"type":"Subgraph"…}` blob; the per-turn model is recovered + * from inside the blob when present, else the whole-store default. Each turn is + * grouped back to its owning conversation (chat tab) by the nearest preceding + * conversation GUID. Duplicate byte-copies of the same reply (the store keeps + * several) are de-duplicated by content, per conversation. + */ +function extractJetBrainsDbTurns(raw: string, repoRootByDir: ReadonlyMap<string, string>): JBDbTurn[] { + const conversations = extractJetBrainsConversations(raw) + // Precompute the byte offset of each conversation GUID's full form so a turn + // can be attributed to the conversation whose id most recently precedes it. + const convById = new Map(conversations.map((c) => [c.id, c])) + + const turns: JBDbTurn[] = [] + const seenReplies = new Set<string>() // keyed by `${conversationId}::${reply}` + const re = /\{"__first__":\{"type":"Subgraph"/g + let m: RegExpExecArray | null + while ((m = re.exec(raw))) { + const { chunk, end } = matchJsonObject(raw, m.index) + re.lastIndex = end + + // Attribute this turn to the conversation whose GUID last appears before it. + let conversationId = '' + let conversationTitle = '' + let bestPos = -1 + for (const c of convById.values()) { + const p = raw.lastIndexOf(c.id, m.index) + if (p > bestPos) { + bestPos = p + conversationId = c.id + conversationTitle = c.title + } + } + + const replyText = extractResponseText(chunk) + // The files this turn referenced (home-relative common dir) → project label. + const conversationProject = inferJetBrainsProject(chunk, repoRootByDir) ?? '' + // A per-turn model token sometimes appears inside the blob. + const model = findJetBrainsModelToken(chunk) + // A failed turn carries an error status / phrase AND produces no reply text. + // Requiring empty text avoids misclassifying a genuine reply that merely + // *discusses* an error (e.g. explaining a stack trace) as a failed turn. + const hasErrorMarker = /error occurred|"isError":true|\\+"status\\+":\\+"(?:error|failed)\\+"/i.test(chunk) + if (hasErrorMarker && !replyText) { + turns.push({ replyText: '', model, errored: true, conversationId, conversationTitle, conversationProject }) + continue + } + if (!replyText) continue // Steps/progress-only blob — no billable output + const dedupeKey = `${conversationId}::${replyText}` + if (seenReplies.has(dedupeKey)) continue + seenReplies.add(dedupeKey) + turns.push({ replyText, model, errored: false, conversationId, conversationTitle, conversationProject }) + } + + // --------------------------------------------------------------------------- + // Fallback: old JetBrains Copilot plugin format (≤1.5.x, e.g. 1.5.59-243) + // --------------------------------------------------------------------------- + // In this format ALL session turns are stored inside ONE large outer Nitrite + // document — a binary-framed JSON object with UUID-keyed Value entries — rather + // than the per-turn {"__first__":{"type":"Subgraph",...}} blobs used by newer + // plugins (≥1.12.x). The AgentRound entries sit one escaping level deeper + // inside the outer document's string values, so `extractResponseText`'s + // depth-unescape loop handles extraction correctly once we feed it the right + // chunk. MVStore keeps two identical copies of the collection; `seenReplies` + // deduplicates them automatically. + // + // Detection heuristic: the __first__/Subgraph path produced no turns AND the + // raw file contains bare 'AgentRound' text (meaning old-format data is present). + if (turns.length === 0 && raw.includes('AgentRound')) { + // The outer Nitrite document is preceded by a single binary framing byte + // (0x81 in practice, but any non-printable/non-ASCII byte in MVStore). + // It starts with a UUID-keyed Value entry: {"<uuid>":{"type":"Value",...}}. + // Hex is matched case-insensitively — an uppercase UUID must not cause the + // whole session to fall through to $0 (the exact bug this path fixes). + const outerDocRe = /[\x00-\x1f\x7f-\xff]\{"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}":\{"type":"Value"/g + let dm: RegExpExecArray | null + while ((dm = outerDocRe.exec(raw))) { + // Skip the leading binary byte; matchJsonObject starts at the '{'. + const docStart = dm.index + 1 + const { chunk, end } = matchJsonObject(raw, docStart) + outerDocRe.lastIndex = end + + // Skip documents that contain no AgentRound data (e.g. empty sessions). + if (!chunk.includes('AgentRound')) continue + + // Attribute to the conversation whose GUID most recently precedes this doc. + let conversationId = '' + let conversationTitle = '' + let bestPos = -1 + for (const c of convById.values()) { + const p = raw.lastIndexOf(c.id, docStart) + if (p > bestPos) { + bestPos = p + conversationId = c.id + conversationTitle = c.title + } + } + + // extractResponseText handles the depth-1 unescape needed to surface the + // AgentRound records, then calls extractAgentRoundReplies for each turn. + // Because the outer document holds ALL turns in one blob we get back a + // single joined string; split it on the '\n' join to yield per-turn texts. + const allReplies = extractResponseText(chunk) + if (!allReplies) continue + + const conversationProject = inferJetBrainsProject(chunk, repoRootByDir) ?? '' + const storeModel = findJetBrainsModelToken(chunk) + + // extractResponseText joins multiple replies with '\n'. Since individual + // replies can themselves span multiple lines we cannot cleanly split here — + // instead we emit one ParsedProviderCall per outer document (one session). + const dedupeKey = `${conversationId}::${allReplies}` + if (seenReplies.has(dedupeKey)) continue + seenReplies.add(dedupeKey) + + turns.push({ + replyText: allReplies, + model: storeModel, + errored: false, + conversationId, + conversationTitle, + conversationProject, + }) + } + } + + // A project derived from ANY turn of a conversation applies to all its turns + // (the files are usually referenced in the first substantive turn only). + const projByConv = new Map<string, string>() + for (const t of turns) { + if (t.conversationProject && !projByConv.has(t.conversationId)) { + projByConv.set(t.conversationId, t.conversationProject) + } + } + for (const t of turns) { + if (!t.conversationProject) t.conversationProject = projByConv.get(t.conversationId) ?? '' + } + + return turns +} + +// --------------------------------------------------------------------------- +// Decoder entry point and per-arm helpers +// --------------------------------------------------------------------------- + +export type CopilotDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set<string> +} +export type CopilotDecodeResult = { + calls: CopilotDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +function pushCall(calls: CopilotDecodedCall[], arm: CopilotCallArm, base: Omit<CopilotDecodedCall, 'arm'>): void { + calls.push({ arm, ...base } as CopilotDecodedCall) +} + +function decodeJsonl(envelope: Extract<CopilotRecordEnvelope, { kind: 'jsonl' }>, seen: Set<string>, calls: CopilotDecodedCall[]): void { + const sessionId = envelope.sessionId + const lines = envelope.lines + + // Detect VS Code transcript format: the first session.start event has + // { producer: 'copilot-agent' } and no outputTokens in messages. + let isTranscript = false + let currentModel = '' + let pendingUserMessage = '' + // Track the active subagent for this session (from subagent.selected events). + // Resets when a new subagent is selected. + let currentSubagentType: string | undefined + + // First pass: detect format and infer transcript model if needed. + for (const line of lines) { + try { + const ev = JSON.parse(line) as CopilotEvent + if (ev.type === 'session.start') { + const data = ev.data as SessionStartData & { producer?: string } + if (data.producer === 'copilot-agent') { + isTranscript = true + } + break + } + if (ev.type === 'session.model_change') break // regular format + } catch { + continue + } + } + + if (isTranscript) { + currentModel = inferTranscriptModel(lines) + if (!currentModel) return // no toolCallIds to infer model from + } + + // Shutdown rollups may lack their own timestamp; remember the last + // stamped event so the supplementary call is never left with an empty + // timestamp, which the date-range filters silently drop. + let lastEventTimestamp = '' + + for (const line of lines) { + let event: CopilotEvent + try { + event = JSON.parse(line) as CopilotEvent + } catch { + continue + } + if (typeof event.timestamp === 'string' && event.timestamp) lastEventTimestamp = event.timestamp + + if (event.type === 'session.start') { + if (!isTranscript) { + currentModel = (event.data as SessionStartData).selectedModel ?? currentModel + } + continue + } + + if (event.type === 'session.model_change') { + currentModel = (event.data as ModelChangeData).newModel ?? currentModel + continue + } + + if (event.type === 'subagent.selected') { + currentSubagentType = (event.data as SubagentSelectedData).agentName + continue + } + + if (event.type === 'user.message') { + pendingUserMessage = (event.data as UserMessageData).content ?? '' + continue + } + + if (event.type === 'session.shutdown') { + // The Copilot CLI writes a per-model token/cost rollup here at + // shutdown: the only place a CLI session records input, cache-read + // and cache-write tokens (assistant.message events carry output + // only). VS Code transcripts never carry this rollup, so this path + // is gated to the CLI (non-transcript) format, leaving VS Code, + // JetBrains and OTel sources untouched. + // + // We emit one supplementary call per model carrying ONLY the + // input/cache tokens the per-turn events lack; output is excluded so + // the assistant.message output (and its cost) is not double-counted. + // Combined with the per-turn output cost, this yields the full, + // CLI-measured session cost. + if (isTranscript) continue + const shutdownData = event.data as SessionShutdownData + const modelMetrics = shutdownData.modelMetrics + if (!isRecord(modelMetrics)) continue + + const shutdownTimestamp = + (event.timestamp ?? '') || timestampToISO(shutdownData.sessionStartTime) || lastEventTimestamp + + for (const [model, metrics] of Object.entries(modelMetrics)) { + if (!model || !isRecord(metrics)) continue + const usage = metrics['usage'] + if (!isRecord(usage)) continue + + const cacheReadTokens = numberOrZero(usage['cacheReadTokens']) + const cacheWriteTokens = numberOrZero(usage['cacheWriteTokens']) + const reasoningTokens = numberOrZero(usage['reasoningTokens']) + // usage.inputTokens is cache-INCLUSIVE (input + cache_read + + // cache_write). calculateCost expects the uncached input alone with + // cache tokens billed separately, so subtract the cache components. + // Clamp at 0 in case a future schema reports input non-inclusively. + const inputTokens = Math.max( + 0, + numberOrZero(usage['inputTokens']) - cacheReadTokens - cacheWriteTokens + ) + + // Nothing this call would add over the per-turn events, so skip it + // to avoid an empty $0 row (output is intentionally excluded). + if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0) continue + + const dedupKey = `copilot:${sessionId}:shutdown:${model}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + pushCall(calls, 'jsonl-shutdown', { + provider: 'copilot', + sessionId, + model, + inputTokens, + outputTokens: 0, + cacheCreationInputTokens: cacheWriteTokens, + cacheReadInputTokens: cacheReadTokens, + cachedInputTokens: 0, + reasoningTokens, + webSearchRequests: 0, + tools: [], + rawBashCommands: [], + timestamp: shutdownTimestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: '', + }) + } + continue + } + + if (event.type === 'assistant.message') { + const msgData = event.data as AssistantMessageData + const { messageId, model: msgModel, outputTokens = 0 } = msgData + const rawRequests = (msgData as { toolRequests?: unknown }).toolRequests + const toolRequests = coerceToolRequests(rawRequests) + + // model may be carried per-message in newer copilot-agent format + if (msgModel) currentModel = msgModel + // Regular JSONL: skip zero-token messages; transcripts don't have tokens + if (!isTranscript && outputTokens === 0) continue + if (!currentModel) continue + + const dedupKey = `copilot:${sessionId}:${messageId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const tools = toolRequests + .map((t) => { + const raw = typeof t === 'object' && t !== null + ? ((t as { name?: unknown; toolName?: unknown }).name ?? (t as { name?: unknown; toolName?: unknown }).toolName) + : null + return typeof raw === 'string' ? normalizeCopilotTool(raw) : null + }) + .filter((t): t is string => t !== null) + + const skills = toolRequests.flatMap((t) => { + if (typeof t !== 'object' || t === null) return [] + const name = (t.name ?? t.toolName) ?? '' + if (name !== 'skill') return [] + const skill = t.arguments?.['skill'] + return typeof skill === 'string' && skill.trim().length > 0 ? [skill.trim()] : [] + }) + + const rawBashCommands = toolRequests.flatMap((t) => { + if (typeof t !== 'object' || t === null) return [] + const name = (t.name ?? t.toolName) ?? '' + if (!BASH_TOOL_NAMES.has(name)) return [] + const cmd = t.arguments?.['command'] + return typeof cmd === 'string' ? [cmd] : [] + }) + + // Copilot JSONL only logs outputTokens; inputTokens are NOT available. + // Cost will be lower than actual API cost. This is the original + // behaviour — OTel data (below) replaces it when available. + + pushCall(calls, 'jsonl-turn', { + provider: 'copilot', + sessionId, + model: currentModel, + inputTokens: 0, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools, + rawBashCommands, + skills: skills.length > 0 ? skills : undefined, + subagentTypes: currentSubagentType ? [currentSubagentType] : undefined, + timestamp: event.timestamp ?? '', + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: pendingUserMessage, + }) + pendingUserMessage = '' + } + } +} + +function decodeChatSession(envelope: Extract<CopilotRecordEnvelope, { kind: 'chatsession' }>, seen: Set<string>, calls: CopilotDecodedCall[]): void { + const content = envelope.content + const project = envelope.project + const fallbackSessionId = envelope.fallbackSessionId + + const root = replayChatSessionJournal(content) + if (!isRecord(root)) return + + const sessionId = readString(root['sessionId']) || fallbackSessionId + const sessionCreatedAt = timestampToISO(root['creationDate']) + const requests = Array.isArray(root['requests']) ? root['requests'] : [] + + for (let index = 0; index < requests.length; index++) { + const rawReq = requests[index] + if (!isRecord(rawReq)) continue + + const result = rawReq['result'] + const resultRecord = isRecord(result) ? result : null + const rawMetadata = resultRecord?.['metadata'] + const metadata = isRecord(rawMetadata) ? rawMetadata : createReplayObject() + + const inputTokens = numberOrZero(metadata['promptTokens']) + const metadataOutputTokens = numberOrZero(metadata['outputTokens']) + const outputTokens = metadataOutputTokens || numberOrZero(rawReq['completionTokens']) + + if (inputTokens === 0 && outputTokens === 0) continue + + const requestId = readString(rawReq['requestId']) || `request-${index}` + const dedupKey = `copilot-chatsession:${sessionId}:${requestId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const model = modelFromChatSessionRequest(rawReq, metadata) + const timestamp = timestampToISO(rawReq['timestamp']) || sessionCreatedAt + + pushCall(calls, 'chatsession', { + provider: 'copilot', + sessionId, + project, + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools: extractChatSessionTools(metadata), + rawBashCommands: [], + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: '', + }) + } +} + +function decodeJetBrains(envelope: Extract<CopilotRecordEnvelope, { kind: 'jetbrains' }>, seen: Set<string>, calls: CopilotDecodedCall[]): void { + const raw = envelope.raw + const sessionId = envelope.sessionId + const mtime = envelope.mtime + const sourceProjectName = envelope.projectName + + const storeModel = inferJetBrainsModel(raw) + const turns = extractJetBrainsDbTurns(raw, envelope.repoRootByDir) + // Dedup keys derive from the reply CONTENT, not the scan position: + // copilot is a durable provider (cached turns are never deleted and a + // re-parse appends any key it hasn't seen), while MVStore compaction + // can rewrite the file with blobs in a different byte order. With + // positional keys, a rewrite that puts a new blob ahead of an old one + // hands the new turn the old turn's key (skipped as seen) and re-emits + // the old turn under a fresh index — double-billing it. The per-hash + // counter keeps genuinely repeated replies and errored turns (which + // share replyText '') distinct within a conversation. + const perContentIndex = new Map<string, number>() + for (const turn of turns) { + // One .db holds many chat tabs; group each turn under its own + // conversation so the user sees one session per tab, not per file. + const convId = turn.conversationId || sessionId + const contentHash = createHash('sha256').update(turn.replyText).digest('hex').slice(0, 12) + const nth = (perContentIndex.get(`${convId}:${contentHash}`) ?? 0) + 1 + perContentIndex.set(`${convId}:${contentHash}`, nth) + const dedupKey = `copilot:jb:${convId}:${contentHash}:${nth}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + // Prefer the per-turn model, else the store default, else a generic + // Copilot bucket so a real reply is never mis-priced as free. + const model = turn.model || storeModel || 'copilot-anthropic-auto' + // Errored turns (failed generation) contribute no billable output. + const outputTokens = turn.errored ? 0 : estimateTokensFromChars(turn.replyText.length) + // Project resolution precedence: + // 1. projectName — the plugin's own recorded label (1.12+), + // joined across kind dirs by store id. Authoritative. + // 2. the git repo root of a file:// path the chat referenced + // (older plugins / when projectName is absent). + // 3. one honest bucket when neither signal exists. + // The conversation TITLE is a chat-thread name, NOT a project, and is + // kept out of `project` (it would otherwise pollute By-Project). + const project = sourceProjectName || turn.conversationProject || 'copilot-jetbrains' + + pushCall(calls, 'jetbrains', { + provider: 'copilot', + sessionId: convId, + project, + model, + inputTokens: 0, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools: [], + rawBashCommands: [], + timestamp: mtime, + speed: 'standard', + deduplicationKey: dedupKey, + // Surface the chat-thread name here (it is the session's label, not + // a project) so it remains visible in session-level views. + userMessage: turn.conversationTitle, + }) + } +} + +function decodeOtel(envelope: Extract<CopilotRecordEnvelope, { kind: 'otel' }>, seen: Set<string>, calls: CopilotDecodedCall[]): void { + for (const conversation of envelope.conversations) { + const conversationId = conversation.conversationId + const project = conversation.project + + // Collect tool names, shell commands and subagent names from the + // execute_tool / invoke_agent spans for each trace. These mirror the + // metadata the JSONL path captures, so the OTel source stays + // equivalent (tools + bashCommands + subagentTypes are all first-class + // call metadata per types.ts). + // + // Subagent attribution: VS Code records a subagent run as an + // invoke_agent span carrying copilot_chat.parent_chat_session_id. The + // root turn agent (gen_ai.agent.name = 'GitHub Copilot Chat') has NO + // parent session and is intentionally excluded, otherwise it would + // surface as a bogus 'GitHub Copilot Chat' entry in the agents view. + // A subagent's invoke_agent span lives in the same trace as that + // subagent's own chat spans, so attributing the agent name per-trace + // labels exactly the subagent's calls. + const toolsByTrace = new Map<string, string[]>() + const bashByTrace = new Map<string, string[]>() + const subagentsByTrace = new Map<string, string[]>() + const chatSpanIds: string[] = [] + const spanMetaById = new Map<string, { trace_id: string; start_time_ms: number; response_model: string | null }>() + // Stands in for the original's per-chat-span `loadSpanAttributesFromTable` + // read: the host loaded each bag once, so a lookup by span id is the same bag. + const attrsBySpanId = new Map<string, SpanAttributes>() + + for (const span of conversation.spans) { + const opName = span.operationName + spanMetaById.set(span.spanId, { trace_id: span.traceId, start_time_ms: span.startTimeMs, response_model: span.responseModel }) + if (span.attrs) attrsBySpanId.set(span.spanId, span.attrs) + + if (opName === 'chat') { + chatSpanIds.push(span.spanId) + continue + } + + if (opName === 'execute_tool') { + const attrs = span.attrs + if (attrs) { + const rawToolName = attrs['gen_ai.tool.name'] as string | undefined + if (rawToolName) { + const existing = toolsByTrace.get(span.traceId) ?? [] + existing.push(normalizeCopilotTool(rawToolName)) + toolsByTrace.set(span.traceId, existing) + + if (BASH_TOOL_NAMES.has(rawToolName)) { + const command = parseToolCommand(attrs['gen_ai.tool.call.arguments']) + if (command) { + const bash = bashByTrace.get(span.traceId) ?? [] + bash.push(command) + bashByTrace.set(span.traceId, bash) + } + } + } + } + continue + } + + if (opName === 'invoke_agent') { + const attrs = span.attrs + if (attrs) { + const parentSession = attrs['copilot_chat.parent_chat_session_id'] + const agentName = attrs['gen_ai.agent.name'] as string | undefined + if (parentSession && agentName) { + const subs = subagentsByTrace.get(span.traceId) ?? [] + subs.push(agentName) + subagentsByTrace.set(span.traceId, subs) + } + } + } + } + + // Yield one CopilotDecodedCall per chat span + for (const spanId of chatSpanIds) { + const spanMetadata = spanMetaById.get(spanId) + if (!spanMetadata) continue + + const attrs = attrsBySpanId.get(spanId) + + const model = + (attrs?.['gen_ai.response.model'] as string | undefined) ?? + (attrs?.['gen_ai.request.model'] as string | undefined) ?? + spanMetadata.response_model ?? + 'unknown' + + const inputTokens = Number(attrs?.['gen_ai.usage.input_tokens'] ?? 0) + const outputTokens = Number(attrs?.['gen_ai.usage.output_tokens'] ?? 0) + const cacheReadTokens = Number(attrs?.['gen_ai.usage.cache_read.input_tokens'] ?? 0) + const cacheCreationTokens = Number(attrs?.['gen_ai.usage.cache_creation.input_tokens'] ?? 0) + + if (inputTokens === 0 && outputTokens === 0) continue + + // Dedup key uses span_id which is globally unique + const dedupKey = `copilot-otel:${spanId}` + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + // Also add a JSONL-style dedupKey pattern so that if the same + // interaction appears in both OTel and JSONL, we don't double-count. + // We use the turn ID from Copilot attributes if available. + const turnId = attrs?.['github.copilot.chat.turn.id'] as string | undefined + if (turnId) { + const jsonlDedupKey = `copilot:${conversationId}:${turnId}` + seen.add(jsonlDedupKey) + } + + const tools = toolsByTrace.get(spanMetadata.trace_id) ?? [] + const rawBashCommands = bashByTrace.get(spanMetadata.trace_id) ?? [] + const subagentTypes = subagentsByTrace.get(spanMetadata.trace_id) + const timestamp = epochToISO(spanMetadata.start_time_ms) + + pushCall(calls, 'otel', { + provider: 'copilot', + sessionId: conversationId, + project, + model, + inputTokens, + outputTokens, + cacheCreationInputTokens: cacheCreationTokens, + cacheReadInputTokens: cacheReadTokens, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + tools, + rawBashCommands, + subagentTypes: subagentTypes && subagentTypes.length > 0 ? subagentTypes : undefined, + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: '', + }) + } + } +} + +export function decodeCopilot(input: CopilotDecodeInput): CopilotDecodeResult { + const seen = input.seenKeys ?? new Set<string>() + const calls: CopilotDecodedCall[] = [] + const envelope = input.records[0] as CopilotRecordEnvelope | undefined + if (!envelope) return { calls, diagnostics: [] } + switch (envelope.kind) { + case 'jsonl': decodeJsonl(envelope, seen, calls); break + case 'chatsession': decodeChatSession(envelope, seen, calls); break + case 'jetbrains': decodeJetBrains(envelope, seen, calls); break + case 'otel': decodeOtel(envelope, seen, calls); break + } + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/copilot/index.ts b/packages/core/src/providers/copilot/index.ts new file mode 100644 index 00000000..f01de8f8 --- /dev/null +++ b/packages/core/src/providers/copilot/index.ts @@ -0,0 +1,31 @@ +// @codeburn/core Copilot provider. +// +// Two layers: +// - Rich pure decode (`decodeCopilot`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied records envelope (the sqlite +// driver, filesystem, and SQL queries stay CLI-side, Category B). +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + collectJetBrainsRepoDirCandidates, + copilotToolNameMap, + decodeCopilot, + normalizeCopilotTool, + type CopilotDecodeInput, + type CopilotDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichCopilotSessionDecode, + type CopilotToObservationsContext, +} from './observations.js' + +export type { + CopilotDecodedCall, + CopilotOtelConversationRecord, + CopilotOtelSpanRecord, + CopilotRecordEnvelope, + SpanAttributes, +} from './types.js' diff --git a/packages/core/src/providers/copilot/observations.ts b/packages/core/src/providers/copilot/observations.ts new file mode 100644 index 00000000..92ed1a49 --- /dev/null +++ b/packages/core/src/providers/copilot/observations.ts @@ -0,0 +1,81 @@ +// Minimizing transform: rich Copilot decode -> the strict observation envelope. +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output — never the user message, project path, or raw +// bash commands captured by the rich decode. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { CopilotDecodedCall } from './types.js' + +/** One Copilot session's rich decode, as the host holds it before minimization. */ +export interface RichCopilotSessionDecode { + sessionId: string + /** Absolute project path; fingerprinted, never emitted raw. */ + projectPath: string + /** Rich calls in decode order. */ + calls: CopilotDecodedCall[] +} + +export interface CopilotToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: CopilotDecodedCall, turnIndex: number): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichCopilotSessionDecode, ctx: CopilotToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'copilot' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Copilot decode (one or many sessions) into the minimized observation + * layer. Returns the `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichCopilotSessionDecode | RichCopilotSessionDecode[], + ctx: CopilotToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/copilot/types.ts b/packages/core/src/providers/copilot/types.ts new file mode 100644 index 00000000..69158f60 --- /dev/null +++ b/packages/core/src/providers/copilot/types.ts @@ -0,0 +1,149 @@ +// Types for Copilot record decoding. These types are shared between the +// CLI-side I/O adapter and the core decoder. + +export type ToolRequest = { + toolName?: string // older format + name?: string // newer format (copilot-agent) + arguments?: Record<string, unknown> +} + +export type SessionStartData = { + selectedModel?: string +} + +export type ModelChangeData = { + newModel: string + previousModel?: string +} + +export type UserMessageData = { + content: string + interactionId?: string +} + +export type AssistantMessageData = { + messageId: string + model?: string // present in newer copilot-agent format + outputTokens: number + interactionId?: string + toolRequests?: ToolRequest[] +} + +export type SubagentSelectedData = { + agentName: string + agentDisplayName?: string + tools?: string[] +} + +// Per-model usage rollup the CLI writes into session.shutdown. inputTokens is +// cache-INCLUSIVE (input + cache_read + cache_write); see the shutdown handler. +export type ShutdownModelUsage = { + inputTokens?: number + outputTokens?: number + cacheReadTokens?: number + cacheWriteTokens?: number + reasoningTokens?: number +} + +export type SessionShutdownData = { + modelMetrics?: Record<string, { usage?: ShutdownModelUsage }> + sessionStartTime?: number +} + +export type CopilotEvent = + | { type: 'session.start'; data: SessionStartData; timestamp?: string } + | { type: 'session.model_change'; data: ModelChangeData; timestamp?: string } + | { type: 'user.message'; data: UserMessageData; timestamp?: string } + | { type: 'assistant.message'; data: AssistantMessageData; timestamp?: string } + | { type: 'subagent.selected'; data: SubagentSelectedData; timestamp?: string } + | { type: 'session.shutdown'; data: SessionShutdownData; timestamp?: string } + +export type ChatJournalPathSegment = string | number +export type ChatSessionRequest = Record<string, unknown> + +// Parsed attribute bag from a span +export interface SpanAttributes { + 'gen_ai.operation.name'?: string + 'gen_ai.response.model'?: string + 'gen_ai.request.model'?: string + 'gen_ai.usage.input_tokens'?: number + 'gen_ai.usage.output_tokens'?: number + 'gen_ai.usage.cache_read.input_tokens'?: number + 'gen_ai.usage.cache_creation.input_tokens'?: number + 'gen_ai.conversation.id'?: string + 'gen_ai.agent.name'?: string + 'gen_ai.tool.name'?: string + 'gen_ai.tool.call.arguments'?: string + 'copilot_chat.parent_chat_session_id'?: string + 'github.copilot.chat.turn.id'?: string + [key: string]: unknown +} + +// One assistant turn recovered from a .db. +export type JBDbTurn = { + replyText: string + model: string + errored: boolean + // The owning conversation (chat tab): its internal GUID and title. One .db + // holds many conversations; turns are grouped back to their tab by this id. + conversationId: string + conversationTitle: string + // The file path this conversation referenced (home-relative common dir), or + // '' if the chat touched no files. Used as the project label. + conversationProject: string +} + +// A conversation (chat tab) recovered from a .db: internal GUID → title. +export type JBConversation = { id: string; title: string } + +/** One OTel span row + its already-loaded attribute bag (host runs the sqlite). */ +export interface CopilotOtelSpanRecord { + spanId: string + traceId: string + operationName: string // '' when the column was null + startTimeMs: number + responseModel: string | null + attrs: SpanAttributes | null // null for spans whose attrs the host did not load +} + +export interface CopilotOtelConversationRecord { + conversationId: string + project: string // already normalized host-side + spans: CopilotOtelSpanRecord[] // ALL spans of this conversation's traces, query order +} + +export type CopilotRecordEnvelope = + | { kind: 'jsonl'; sessionId: string; lines: string[] } + | { kind: 'chatsession'; fallbackSessionId: string; project: string; content: string } + | { kind: 'jetbrains'; sessionId: string; mtime: string; projectName?: string + raw: string; repoRootByDir: ReadonlyMap<string, string> } + | { kind: 'otel'; conversations: CopilotOtelConversationRecord[] } + +export type CopilotCallArm = + | 'jsonl-turn' | 'jsonl-shutdown' | 'chatsession' | 'jetbrains' | 'otel' + +export interface CopilotDecodedCall { + /** Which parse arm produced this call. The host switches on it in toProviderCall. */ + arm: CopilotCallArm + provider: 'copilot' + sessionId: string + /** Present only for arms that set it today (chatsession / jetbrains / otel). */ + project?: string + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + /** RAW command strings, in emission order. The host reduces them. */ + rawBashCommands: string[] + skills?: string[] + subagentTypes?: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index c212421e..fa0bd6d2 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -167,6 +167,8 @@ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/quickdesk/types.ts', 'src/providers/devin/decode.ts', 'src/providers/devin/types.ts', + 'src/providers/copilot/decode.ts', + 'src/providers/copilot/types.ts', ]) describe('architecture gate: no classification or free text in @codeburn/core source', () => { diff --git a/packages/core/tests/content-smuggling.test.ts b/packages/core/tests/content-smuggling.test.ts index 1b9848ee..edc73212 100644 --- a/packages/core/tests/content-smuggling.test.ts +++ b/packages/core/tests/content-smuggling.test.ts @@ -34,6 +34,7 @@ import { decodeWarp, toObservations as toWarpObservations } from '../src/provide import { decodeCursorAgent, toObservations as toCursorAgentObservations } from '../src/providers/cursor-agent/index.js' import { decodeQuickdesk, toObservations as toQuickdeskObservations } from '../src/providers/quickdesk/index.js' import { decodeDevin, toObservations as toDevinObservations } from '../src/providers/devin/index.js' +import { decodeCopilot, toObservations as toCopilotObservations } from '../src/providers/copilot/index.js' import type { DecodeContext } from '../src/contracts.js' import type { ZedThreadRow } from '../src/providers/zed/index.js' @@ -882,6 +883,90 @@ describe('content-smuggling guardrail: real goose decode -> toObservations is se }) }) +describe('content-smuggling guardrail: real copilot decode -> toObservations is secret-free', () => { + // A hostile Copilot session planting every secret in the free-text fields the + // decode captures: the user prompt (jsonl user.message), a Bash command, a + // hostile tool NAME carrying a command line, and a JetBrains conversation + // title + reply blob. The observation envelope MUST surface none of them. + const copilotContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'copilot', sourceRef: 'ref' } + + function decodeAndMinimize() { + const jsonlEnvelope = { + kind: 'jsonl' as const, + sessionId: 'sess-hostile-jsonl', + lines: [ + JSON.stringify({ type: 'user.message', timestamp: '2026-07-17T10:00:00.000Z', data: { content: SECRETS.prompt } }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-07-17T10:00:05.000Z', + data: { + messageId: 'msg-hostile-1', + model: 'gpt-4.1', + outputTokens: 100, + toolRequests: [ + { name: 'bash', arguments: { command: SECRETS.commandLine } }, + // A hostile tool NAME carrying a command line: fails canonical charset. + { name: SECRETS.commandLine, arguments: {} }, + ], + }, + }), + ], + } + + const convGuid = '11111111-1111-1111-1111-111111111111' + const convTitle = `${SECRETS.apiKey} ${SECRETS.fileContent}` + const convRecord = `$${convGuid}t\x00\x04namesq\x00\x01?@\x00\x00w\x00\x00t\x00value t\x00${convTitle}t\x00\x06sourcet\x00copilotx` + const innerMd = { type: 'Markdown', data: JSON.stringify({ text: `${SECRETS.apiKey} ${SECRETS.fileContent}`, annotations: [] }) } + const valueMap: Record<string, unknown> = { + 'a1b2c3d4-0000-0000-0000-000000000001': { type: 'Value', value: JSON.stringify(innerMd) }, + } + const blob = JSON.stringify({ __first__: { type: 'Subgraph', value: JSON.stringify(valueMap) } }) + const raw = 'H:2,block:9,blockSize:1000,format:3\n' + + 'com.github.copilot.agent.session.persistence.nitrite.entity.NtAgentTurn\n' + + convRecord + '\n' + blob + '\n' + const jbEnvelope = { + kind: 'jetbrains' as const, + sessionId: 'sess-hostile-jb', + mtime: '2026-07-17T10:00:00.000Z', + raw, + repoRootByDir: new Map<string, string>(), + } + + const { calls: jsonlCalls } = decodeCopilot({ records: [jsonlEnvelope], context: copilotContext }) + const { calls: jbCalls } = decodeCopilot({ records: [jbEnvelope], context: copilotContext }) + const { sessions } = toCopilotObservations( + [ + { sessionId: 'sess-hostile-jsonl', projectPath: SECRETS.absPath, calls: jsonlCalls }, + { sessionId: 'sess-hostile-jb', projectPath: SECRETS.absPath, calls: jbCalls }, + ], + { privacyKey: 'test-privacy-key', provider: 'copilot' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile copilot records', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + describe('content-smuggling guardrail: diagnostic detail rejects paths', () => { it('rejects an absolute path', () => { expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false) diff --git a/packages/core/tests/providers/copilot-decode.test.ts b/packages/core/tests/providers/copilot-decode.test.ts new file mode 100644 index 00000000..9eb54af5 --- /dev/null +++ b/packages/core/tests/providers/copilot-decode.test.ts @@ -0,0 +1,326 @@ +import { describe, it, expect } from 'vitest' +import { collectJetBrainsRepoDirCandidates, decodeCopilot } from '../../src/providers/copilot/decode.js' +import type { CopilotRecordEnvelope } from '../../src/providers/copilot/types.js' + +const ctx = { privacyKey: 'test-key', providerId: 'copilot', sourceRef: 'ref' } + +describe('decodeCopilot — jsonl arm', () => { + it('skips a zero-token assistant.message in non-transcript format', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ type: 'user.message', data: { content: 'hello' } }), + JSON.stringify({ type: 'assistant.message', data: { messageId: 'm1', outputTokens: 0 } }), + ], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(0) + expect(seen.size).toBe(0) + }) + + it('falls back to model from session.start and preserves pendingUserMessage across skipped messages', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ type: 'session.start', data: { selectedModel: 'gpt-4.1' } }), + JSON.stringify({ type: 'user.message', data: { content: 'pending message' } }), + JSON.stringify({ type: 'assistant.message', data: { messageId: 'skip', outputTokens: 0 } }), + JSON.stringify({ type: 'assistant.message', data: { messageId: 'keep', outputTokens: 10 } }), + ], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect(calls[0]?.userMessage).toBe('pending message') + expect(calls[0]?.model).toBe('gpt-4.1') + expect(seen.has('copilot:s1:keep')).toBe(true) + }) + + it('dedups by message id and tags the arm', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ type: 'assistant.message', data: { messageId: 'm1', outputTokens: 10, model: 'gpt-4.1' } }), + JSON.stringify({ type: 'assistant.message', data: { messageId: 'm1', outputTokens: 20, model: 'gpt-5' } }), + ], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect(calls[0]?.arm).toBe('jsonl-turn') + expect(calls[0]?.outputTokens).toBe(10) + expect(seen.size).toBe(1) + }) + + it('emits a shutdown arm with cache-inclusive input tokens priced separately', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ type: 'assistant.message', data: { messageId: 'm1', outputTokens: 10 } }), + JSON.stringify({ + type: 'session.shutdown', + data: { + modelMetrics: { + 'gpt-4.1': { + usage: { inputTokens: 1100, cacheReadTokens: 100, cacheWriteTokens: 50, reasoningTokens: 5 }, + }, + }, + }, + }), + ], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + const shutdown = calls.find(c => c.arm === 'jsonl-shutdown') + expect(shutdown).toBeDefined() + expect(shutdown?.inputTokens).toBe(950) // 1100 - 100 - 50 + expect(shutdown?.cacheReadInputTokens).toBe(100) + expect(shutdown?.cacheCreationInputTokens).toBe(50) + expect(shutdown?.reasoningTokens).toBe(5) + expect(shutdown?.outputTokens).toBe(0) + expect(seen.has('copilot:s1:shutdown:gpt-4.1')).toBe(true) + }) + + it('skips shutdown when all cache/input tokens are zero', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ + type: 'session.shutdown', + data: { + modelMetrics: { + 'gpt-4.1': { usage: { inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } }, + }, + }, + }), + ], + } + const { calls } = decodeCopilot({ records: [envelope], context: ctx }) + expect(calls).toHaveLength(0) + }) + + it('skips transcript files with no inferable model', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'jsonl', + sessionId: 's1', + lines: [ + JSON.stringify({ type: 'session.start', data: { producer: 'copilot-agent' } }), + JSON.stringify({ type: 'assistant.message', data: { messageId: 'm1' } }), + ], + } + const { calls } = decodeCopilot({ records: [envelope], context: ctx }) + expect(calls).toHaveLength(0) + }) +}) + +describe('decodeCopilot — chatsession arm', () => { + it('replays a kind-0/1/2 journal and skips zero-token requests', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'chatsession', + fallbackSessionId: 'fallback', + project: 'myproject', + content: [ + JSON.stringify({ kind: 0, v: { version: 3, creationDate: 1780157113020, sessionId: 'cs1', requests: [] } }), + JSON.stringify({ kind: 1, k: ['requests'], v: [{ requestId: 'r1', modelId: 'copilot/gpt-4.1', timestamp: 1780157113100, result: { metadata: { promptTokens: 100, outputTokens: 10 } } }] }), + JSON.stringify({ kind: 2, k: ['requests'], v: [{ requestId: 'r2', modelId: 'copilot/gpt-4.1', timestamp: 1780157113200, result: { metadata: { promptTokens: 0, outputTokens: 0 } } }] }), + ].join('\n'), + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect(calls[0]?.arm).toBe('chatsession') + expect(calls[0]?.model).toBe('gpt-4.1') + expect(calls[0]?.sessionId).toBe('cs1') + expect(seen.has('copilot-chatsession:cs1:r1')).toBe(true) + expect(seen.has('copilot-chatsession:cs1:r2')).toBe(false) + }) +}) + +describe('decodeCopilot — jetbrains arm', () => { + it('extracts an ask-mode turn and dedups by content hash', () => { + const convGuid = '11111111-1111-1111-1111-111111111111' + const convRecord = `$${convGuid}t\x00\x04namesq\x00\x01?@\x00\x00w\x00\x00t\x00value t\x00Hello Worldt\x00\x06sourcet\x00copilotx` + const innerMd = { type: 'Markdown', data: JSON.stringify({ text: 'Ask reply', annotations: [] }) } + const valueMap: Record<string, unknown> = { + 'a1b2c3d4-0000-0000-0000-000000000001': { type: 'Value', value: JSON.stringify(innerMd) }, + } + const blob = JSON.stringify({ __first__: { type: 'Subgraph', value: JSON.stringify(valueMap) } }) + const raw = 'H:2,block:9,blockSize:1000,format:3\n' + + 'com.github.copilot.agent.session.persistence.nitrite.entity.NtAgentTurn\n' + + convRecord + '\n' + blob + '\n' + blob + '\n' + const envelope: CopilotRecordEnvelope = { + kind: 'jetbrains', + sessionId: 'jb1', + mtime: '2026-07-17T10:00:00.000Z', + raw, + repoRootByDir: new Map(), + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect(calls[0]?.arm).toBe('jetbrains') + expect(calls[0]?.outputTokens).toBe(3) // ceil("Ask reply".length / 4) = 3 + expect(calls[0]?.userMessage).toBe('Hello World') + }) +}) + +describe('decodeCopilot — otel arm', () => { + it('skips zero-token chat spans and records tool/subagent metadata per trace', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'otel', + conversations: [{ + conversationId: 'conv1', + project: 'myproject', + spans: [ + { + spanId: 'span-chat', + traceId: 'trace1', + operationName: 'chat', + startTimeMs: 1000, + responseModel: 'gpt-4.1', + attrs: { + 'gen_ai.response.model': 'gpt-4.1', + 'gen_ai.usage.input_tokens': 100, + 'gen_ai.usage.output_tokens': 50, + }, + }, + { + spanId: 'span-tool', + traceId: 'trace1', + operationName: 'execute_tool', + startTimeMs: 1100, + responseModel: null, + attrs: { + 'gen_ai.tool.name': 'bash', + 'gen_ai.tool.call.arguments': JSON.stringify({ command: 'ls -la' }), + }, + }, + { + spanId: 'span-zero', + traceId: 'trace1', + operationName: 'chat', + startTimeMs: 1200, + responseModel: 'gpt-4.1', + attrs: { + 'gen_ai.usage.input_tokens': 0, + 'gen_ai.usage.output_tokens': 0, + }, + }, + { + spanId: 'span-subagent', + traceId: 'trace1', + operationName: 'invoke_agent', + startTimeMs: 1300, + responseModel: null, + attrs: { + 'gen_ai.agent.name': 'Explore', + 'copilot_chat.parent_chat_session_id': 'parent1', + }, + }, + { + spanId: 'span-root-agent', + traceId: 'trace2', + operationName: 'invoke_agent', + startTimeMs: 1400, + responseModel: null, + attrs: { + 'gen_ai.agent.name': 'GitHub Copilot Chat', + }, + }, + ], + }], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect(calls[0]?.arm).toBe('otel') + expect(calls[0]?.tools).toEqual(['Bash']) + expect(calls[0]?.rawBashCommands).toEqual(['ls -la']) + expect(calls[0]?.subagentTypes).toEqual(['Explore']) + expect(seen.has('copilot-otel:span-chat')).toBe(true) + expect(seen.has('copilot-otel:span-zero')).toBe(false) + }) +}) + +describe('collectJetBrainsRepoDirCandidates', () => { + // The scanner must be a SUPERSET of the per-chunk `file://(/[^"\\]+?)(?:\\|")` + // walk the decoder runs: the host resolves each candidate to a repo root, and + // a dir the scanner misses degrades to "no project" instead of the real repo. + const perChunkScan = (raw: string): string[] => { + const re = /file:\/\/(\/[^"\\]+?)(?:\\|")/g + const dirs = new Set<string>() + let m: RegExpExecArray | null + while ((m = re.exec(raw))) { + let p = m[1]! + try { p = decodeURIComponent(p) } catch { /* leave as-is */ } + const dir = p.slice(0, p.lastIndexOf('/')) + if (dir.startsWith('/')) dirs.add(dir) + } + return [...dirs] + } + + const raws = [ + '{"data":"file:///home/user/repo/src/Main.java"}', + '{"data":"file:///home/user/pipe|repo/src/Main.java"}', + '{"data":"file:///home/user/my%20repo/src/Main.java"}', + '{"a":"file:///one/a.ts\\\\","b":"file:///two/b.ts"}', + 'file://file:///nested/c.ts"', + '{"data":"file:///no-terminator/d.ts', + 'no file uris here at all', + ] + + for (const raw of raws) { + it(`covers every dir the per-chunk scan finds: ${JSON.stringify(raw).slice(0, 48)}`, () => { + const candidates = new Set(collectJetBrainsRepoDirCandidates(raw)) + for (const dir of perChunkScan(raw)) expect(candidates.has(dir)).toBe(true) + }) + } + + it('keeps a pipe character inside the path', () => { + expect(collectJetBrainsRepoDirCandidates('{"d":"file:///home/pipe|repo/src/Main.java"}')) + .toEqual(['/home/pipe|repo/src']) + }) +}) + +describe('decodeCopilot — otel cross-source dedup side effect (O17)', () => { + it('adds copilot:<conv>:<turnId> to seenKeys without emitting a call', () => { + const envelope: CopilotRecordEnvelope = { + kind: 'otel', + conversations: [{ + conversationId: 'conv1', + project: 'p', + spans: [{ + spanId: 'span-a', traceId: 't1', operationName: 'chat', startTimeMs: 1000, responseModel: 'gpt-4.1', + attrs: { 'gen_ai.usage.input_tokens': 5, 'gen_ai.usage.output_tokens': 1, 'github.copilot.chat.turn.id': 'turn-9' }, + }], + }], + } + const seen = new Set<string>() + const { calls } = decodeCopilot({ records: [envelope], context: ctx, seenKeys: seen }) + expect(calls).toHaveLength(1) + expect([...seen].sort()).toEqual(['copilot-otel:span-a', 'copilot:conv1:turn-9']) + }) + + it('emits a span shared by two conversations only once (O21)', () => { + const span = { + spanId: 'span-shared', traceId: 't1', operationName: 'chat', startTimeMs: 1000, responseModel: 'gpt-4.1', + attrs: { 'gen_ai.usage.input_tokens': 5, 'gen_ai.usage.output_tokens': 1 }, + } + const envelope: CopilotRecordEnvelope = { + kind: 'otel', + conversations: [ + { conversationId: 'conv-a', project: 'p', spans: [span] }, + { conversationId: 'conv-b', project: 'p', spans: [span] }, + ], + } + const { calls } = decodeCopilot({ records: [envelope], context: ctx }) + expect(calls).toHaveLength(1) + expect(calls[0]?.sessionId).toBe('conv-a') + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index f8d3796b..4ae19a69 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -37,6 +37,7 @@ export default defineConfig({ 'src/providers/cursor-agent/index.ts', 'src/providers/quickdesk/index.ts', 'src/providers/devin/index.ts', + 'src/providers/copilot/index.ts', ], format: ['esm'], target: 'node20',