diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index 5c8417aec68fb5..249f327da8f02c 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -155,6 +155,7 @@ export interface IAgentHostToolInvokedReport { toolSourceKind: string; result: ToolInvokedResult; invocationTimeMs?: number; + resultSizeInCharacters: number; } export interface IAgentHostAskQuestionsToolInvokedEvent { @@ -811,6 +812,7 @@ export class AgentHostTelemetryReporter { toolSourceKind: report.toolSourceKind, invocationTimeMs: report.invocationTimeMs, provider: report.provider, + resultSizeInCharacters: report.resultSizeInCharacters, }); } diff --git a/src/vs/platform/agentHost/node/agentHostToolCallTracker.ts b/src/vs/platform/agentHost/node/agentHostToolCallTracker.ts index 76b30e5f104447..1a6ffb15e13624 100644 --- a/src/vs/platform/agentHost/node/agentHostToolCallTracker.ts +++ b/src/vs/platform/agentHost/node/agentHostToolCallTracker.ts @@ -143,6 +143,7 @@ export class AgentHostToolCallTracker extends Disposable { this._toolCalls.delete(key); const resultBucket = deriveToolInvokedResult(result); const totalTimeMs = timing.lifecycleStopWatch.elapsed(); + const resultSizeInCharacters = JSON.stringify(result).length; this._reporter.toolInvoked({ provider: timing.provider, @@ -151,6 +152,7 @@ export class AgentHostToolCallTracker extends Disposable { toolSourceKind: timing.toolSourceKind, result: resultBucket, invocationTimeMs: timing.invocationStopWatch?.elapsed(), + resultSizeInCharacters, }); const stalled = this._stalledToolCalls.get(key); diff --git a/src/vs/platform/agentHost/test/node/agentHostToolCallTelemetry.test.ts b/src/vs/platform/agentHost/test/node/agentHostToolCallTelemetry.test.ts index 5c0370a51f2553..051aa818a208b2 100644 --- a/src/vs/platform/agentHost/test/node/agentHostToolCallTelemetry.test.ts +++ b/src/vs/platform/agentHost/test/node/agentHostToolCallTelemetry.test.ts @@ -17,7 +17,7 @@ import { ITelemetryService, TelemetryLevel } from '../../../telemetry/common/tel import { AgentSession, IAgent } from '../../common/agentService.js'; import { SessionInputRequestKind } from '../../common/state/protocol/state.js'; import { ActionType, type ChatAction } from '../../common/state/sessionActions.js'; -import { buildDefaultChatUri, MessageKind, SessionStatus, ToolCallConfirmationReason, ToolCallContributorKind, ToolCallStatus, type ToolCallContributor, type ToolCallResult } from '../../common/state/sessionState.js'; +import { buildDefaultChatUri, MessageKind, SessionStatus, ToolCallConfirmationReason, ToolCallContributorKind, ToolCallStatus, ToolResultContentType, type ToolCallContributor, type ToolCallResult } from '../../common/state/sessionState.js'; import { IAgentHostCheckpointService, NULL_CHECKPOINT_SERVICE } from '../../common/agentHostCheckpointService.js'; import { IAgentHostTerminalManager } from '../../node/agentHostTerminalManager.js'; import { AgentHostLocalTurns } from '../../node/agentHostLocalTurns.js'; @@ -235,6 +235,7 @@ suite('AgentSideEffects — tool call telemetry', () => { toolSourceKind: 'agentHost', provider: 'mock', invocationTimeMs: true, + resultSizeInCharacters: 41, }, }]); }); @@ -256,6 +257,7 @@ suite('AgentSideEffects — tool call telemetry', () => { toolSourceKind: 'mcp', provider: 'mock', invocationTimeMs: undefined, + resultSizeInCharacters: 90, }, }]); }); @@ -284,10 +286,25 @@ suite('AgentSideEffects — tool call telemetry', () => { toolSourceKind: 'client', provider: 'mock', invocationTimeMs: true, + resultSizeInCharacters: 47, }, }]); }); + test('includes result content in the serialized result size', () => { + setupSession(); + startTurn('turn-1'); + + toolStart('turn-1', 'tc-read', 'read_file'); + toolComplete('turn-1', 'tc-read', { + success: true, + pastTenseMessage: 'read files', + content: [{ type: ToolResultContentType.Text, text: 'alpha\nbeta' }], + }); + + assert.deepStrictEqual(toolEvents()[0].data.resultSizeInCharacters, 97); + }); + test('only accepts contributor refinements that preserve execution ownership', async () => { setupSession(); startTurn('turn-1'); diff --git a/src/vs/platform/telemetry/common/languageModelToolTelemetry.ts b/src/vs/platform/telemetry/common/languageModelToolTelemetry.ts index fc5cb83fe088d7..b4e84c7c71e5a9 100644 --- a/src/vs/platform/telemetry/common/languageModelToolTelemetry.ts +++ b/src/vs/platform/telemetry/common/languageModelToolTelemetry.ts @@ -22,6 +22,7 @@ export type LanguageModelToolInvokedEvent = LanguageModelToolTelemetryData & { prepareTimeMs?: number; invocationTimeMs?: number; provider?: string; + resultSizeInCharacters?: number; }; export type LanguageModelToolInvokedClassification = LanguageModelToolTelemetryClassification & { @@ -29,6 +30,7 @@ export type LanguageModelToolInvokedClassification = LanguageModelToolTelemetryC prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' }; invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' }; provider?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The agent host provider that invoked the tool (e.g. copilotcli, claude, codex), if applicable.' }; + resultSizeInCharacters?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Length of the serialized Agent Host tool result in UTF-16 code units, if applicable.' }; owner: 'roblourens'; comment: 'Provides insight into the usage of language model tools.'; };