feat(telemetry): improve tracking for docs chatbot VSCODE-660 (#900)

Author: gagik

src/participant/constants.ts

@@ -1,21 +1,12 @@
 import type * as vscode from 'vscode';
 import { ChatMetadataStore } from './chatMetadata';
+import type { ParticipantResponseType } from './participantTypes';
 
 export const CHAT_PARTICIPANT_ID = 'mongodb.participant';
 export const CHAT_PARTICIPANT_MODEL = 'gpt-4o';
 export const COPILOT_EXTENSION_ID = 'GitHub.copilot';
 export const COPILOT_CHAT_EXTENSION_ID = 'GitHub.copilot-chat';
 
-export type ParticipantResponseType =
-  | 'query'
-  | 'schema'
-  | 'docs'
-  | 'generic'
-  | 'emptyRequest'
-  | 'cancelledRequest'
-  | 'askToConnect'
-  | 'askForNamespace';
-
 export const codeBlockIdentifier = {
   start: '```javascript',
   end: '```',

src/participant/participant.ts

@@ -1540,22 +1540,23 @@ export default class ParticipantController {
   }
 
   async _handleDocsRequestWithChatbot({
-    prompt,
+    request,
     chatId,
     token,
     stream,
     context,
   }: {
-    prompt: string;
     chatId: string;
     token: vscode.CancellationToken;
     context: vscode.ChatContext;
+    request: vscode.ChatRequest;
     stream: vscode.ChatResponseStream;
   }): Promise<{
     responseContent: string;
     responseReferences?: Reference[];
     docsChatbotMessageId: string;
   }> {
+    const prompt = request.prompt;
     stream.push(
       new vscode.ChatResponseProgressPart('Consulting MongoDB documentation...')
     );
@@ -1597,6 +1598,10 @@ export default class ParticipantController {
       signal: abortController.signal,
     });
 
+    const stats = Prompts.docs.getStats(history, { request, context });
+
+    this._telemetryService.trackParticipantPrompt(stats);
+
     log.info('Docs chatbot message sent', {
       chatId,
       docsChatbotConversationId,
@@ -1694,7 +1699,7 @@ export default class ParticipantController {
 
     try {
       docsResult = await this._handleDocsRequestWithChatbot({
-        prompt: request.prompt,
+        request,
         chatId,
         token,
         stream,

src/participant/prompts/docs.ts

@@ -0,0 +1,17 @@
+import type { ParticipantPromptProperties } from '../../telemetry/telemetryService';
+import type { PromptArgsBase } from './promptBase';
+import { PromptBase } from './promptBase';
+import type * as vscode from 'vscode';
+
+export class DocsPrompt extends PromptBase<PromptArgsBase> {
+  protected getAssistantPrompt(): string {
+    throw new Error('Method not implemented.');
+  }
+
+  public getStats(
+    messages: vscode.LanguageModelChatMessage[],
+    { request, context }: PromptArgsBase
+  ): ParticipantPromptProperties {
+    return super.getStats(messages, { request, context }, false);
+  }
+}

src/participant/prompts/index.ts

@@ -8,11 +8,13 @@ import { SchemaPrompt } from './schema';
 import { ExportToPlaygroundPrompt } from './exportToPlayground';
 import { ExportToLanguagePrompt } from './exportToLanguage';
 import { isContentEmpty } from './promptBase';
+import { DocsPrompt } from './docs';
 
 export { getContentLength } from './promptBase';
 
 export class Prompts {
   public static generic = new GenericPrompt();
+  public static docs = new DocsPrompt();
   public static intent = new IntentPrompt();
   public static namespace = new NamespacePrompt();
   public static query = new QueryPrompt();

src/test/suite/participant/participant.test.ts

@@ -43,7 +43,10 @@ import {
 import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensProvider';
 import PlaygroundResultProvider from '../../../editors/playgroundResultProvider';
 import { CollectionTreeItem, DatabaseTreeItem } from '../../../explorer';
-import type { SendMessageToParticipantOptions } from '../../../participant/participantTypes';
+import type {
+  ParticipantRequestType,
+  SendMessageToParticipantOptions,
+} from '../../../participant/participantTypes';
 import { DocumentSource } from '../../../documentSource';
 
 // The Copilot's model in not available in tests,
@@ -125,7 +128,7 @@ suite('Participant Controller Test Suite', function () {
     );
 
   const assertCommandTelemetry = (
-    command: string,
+    command: ParticipantRequestType,
     chatRequest: vscode.ChatRequest,
     {
       expectSampleDocs = false,
@@ -148,10 +151,13 @@ suite('Participant Controller Test Suite', function () {
     expect(properties.has_sample_documents).to.equal(expectSampleDocs);
     expect(properties.history_size).to.equal(chatContextStub.history.length);
 
-    // Total message length includes participant as well as user prompt
-    expect(properties.total_message_length).to.be.greaterThan(
-      properties.user_input_length
-    );
+    /** For docs chatbot requests, the length of the prompt would be longer as it gets the prompt history prepended.*/
+    if (command !== 'docs') {
+      // Total message length includes participant as well as user prompt
+      expect(properties.total_message_length).to.be.greaterThan(
+        properties.user_input_length
+      );
+    }
 
     // User prompt length should be at least equal to the supplied user prompt, but my occasionally
     // be greater - e.g. when we enhance the context.
@@ -1709,8 +1715,14 @@ Schema:
           expect(fetchStub).to.have.been.called;
           expect(sendRequestStub).to.have.not.been.called;
 
-          assertResponseTelemetry('docs/chatbot', {
+          assertCommandTelemetry('docs', chatRequestMock, {
+            expectSampleDocs: false,
             callIndex: 0,
+            expectedInternalPurpose: undefined,
+          });
+
+          assertResponseTelemetry('docs/chatbot', {
+            callIndex: 1,
           });
         });