chore(chat): Report response telemetry VSCODE-603 (#845)

Author: nirinchev

src/participant/participant.ts

@@ -190,14 +190,21 @@ export default class ParticipantController {
     modelInput: ModelInput;
     stream: vscode.ChatResponseStream;
     token: vscode.CancellationToken;
-  }): Promise<void> {
+  }): Promise<{ outputLength: number }> {
     const chatResponse = await this._getChatResponse({
       modelInput,
       token,
     });
+
+    let length = 0;
     for await (const fragment of chatResponse.text) {
       stream.markdown(fragment);
+      length += fragment.length;
     }
+
+    return {
+      outputLength: length,
+    };
   }
 
   _streamCodeBlockActions({
@@ -236,22 +243,34 @@ export default class ParticipantController {
     modelInput: ModelInput;
     stream: vscode.ChatResponseStream;
     token: vscode.CancellationToken;
-  }): Promise<void> {
+  }): Promise<{
+    outputLength: number;
+    hasCodeBlock: boolean;
+  }> {
     const chatResponse = await this._getChatResponse({
       modelInput,
       token,
     });
 
+    let outputLength = 0;
+    let hasCodeBlock = false;
     await processStreamWithIdentifiers({
       processStreamFragment: (fragment: string) => {
         stream.markdown(fragment);
+        outputLength += fragment.length;
       },
       onStreamIdentifier: (content: string) => {
         this._streamCodeBlockActions({ runnableContent: content, stream });
+        hasCodeBlock = true;
       },
       inputIterable: chatResponse.text,
       identifier: codeBlockIdentifier,
     });
+
+    return {
+      outputLength,
+      hasCodeBlock,
+    };
   }
 
   // This will stream all of the response content and create a string from it.
@@ -287,10 +306,19 @@ export default class ParticipantController {
       connectionNames: this._getConnectionNames(),
     });
 
-    await this.streamChatResponseContentWithCodeActions({
-      modelInput,
-      token,
-      stream,
+    const { hasCodeBlock, outputLength } =
+      await this.streamChatResponseContentWithCodeActions({
+        modelInput,
+        token,
+        stream,
+      });
+
+    this._telemetryService.trackCopilotParticipantResponse({
+      command: 'generic',
+      has_cta: false,
+      found_namespace: false,
+      has_runnable_content: hasCodeBlock,
+      output_length: outputLength,
     });
 
     return genericRequestChatResult(context.history);
@@ -995,6 +1023,7 @@ export default class ParticipantController {
       context,
       token,
     });
+
     if (!databaseName || !collectionName) {
       return await this._askForNamespace({
         command: '/schema',
@@ -1056,7 +1085,7 @@ export default class ParticipantController {
       connectionNames: this._getConnectionNames(),
       ...(sampleDocuments ? { sampleDocuments } : {}),
     });
-    await this.streamChatResponse({
+    const response = await this.streamChatResponse({
       modelInput,
       stream,
       token,
@@ -1072,6 +1101,14 @@ export default class ParticipantController {
       ],
     });
 
+    this._telemetryService.trackCopilotParticipantResponse({
+      command: 'schema',
+      has_cta: true,
+      found_namespace: true,
+      has_runnable_content: false,
+      output_length: response.outputLength,
+    });
+
     return schemaRequestChatResult(context.history);
   }
 
@@ -1160,10 +1197,19 @@ export default class ParticipantController {
       ...(sampleDocuments ? { sampleDocuments } : {}),
     });
 
-    await this.streamChatResponseContentWithCodeActions({
-      modelInput,
-      stream,
-      token,
+    const { hasCodeBlock, outputLength } =
+      await this.streamChatResponseContentWithCodeActions({
+        modelInput,
+        stream,
+        token,
+      });
+
+    this._telemetryService.trackCopilotParticipantResponse({
+      command: 'query',
+      has_cta: false,
+      found_namespace: true,
+      has_runnable_content: hasCodeBlock,
+      output_length: outputLength,
     });
 
     return queryRequestChatResult(context.history);
@@ -1239,11 +1285,12 @@ export default class ParticipantController {
       connectionNames: this._getConnectionNames(),
     });
 
-    await this.streamChatResponseContentWithCodeActions({
-      modelInput,
-      stream,
-      token,
-    });
+    const { hasCodeBlock, outputLength } =
+      await this.streamChatResponseContentWithCodeActions({
+        modelInput,
+        stream,
+        token,
+      });
 
     this._streamResponseReference({
       reference: {
@@ -1252,6 +1299,14 @@ export default class ParticipantController {
       },
       stream,
     });
+
+    this._telemetryService.trackCopilotParticipantResponse({
+      command: 'docs/copilot',
+      has_cta: true,
+      found_namespace: false,
+      has_runnable_content: hasCodeBlock,
+      output_length: outputLength,
+    });
   }
 
   _streamResponseReference({
@@ -1307,6 +1362,14 @@ export default class ParticipantController {
       if (docsResult.responseContent) {
         stream.markdown(docsResult.responseContent);
       }
+
+      this._telemetryService.trackCopilotParticipantResponse({
+        command: 'docs/chatbot',
+        has_cta: !!docsResult.responseReferences,
+        found_namespace: false,
+        has_runnable_content: false,
+        output_length: docsResult.responseContent?.length ?? 0,
+      });
     } catch (error) {
       // If the docs chatbot API is not available, fall back to Copilot’s LLM and include
       // the MongoDB documentation link for users to go to our documentation site directly.

src/telemetry/telemetryService.ts

@@ -119,6 +119,14 @@ export type ParticipantPromptProperties = {
   internal_purpose: InternalPromptPurpose;
 };
 
+export type ParticipantResponseProperties = {
+  command: string;
+  has_cta: boolean;
+  has_runnable_content: boolean;
+  found_namespace: boolean;
+  output_length: number;
+};
+
 export function chatResultFeedbackKindToTelemetryValue(
   kind: vscode.ChatResultFeedbackKind
 ): TelemetryFeedbackKind {
@@ -148,7 +156,9 @@ type TelemetryEventProperties =
   | SavedConnectionsLoadedProperties
   | SurveyActionProperties
   | ParticipantFeedbackProperties
-  | ParticipantResponseFailedProperties;
+  | ParticipantResponseFailedProperties
+  | ParticipantPromptProperties
+  | ParticipantResponseProperties;
 
 export enum TelemetryEventTypes {
   PLAYGROUND_CODE_EXECUTED = 'Playground Code Executed',
@@ -172,6 +182,7 @@ export enum TelemetryEventTypes {
   PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown',
   PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed',
   PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted',
+  PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated',
 }
 
 export enum ParticipantErrorTypes {
@@ -438,4 +449,8 @@ export default class TelemetryService {
   trackCopilotParticipantPrompt(stats: ParticipantPromptProperties): void {
     this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats);
   }
+
+  trackCopilotParticipantResponse(props: ParticipantResponseProperties): void {
+    this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props);
+  }
 }