(EAI-998): Low-hanging improvements to existing tracing metrics (#691)

* update tracing metrics

* clrify metric names

* code cleanup

* fix type err in tests

---------

Co-authored-by: Ben Perlmutter <mongodben@mongodb.com>

Author: mongodben

packages/chatbot-server-mongodb-public/src/tracing/extractTracingData.ts

@@ -63,6 +63,9 @@ export function extractTracingData(
     tags.push("llm_does_not_know");
   }
 
+  const rating = evalAssistantMessage?.rating;
+  const comment = evalAssistantMessage?.userComment;
+
   return {
     tags,
     rejectQuery,
@@ -71,6 +74,8 @@ export function extractTracingData(
     numRetrievedChunks,
     userMessage: previousUserMessage,
     assistantMessage: evalAssistantMessage,
+    rating,
+    comment,
   };
 }
 

packages/chatbot-server-mongodb-public/src/tracing/getLlmAsAJudgeScores.test.ts

@@ -49,6 +49,8 @@ describe("getLlmAsAJudgeScores", () => {
     llmDoesNotKnow: false,
     numRetrievedChunks: 1,
     rejectQuery: false,
+    rating: undefined,
+    comment: undefined,
   } satisfies Parameters<typeof getLlmAsAJudgeScores>[1];
 
   it("shouldn't judge verified answer", async () => {

packages/chatbot-server-mongodb-public/src/tracing/routesUpdateTraceHandlers.ts

@@ -145,13 +145,25 @@ function getTracingScores(
   k: number
 ) {
   return {
-    RejectedQuery: tracingData.rejectQuery === true ? 1 : null,
-    VerifiedAnswer: tracingData.isVerifiedAnswer === true ? 1 : null,
-    LlmDoesNotKnow: tracingData.llmDoesNotKnow === true ? 1 : null,
-    [`RetrievedChunksOver${k}`]:
-      tracingData.isVerifiedAnswer !== true
-        ? tracingData.numRetrievedChunks / k
-        : null,
+    // These metrics should start at 0,
+    // and are updated in other update trace handlers as needed
+    HasRating: tracingData.rating !== undefined ? 1 : 0,
+    HasComment: tracingData.comment !== undefined ? 1 : 0,
+    VerifiedAnswer: tracingData.isVerifiedAnswer === true ? 1 : 0,
+    // Only calculate these metrics if the answer is not verified
+    InputGuardrailPass: tracingData.isVerifiedAnswer
+      ? null
+      : tracingData.rejectQuery === true
+      ? 0
+      : 1,
+    LlmAnswerAttempted: tracingData.isVerifiedAnswer
+      ? null
+      : tracingData.llmDoesNotKnow === true
+      ? 0
+      : 1,
+    [`RetrievedChunksOver${k}`]: tracingData.isVerifiedAnswer
+      ? null
+      : tracingData.numRetrievedChunks / k,
   };
 }
 
@@ -214,7 +226,10 @@ export function makeRateMessageUpdateTrace({
     try {
       logger.updateSpan({
         id: traceId,
-        scores: await getLlmAsAJudgeScores(llmAsAJudge, tracingData),
+        scores: {
+          ...(await getLlmAsAJudgeScores(llmAsAJudge, tracingData)),
+          HasRating: 1,
+        },
       });
     } catch (error) {
       logRequest({
@@ -309,6 +324,7 @@ export function makeCommentMessageUpdateTrace({
       logger.updateSpan({
         id: traceId,
         scores: {
+          HasComment: 1,
           CommentSentiment: (
             await judgeMongoDbChatbotCommentSentiment({
               judgeLlm,

packages/mongodb-chatbot-server/src/routes/conversations/commentMessage.ts

@@ -150,9 +150,6 @@ export function makeCommentMessageRoute({
         braintrustLogger.logFeedback({
           id: traceId,
           comment,
-          scores: {
-            HasComment: 1,
-          },
         });
         await updateTraceIfExists({
           updateTrace,