(EAI-626): Send slack message on user comment to feedback channel (#601)

* add braintrust to core

* Update packages/mongodb-chatbot-server/src/routes/conversations/addMessageToConversation.ts

* working server logging instrumentation

* checkpoint

* target more recent version to use es2023

* update lib to latest

* update tracing

* lets see

* braintrust tracing in test

* no trace in test

* tracing tests

* up timeout on flaky test

* chatbot staging

* add basic LLM as a judge tracing

* tracing w/ LLM as a judge

* add judge vars to drone

* standardize custom tracing

* additional clean up

* remove unused import

* clean up

* implement NL feedback

* add user message rating eval

* remove unused tests

* trace correct route

* remove console.log

* remove trace

* working comment eval, mid refactor

* comment sentiment eval

* Get started

* working, pretty slack integration

* post to slack

* add env vars

---------

Co-authored-by: Ben Perlmutter <mongodben@mongodb.com>

Author: mongodben

package-lock.json

@@ -16,6 +16,7 @@ env:
   JUDGE_LLM: "gpt-4o-mini"
   JUDGE_EMBEDDING_MODEL: "text-embedding-3-small"
   # TODO:(EAI-644) add prod tracing
+  # SLACK_COMMENT_CONVERSATION_ID: "C07A68V63EH"
   # BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME: "chatbot-responses-prod"
 
 envSecrets:
@@ -24,6 +25,7 @@ envSecrets:
   OPENAI_API_KEY: docs-chatbot-prod
   # TODO:(EAI-644) add prod tracing
   # BRAINTRUST_TRACING_API_KEY: docs-chatbot-prod
+  # SLACK_BOT_TOKEN: docs-chatbot-prod
 
 ingress:
   enabled: true

packages/chatbot-server-mongodb-public/environments/production.yml

@@ -16,12 +16,14 @@ env:
   BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME: "chatbot-responses-staging"
   JUDGE_LLM: "gpt-4o-mini"
   JUDGE_EMBEDDING_MODEL: "text-embedding-3-small"
+  SLACK_COMMENT_CONVERSATION_ID: "C08AUU9M1AL"
 
 envSecrets:
   MONGODB_CONNECTION_URI: docs-chatbot-staging
   OPENAI_ENDPOINT: docs-chatbot-staging
   OPENAI_API_KEY: docs-chatbot-staging
   BRAINTRUST_TRACING_API_KEY: docs-chatbot-staging
+  SLACK_BOT_TOKEN: docs-chatbot-staging
 
 ingress:
   enabled: true

packages/chatbot-server-mongodb-public/environments/staging.yml

@@ -26,6 +26,7 @@
     "evaluate": "braintrust eval"
   },
   "dependencies": {
+    "@slack/web-api": "^7.8.0",
     "common-tags": "^1.8.2",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.0.3",
@@ -34,6 +35,8 @@
     "mongodb-chatbot-verified-answers": "*",
     "mongodb-rag-core": "*",
     "pm2": "^5.3.0",
+    "slack-block-builder": "^2.8.0",
+    "slackify-markdown": "^4.4.0",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.2"
   },

packages/chatbot-server-mongodb-public/package.json

@@ -224,6 +224,8 @@ const llmAsAJudgeConfig = {
   },
 };
 
+const { SLACK_BOT_TOKEN, SLACK_COMMENT_CONVERSATION_ID } = process.env;
+
 export const config: AppConfig = {
   conversationsRouterConfig: {
     llm,
@@ -244,7 +246,15 @@ export const config: AppConfig = {
     rateMessageUpdateTrace: makeRateMessageUpdateTrace(llmAsAJudgeConfig),
     commentMessageUpdateTrace: makeCommentMessageUpdateTrace(
       openAiClient,
-      JUDGE_LLM
+      JUDGE_LLM,
+      SLACK_BOT_TOKEN !== undefined &&
+        SLACK_COMMENT_CONVERSATION_ID !== undefined
+        ? {
+            token: SLACK_BOT_TOKEN,
+            conversationId: SLACK_COMMENT_CONVERSATION_ID,
+            llmAsAJudge: llmAsAJudgeConfig,
+          }
+        : undefined
     ),
     generateUserPrompt,
     systemPrompt,

packages/chatbot-server-mongodb-public/src/config.ts

@@ -0,0 +1,42 @@
+import { Message } from "mongodb-rag-core";
+import { ObjectId } from "mongodb-rag-core/mongodb";
+import { strict as assert } from "assert";
+
+export function extractSampleMessages({
+  messages,
+  targetMessageId,
+  maxNumMessagesBefore = 5,
+  maxNumMessagesAfter = 2,
+}: {
+  messages: Message[];
+  targetMessageId: ObjectId;
+  maxNumMessagesBefore?: number;
+  maxNumMessagesAfter?: number;
+}) {
+  const commentIdx = messages.findLastIndex((message) =>
+    message.id.equals(targetMessageId)
+  );
+  assert(
+    commentIdx !== -1,
+    `Comment for message with ID ${targetMessageId.toHexString()} not found in messages with a comment.`
+  );
+  const sampleMessagesStartIndex = Math.max(
+    0,
+    commentIdx - maxNumMessagesBefore
+  );
+  const sampleMessagesEndIndex = Math.min(
+    messages.length,
+    commentIdx + maxNumMessagesAfter
+  );
+  const sampleMessages = messages.slice(
+    sampleMessagesStartIndex,
+    sampleMessagesEndIndex
+  );
+  const targetMessageIndex = sampleMessages.findIndex((message) =>
+    message.id.equals(targetMessageId)
+  );
+  return {
+    sampleMessages,
+    targetMessageIndex,
+  };
+}

packages/chatbot-server-mongodb-public/src/tracing/extractSampleMessages.ts

@@ -1,10 +1,10 @@
-import { AssistantMessage, DbMessage, Message } from "mongodb-rag-core";
+import { Message } from "mongodb-rag-core";
 import { ObjectId } from "mongodb-rag-core/mongodb";
 import { OpenAI } from "mongodb-rag-core/openai";
 import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
-import { strict as assert } from "assert";
 import { wrapTraced } from "mongodb-rag-core/braintrust";
+import { extractSampleMessages } from "./extractSampleMessages";
 
 export interface CommentSentimentParams {
   judgeLlm: string;
@@ -104,29 +104,16 @@ function makeUserMessage(
   messages: Message[],
   messageWithCommentId: ObjectId
 ): OpenAI.Chat.Completions.ChatCompletionUserMessageParam {
-  const commentIdx = messages.findLastIndex(
-    (message): message is DbMessage<AssistantMessage> =>
-      message.id.equals(messageWithCommentId) && "userComment" in message
-  );
-  assert(
-    commentIdx !== -1,
-    `Comment for message with ID ${messageWithCommentId.toHexString()} not found in messages with a comment.`
-  );
-  const sampleMessagesStartIndex = Math.max(0, commentIdx - 5);
-  const sampleMessagesEndIndex = Math.min(messages.length, commentIdx + 2);
-  const sampleMessages = messages.slice(
-    sampleMessagesStartIndex,
-    sampleMessagesEndIndex
-  );
-  const sampleMessagesTargetCommentIdx = sampleMessages.findIndex((message) =>
-    message.id.equals(messageWithCommentId)
-  );
+  const { sampleMessages, targetMessageIndex } = extractSampleMessages({
+    messages,
+    targetMessageId: messageWithCommentId,
+  });
   let transcript = "Conversation Transcript:\n\n";
   sampleMessages.forEach((message, i) => {
     transcript += `${formatRole(message.role)}:\n ${message.content}\n`;
     if (message.role === "assistant" && typeof message.rating === "boolean") {
       transcript += `\nUser rating: ${message.rating}\n`;
-      if (i === sampleMessagesTargetCommentIdx) {
+      if (i === targetMessageIndex) {
         transcript += `User comment to analyze: ${message.userComment}\n`;
       }
     }

packages/chatbot-server-mongodb-public/src/tracing/mongoDbChatbotCommentSentiment.ts

@@ -0,0 +1,50 @@
+import { ObjectId } from "mongodb-rag-core/mongodb";
+import { postCommentToSlack } from "./postCommentToSlack";
+import "dotenv/config";
+describe.skip("postCommentToSlack", () => {
+  it("should post message to slack", async () => {
+    const id = new ObjectId();
+    await postCommentToSlack({
+      slackToken: process.env.SLACK_BOT_TOKEN!,
+      slackConversationId: process.env.SLACK_COMMENT_CONVERSATION_ID!,
+      conversation: {
+        _id: new ObjectId(),
+        createdAt: new Date(),
+        messages: [
+          {
+            role: "user",
+            content: "hey",
+            id: new ObjectId(),
+            createdAt: new Date(),
+          },
+          {
+            role: "assistant",
+            content: "hello",
+            rating: true,
+            userComment: "good",
+            id,
+            createdAt: new Date(),
+            references: [
+              {
+                title: "title",
+                url: "https://example.com",
+              },
+            ],
+          },
+        ],
+      },
+      messageWithCommentId: id,
+      llmAsAJudge: {
+        judgeEmbeddingModel: process.env.JUDGE_EMBEDDING_MODEL!,
+        judgeModel: process.env.JUDGE_LLM!,
+        openAiConfig: {
+          azureOpenAi: {
+            apiKey: process.env.OPENAI_API_KEY!,
+            endpoint: process.env.OPENAI_ENDPOINT!,
+            apiVersion: process.env.OPENAI_API_VERSION!,
+          },
+        },
+      },
+    });
+  });
+});

packages/chatbot-server-mongodb-public/src/tracing/postCommentToSlack.test.ts

@@ -0,0 +1,150 @@
+import { AssistantMessage, Conversation } from "mongodb-rag-core";
+import { ObjectId } from "mongodb-rag-core/mongodb";
+import { WebClient } from "@slack/web-api";
+import { Blocks, Message as BuilderMessage, Md } from "slack-block-builder";
+import { extractTracingData } from "./extractTracingData";
+import { strict as assert } from "assert";
+import slackify from "slackify-markdown";
+import { getLlmAsAJudgeScores, LlmAsAJudge } from "./getLlmAsAJudgeScores";
+
+export interface PostCommentToSlackParams {
+  slackToken: string;
+  slackConversationId: string;
+  conversation: Conversation;
+  messageWithCommentId: ObjectId;
+  llmAsAJudge: LlmAsAJudge;
+}
+export async function postCommentToSlack({
+  slackToken,
+  slackConversationId,
+  conversation,
+  messageWithCommentId,
+  llmAsAJudge,
+}: PostCommentToSlackParams) {
+  const client = new WebClient(slackToken);
+  const builder = await makeSlackMessageText(
+    conversation,
+    messageWithCommentId,
+    slackConversationId,
+    llmAsAJudge
+  );
+  const res = await client.chat.postMessage({
+    ...builder,
+    unfurl_links: false,
+    unfurl_media: false,
+  });
+  return res;
+}
+
+async function makeSlackMessageText(
+  conversation: Conversation,
+  messageWithCommentId: ObjectId,
+  slackConversationId: string,
+  llmAsAJudge: LlmAsAJudge
+) {
+  const tracingData = extractTracingData(
+    conversation.messages,
+    messageWithCommentId
+  );
+  const {
+    isVerifiedAnswer,
+    llmDoesNotKnow,
+    rejectQuery,
+    tags,
+    numRetrievedChunks,
+    assistantMessage,
+    userMessage,
+  } = tracingData;
+
+  assert(assistantMessage, "Assistant message not found");
+  assert(userMessage, "User message not found");
+  const { rating, userComment } = extractFeedback(assistantMessage);
+  assert(rating, "Rating not found");
+  assert(userComment, "User comment not found");
+
+  const scores = await getLlmAsAJudgeScores(llmAsAJudge, tracingData);
+
+  const feedbackMd = `${Md.bold(
+    rating === true ? "👍 Positive Feedback" : "👎 Negative Feedback"
+  )}
+
+${Md.bold("User Comment:")}
+${userComment}`;
+
+  const messagesMd = `${Md.blockquote(
+    `${Md.bold(Md.emoji("bust_in_silhouette") + ` User`)}`
+  )}
+
+${slackify(fixNewLines(userMessage.content))}
+
+${`${Md.blockquote(Md.bold(Md.emoji("robot_face") + ` Assistant`))}`}
+
+${slackify(fixNewLines(assistantMessage.content))}`;
+
+  const referencesMd =
+    Md.bold("References:") +
+    "\n" +
+    (assistantMessage.references && assistantMessage?.references.length > 0
+      ? `${assistantMessage.references
+          .map((ref) => {
+            return `${Md.listBullet(Md.link(ref.url, ref.title))}`;
+          })
+          .join("\n")}`
+      : "No References");
+
+  const messageMetadataMd = `${Md.bold("Metadata:")}
+
+${Md.listBullet(`Verified Answer: ${isVerifiedAnswer ? "Yes" : "No"}`)}
+${Md.listBullet(`LLM Does Not Know: ${llmDoesNotKnow ? "Yes" : "No"}`)}
+${Md.listBullet(`Rejected Query: ${rejectQuery ? "Yes" : "No"}`)}
+${Md.listBullet(`Number of Retrieved Chunks: ${numRetrievedChunks}`)}
+${Md.listBullet(`Tags: ${tags.map(Md.codeInline).join(", ")}`)}`;
+
+  const idMetadataMd = `Conversation ID: ${Md.codeInline(
+    conversation._id.toHexString()
+  )}
+Message ID/ Braintrust Trace ID: ${Md.codeInline(
+    messageWithCommentId.toHexString()
+  )}`;
+
+  const scoresMd = `${Md.bold("Scores:")}
+${
+  scores
+    ? Object.entries(scores)
+        .map(([label, score]) => {
+          return `${Md.listBullet(`${Md.bold(label)}: ${score}`)}`;
+        })
+        .join("\n")
+    : "No LLM-as-Judge Scores"
+}`;
+
+  return BuilderMessage({
+    channel: slackConversationId,
+    text: "User Feedback",
+  })
+    .blocks(
+      Blocks.Section({ text: feedbackMd }),
+      Blocks.Divider(),
+      Blocks.Section({ text: messagesMd }),
+      Blocks.Section({ text: referencesMd }),
+      Blocks.Divider(),
+      Blocks.Section({ text: scoresMd }),
+      Blocks.Divider(),
+      Blocks.Section({ text: messageMetadataMd }),
+      Blocks.Divider(),
+      Blocks.Section({ text: idMetadataMd })
+    )
+    .asUser()
+    .buildToObject();
+}
+
+function fixNewLines(text: string) {
+  return text.replaceAll("\\n", "\n");
+}
+
+function extractFeedback(assistantMessage: AssistantMessage) {
+  return {
+    rating: assistantMessage.rating,
+    userComment: assistantMessage.userComment,
+  };
+}