(EAI-1119) add category, clearer logging, cron runs on dev db too (#838)

* add category, clearer logging, cron runs on dev db too

* better variable name and description

Author: yakubova92

packages/scripts/environments/production.yml

@@ -57,7 +57,7 @@ cronJobs:
     backoffLimit: 2
 
   - name: process-citations-for-metrics
-    schedule: "0 3 * * *" # every day at 3 AM UTC
+    schedule: "1 0 * * *" # every day at 00:01 (12:01 AM) UTC
     command: ["npm", "run", "scripts:getLLMAnswers"]
     env:
       MONGODB_DATABASE_NAME: docs-chatbot-prod

packages/scripts/environments/staging.yml

@@ -86,6 +86,27 @@ cronJobs:
         memory: 5Gi
     backoffLimit: 2
 
+  - name: process-citations-for-metrics-dev
+    schedule: "30 0 * * *" # every day at 00:30 (12:30 AM) UTC
+    command: ["npm", "run", "scripts:getLLMAnswers"]
+    env:
+      MONGODB_DATABASE_NAME: docs-chatbot-dev
+      PROFOUND_API_URL: https://api.tryprofound.com/v1
+      BRAINTRUST_ENDPOINT: https://api.braintrust.dev/v1/proxy
+    envSecrets:
+      MONGODB_CONNECTION_URI: docs-chatbot-scripts-admin-dev
+      PROFOUND_API_KEY: docs-chatbot-staging
+      PROFOUND_CATALOG_ID_EDU: docs-chatbot-staging
+      BRAINTRUST_API_KEY: docs-chatbot-staging
+    resources:
+      requests:
+        cpu: 100m
+        memory: 2Gi
+      limits:
+        cpu: 500m
+        memory: 5Gi
+    backoffLimit: 2
+
 # Alerts
 defaultAlerts:
   enabled: true

packages/scripts/src/profound/getAndProcessAnswers.ts

@@ -75,6 +75,9 @@ interface CaseByProfoundPromptId {
     expected: string;
     tags: string[];
     caseId: ObjectId;
+    metadata: {
+      category: string;
+    }
   };
 }
 const casesByPromptId = async (
@@ -86,6 +89,7 @@ const casesByPromptId = async (
       expected: doc.expected,
       tags: doc.tags,
       caseId: doc._id,
+      metadata: doc.metadata,
     };
     return map;
   }, {} as CaseByProfoundPromptId);
@@ -189,6 +193,7 @@ export const main = async (startDateArg?: string, endDateArg?: string) => {
   };
   const referenceAlignmentFn = makeReferenceAlignment(openAiClient, config);
   const answerRecords: any[] = [];
+  const promptsWithNoAssociatedCase = new Set()
   const { results, errors } = await PromisePool.for(answers)
     .withConcurrency(model.maxConcurrency ?? 5)
     .process(async (currentAnswer) => {
@@ -204,7 +209,7 @@ export const main = async (startDateArg?: string, endDateArg?: string) => {
       const currentPromptId = currentAnswer.prompt_id;
       const currentCase = casesByPromptMap[currentPromptId];
       if (!currentCase) {
-        console.log(`No case found for ${currentPrompt}`);
+        promptsWithNoAssociatedCase.add(`${currentPromptId} - ${currentPrompt}`);
       }
 
       // calculate reference alignment score
@@ -239,8 +244,8 @@ export const main = async (startDateArg?: string, endDateArg?: string) => {
       } catch (err) {
         console.error("Error in referenceAlignmentFn:", {
           prompt: currentAnswer.prompt,
-          response: currentAnswer.response,
-          expected: currentCase?.expected,
+          profoundPromptId: currentAnswer.prompt_id,
+          profoundRunId: currentAnswer.run_id,
           error: err,
         });
         referenceAlignment = {
@@ -283,12 +288,17 @@ export const main = async (startDateArg?: string, endDateArg?: string) => {
         expectedResponse: currentCase?.expected,
         profoundPromptId: currentAnswer.prompt_id,
         profoundRunId: currentAnswer.run_id,
-        dataset: currentCase ? getDataset(currentCase.tags, datasetsByTagMap) : null
+        dataset: currentCase ? getDataset(currentCase.tags, datasetsByTagMap) : null,
+        category: currentCase ? currentCase.metadata.category : null
       };
       answerRecords.push(answerEngineRecord);
       return answerEngineRecord;
     });
 
+  console.log(`Found ${promptsWithNoAssociatedCase.size} prompts with no associated case:`)
+  promptsWithNoAssociatedCase.forEach((promptInfo: any) => {
+    console.log(` - ${promptInfo}`);
+  });
   // update the llm_answers collection
   if (answerRecords.length > 0) {
     const bulkOps = answerRecords.map((record) => ({
@@ -306,7 +316,7 @@ export const main = async (startDateArg?: string, endDateArg?: string) => {
       const inserted = result.upsertedCount || 0;
       const updated = result.modifiedCount || 0;
       console.log(
-        `BulkWrite to llm_answers collection completed: ${inserted} inserted, ${updated} updated (out of ${answerRecords.length} records).`
+        `BulkWrite to llm_answers collection completed: ${inserted} inserted, ${updated} updated (out of ${answerRecords.length} records between ${start.toISOString()} and ${end.toISOString()}).`
       );
     } catch (err) {
       console.error("BulkWrite to llm_answers collection failed:", err);