fix: set correct model for custom analysis

Currently we're using the same model for the custom categories as for the eval itself. This is incorrect because the model may not be available.

These changes add the option to set the model and default to Gemini 2.5 Flash Lite.

Author: crisbeto

runner/configuration/constants.ts

@@ -17,6 +17,9 @@ export const DEFAULT_MODEL_NAME = 'gemini-2.5-pro'; // slower than `flash`, but
  */
 export const DEFAULT_AUTORATER_MODEL_NAME = 'gemini-2.5-flash'; // use less expensive model
 
+/** Model used for AI summarization by default. */
+export const DEFAULT_SUMMARY_MODEL = 'gemini-2.5-flash-lite';
+
 /** Name of the root folder where we store LLM-generated code for debugging */
 export const LLM_OUTPUT_DIR = join(rootDir, 'llm-output');
 

runner/configuration/environment-config.ts

@@ -108,6 +108,7 @@ export const environmentConfigSchema = z.object({
       z.object({
         name: z.string(),
         path: z.string(),
+        model: z.string().optional(),
         reportsFilter: z
           .enum([ReportContextFilter.AllReports, ReportContextFilter.NonPerfectReports])
           .optional(),

runner/configuration/environment.ts

@@ -18,6 +18,7 @@ import {EnvironmentConfig} from './environment-config.js';
 import {EvalPromptWithMetadata, MultiStepPrompt} from './prompts.js';
 import {renderPromptTemplate} from './prompt-templating.js';
 import {getSha256Hash} from '../utils/hashing.js';
+import {DEFAULT_SUMMARY_MODEL} from './constants.js';
 
 interface CategoryConfig {
   name: string;
@@ -27,6 +28,7 @@ interface CategoryConfig {
 interface AnalysisPrompt {
   name: string;
   prompt: string;
+  model: string;
   reportsFilter: ReportContextFilter;
   ratingsFilter: RatingContextFilter;
 }
@@ -463,12 +465,13 @@ export class Environment {
   private resolveAnalysisPrompts(config: EnvironmentConfig): AnalysisPrompt[] {
     const result: AnalysisPrompt[] = [];
 
-    config.analysisPrompts?.forEach(({name, path, reportsFilter, ratingsFilter}) => {
+    config.analysisPrompts?.forEach(({name, path, model, reportsFilter, ratingsFilter}) => {
       const prompt = this.renderEnvironmentPrompt(path).result;
 
       result.push({
         name,
         prompt,
+        model: model || DEFAULT_SUMMARY_MODEL,
         reportsFilter: reportsFilter ?? ReportContextFilter.NonPerfectReports,
         ratingsFilter: ratingsFilter ?? RatingContextFilter.NonPerfectRatings,
       });

runner/orchestration/generate-summary.ts

@@ -12,7 +12,7 @@ import {AssessmentResult, CompletionStats, RunSummary} from '../shared-interface
 export async function prepareSummary(
   generateAiSummaryLlm: GenkitRunner | null,
   abortSignal: AbortSignal,
-  model: string,
+  evalRunModel: string,
   env: Environment,
   assessments: AssessmentResult[],
   completionStats: CompletionStats,
@@ -75,7 +75,7 @@ export async function prepareSummary(
             abortSignal,
             assessments,
             [],
-            model,
+            config.model,
             {
               reportContextFilter: config.reportsFilter,
               ratingContextFilter: config.ratingsFilter,
@@ -101,7 +101,7 @@ export async function prepareSummary(
   const executorInfo = await env.executor.getExecutorInfo?.();
 
   return {
-    model,
+    model: evalRunModel,
     environmentId: env.id,
     displayName: env.displayName,
     framework: {

runner/reporting/report-ai-summary.ts

@@ -1,4 +1,5 @@
 import {GenkitRunner} from '../codegen/genkit/genkit-runner.js';
+import {DEFAULT_SUMMARY_MODEL} from '../configuration/constants.js';
 import {AssessmentResult, ReportContextFilter, RatingContextFilter} from '../shared-interfaces.js';
 import {chatWithReportAI} from './report-ai-chat.js';
 
@@ -7,7 +8,7 @@ export async function summarizeReportWithAI(
   abortSignal: AbortSignal,
   assessments: AssessmentResult[],
 ) {
-  const model = 'gemini-2.5-flash-lite';
+  const model = DEFAULT_SUMMARY_MODEL;
 
   if (!llm.getSupportedModels().includes(model)) {
     throw new Error(`Unable to generate AI summary due to unsupported model: ${model}`);