fix: prevent agent loops when ReadMediaFile is called on text files (#1218)

packages/agent-core/src/agent/turn/tool-dedup.ts

@@ -40,6 +40,27 @@ const REPEAT_REMINDER_2_START = 5;
 const REPEAT_REMINDER_3_START = 8;
 const REPEAT_FORCE_STOP_STREAK = 12;
 
+// ── Wrong-tool error detection ────────────────────────────────────────
+//
+// When a tool error is *categorical* — the model fundamentally used the
+// wrong tool for the task (e.g. ReadMediaFile on a text file) — retries
+// are never productive. These errors escalate faster than generic
+// transient failures.
+
+const WRONG_TOOL_ERROR_PATTERNS = [
+  /is a text file/i,
+  /is not a supported image or video/i,
+  /is not a supported image and video/i,
+  /cannot read text files/i,
+  /only reads image/i,
+  /only works with images and videos/i,
+  /must use the read tool/i,
+];
+
+/** Faster escalation for wrong-tool errors. */
+const WRONG_TOOL_REMINDER_2_START = 2;
+const WRONG_TOOL_FORCE_STOP_STREAK = 4;
+
 interface Deferred<T> {
   readonly promise: Promise<T>;
   resolve(value: T): void;
@@ -85,6 +106,29 @@ function forceStopResult(
   return { ...withReminder, stopTurn: true };
 }
 
+/**
+ * Returns the text content of a tool result, collapsing ContentPart[]
+ * arrays into a single string for pattern matching.
+ */
+function toolOutputText(output: ExecutableToolResult['output']): string {
+  if (typeof output === 'string') return output;
+  return output
+    .filter(
+      (p): p is Extract<ContentPart, { type: 'text' }> => p.type === 'text',
+    )
+    .map((p) => p.text)
+    .join('');
+}
+
+/**
+ * Detects tool errors that indicate the model used the wrong tool for the
+ * task — category errors that will never succeed on retry.
+ */
+function isWrongToolError(output: ExecutableToolResult['output']): boolean {
+  const text = toolOutputText(output);
+  return WRONG_TOOL_ERROR_PATTERNS.some((pattern) => pattern.test(text));
+}
+
 /**
  * Placeholder result returned from `checkSameStep` for a duplicate call. Never
  * reaches the model — it is replaced in `finalizeResult` by awaiting the
@@ -107,10 +151,14 @@ const DEDUP_PLACEHOLDER_RESULT: ExecutableToolResult = { output: '' };
  *   reminder once the streak hits 3. The reminder escalates as the streak
  *   grows: r1 (gentle nudge) from streak 3, r2 (concrete repeat report) from
  *   streak 5, r3 (dead-end stop instruction) from streak 8. From streak 12
- *   onward the turn is force-stopped via `{ stopTurn: true }` so the loop
- *   cannot keep spinning on the same call. Force-stop does not flip a
- *   successful tool result into an error — the underlying tool's `isError`
- *   is preserved.
+ *   onward the turn is force-stopped via `{ stopTurn: true }`.
+ *
+ *   For *wrong-tool* errors (e.g. ReadMediaFile on a text file), escalation
+ *   is faster: r2 from streak 2 and force-stop from streak 4, because
+ *   retrying the wrong tool never helps.
+ *
+ *   Force-stop does not flip a successful tool result into an error — the
+ *   underlying tool's `isError` is preserved.
  *
  * Telemetry: every finalized original call with streak >= 2 emits a
  * `tool_call_repeat` event carrying the current streak count as `repeat_count`
@@ -231,8 +279,25 @@ export class ToolCallDeduplicator {
     }
 
     let finalResult = result;
-    let action: 'none' | 'r1' | 'r2' | 'r3' | 'stop' = 'none';
-    if (streak >= REPEAT_FORCE_STOP_STREAK) {
+    let action: 'none' | 'r1' | 'r2' | 'r2_early' | 'r3' | 'stop' = 'none';
+
+    // Wrong-tool errors (category errors that retries will never fix)
+    // escalate faster than generic transient failures.
+    // Only trigger on actual tool errors (isError === true) — successful
+    // output that coincidentally contains one of these phrases must not be
+    // fast-tracked.
+    if (result.isError === true && isWrongToolError(result.output)) {
+      if (streak >= WRONG_TOOL_FORCE_STOP_STREAK) {
+        finalResult = forceStopResult(result, REMINDER_TEXT_3);
+        action = 'stop';
+      } else if (streak >= WRONG_TOOL_REMINDER_2_START) {
+        finalResult = appendReminder(
+          result,
+          makeReminderText2(toolName, streak, args),
+        );
+        action = 'r2_early';
+      }
+    } else if (streak >= REPEAT_FORCE_STOP_STREAK) {
       finalResult = forceStopResult(result, REMINDER_TEXT_3);
       action = 'stop';
     } else if (streak >= REPEAT_REMINDER_3_START) {
@@ -267,4 +332,8 @@ export const __testing = {
   REPEAT_REMINDER_2_START,
   REPEAT_REMINDER_3_START,
   REPEAT_FORCE_STOP_STREAK,
+  WRONG_TOOL_REMINDER_2_START,
+  WRONG_TOOL_FORCE_STOP_STREAK,
+  toolOutputText,
+  isWrongToolError,
 };

packages/agent-core/src/tools/builtin/file/read-media.md

@@ -1,13 +1,14 @@
+**CRITICAL: This tool ONLY reads image files (PNG, JPG, GIF, WebP, etc.) and video files (MP4, MOV, AVI, etc.). It CANNOT read text files — never pass a .md, .ts, .json, .txt, or any other text file path to this tool. For text files, use the Read tool. To list directories, use `ls` via Bash for a known directory, or Glob for pattern search.**
+
 Read media content from a file.
 
 **Tips:**
 - Make sure you follow the description of each tool parameter.
 - A `<system>` tag is given before the file content; it summarizes the mime type, byte size and, for images, the original pixel dimensions. When outputting coordinates, give relative coordinates first and compute absolute coordinates from the original image size. After generating or editing media via commands or scripts, read the result back before continuing.
 - The system will notify you when there is anything wrong when reading the file.
 - This tool is a tool that you typically want to use in parallel. Always read multiple files in one response when possible.
-- This tool can only read image or video files. To read text files, use the Read tool. To list directories, use `ls` via Bash for a known directory, or Glob for pattern search.
 - If the file doesn't exist or path is invalid, an error will be returned.
 - The maximum size that can be read is {{ MAX_MEDIA_MEGABYTES }}MB. An error will be returned if the file is larger than this limit.
 - The media content will be returned in a form that you can directly view and understand.
 
-**Capabilities**
+**Capabilities**

packages/agent-core/src/tools/builtin/file/read-media.ts

@@ -50,9 +50,10 @@ export const ReadMediaFileInputSchema = z.object({
   path: z
     .string()
     .describe(
-      'Path to an image or video file. Relative paths resolve against the working directory; ' +
+      'Path to an IMAGE or VIDEO file (e.g., .png, .jpg, .mp4). ' +
+        'Relative paths resolve against the working directory; ' +
         'a path outside the working directory must be absolute. ' +
-        'Directories and text files are not supported.',
+        'Text files (e.g., .md, .ts, .txt, .json) are REJECTED — use Read instead.',
     ),
 });
 
@@ -179,15 +180,20 @@ export class ReadMediaFileTool implements BuiltinTool<ReadMediaFileInput> {
       if (fileType.kind === 'text') {
         return {
           isError: true,
-          output: `"${args.path}" is a text file. Use Read to read text files.`,
+          output:
+            `"${args.path}" is a TEXT FILE, not an image or video. ` +
+            `ReadMediaFile CANNOT read text files. ` +
+            `You MUST use the Read tool to read this file. ` +
+            `Do NOT call ReadMediaFile for this file again.`,
         };
       }
       if (fileType.kind === 'unknown') {
         return {
           isError: true,
           output:
-            `"${args.path}" is not a supported image or video file. ` +
-            'Use Read for text files, or Bash or an MCP tool for other binary formats.',
+            `"${args.path}" is NOT a supported image or video file. ` +
+            `ReadMediaFile ONLY works with images and videos. ` +
+            'Use the Read tool for text files, or Bash / an MCP tool for other binary formats.',
         };
       }