diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 9eb57abaea15..4e285964be2a 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1,6 +1,7 @@ #include "server-tools.h" #include +#include "base64.hpp" #include #include @@ -1061,6 +1062,107 @@ struct server_tool_get_datetime : server_tool { } }; +// +// read_media: read a media file (image or audio) and return base64-encoded data with metadata +// + +static constexpr size_t SERVER_TOOL_READ_MEDIA_MAX_SIZE = 16 * 1024 * 1024; // 16 MB +static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_FILE = "File: "; +static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_SIZE = "Size: "; +static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_MIME = "MIME: "; + +static std::string get_mime_from_extension(const std::string & path) { + static const std::unordered_map mime_map = { + // Images + {".png", "image/png"}, + {".jpg", "image/jpeg"}, + {".jpeg", "image/jpeg"}, + {".webp", "image/webp"}, + {".bmp", "image/bmp"}, + {".tiff", "image/tiff"}, + {".tif", "image/tiff"}, + {".gif", "image/gif"}, + // Audio + {".mp3", "audio/mpeg"}, + {".wav", "audio/wav"}, + {".ogg", "audio/ogg"}, + {".flac", "audio/flac"}, + {".m4a", "audio/mp4"}, + {".opus", "audio/opus"}, + }; + auto ext = fs::path(path).extension().string(); + auto it = mime_map.find(ext); + return (it != mime_map.end()) ? it->second : "application/octet-stream"; +} + +struct server_tool_read_media : server_tool { + server_tool_read_media() { + name = "read_media"; + display_name = "Read media file"; + permission_write = false; + } + + json get_definition() const override { + return { + {"type", "function"}, + {"function", { + {"name", name}, + {"description", "Read a media file (audio, image) from disk and return it as base64-encoded data with metadata."}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"path", {{"type", "string"}, {"description", "Absolute path to the media file."}}}, + }}, + {"required", json::array({"path"})}, + }}, + }}, + }; + } + + json invoke(json params, server_tool::stream *) const override { + std::string path = params.at("path").get(); + + auto io = make_tools_io(params); + + uintmax_t file_size = 0; + if (!io->file_size(path, file_size)) { + return {{"error", "cannot stat file: " + path}}; + } + if (file_size > SERVER_TOOL_READ_MEDIA_MAX_SIZE) { + return {{"error", string_format( + "media file too large (%zu bytes, max %zu)", + (size_t)file_size, SERVER_TOOL_READ_MEDIA_MAX_SIZE)}}; + } + + std::string content; + if (!io->read_file(path, content)) { + return {{"error", "failed to open file: " + path}}; + } + + std::string mime = get_mime_from_extension(path); + std::string b64 = base64::encode(content.data(), content.size()); + + // Return as plain_text_response with a data URI line so the UI can + // extract it as an attachment (via extractBase64Attachments) + // and replace it with some placeholder + std::string data_uri = "data:" + mime + ";base64," + b64; + + return { + {"plain_text_response", + string_format( + "%s%s\n%s%zu bytes\n%s%s\n%s", + SERVER_TOOL_READ_MEDIA_PREFIX_FILE, path.c_str(), + SERVER_TOOL_READ_MEDIA_PREFIX_SIZE, (size_t)file_size, + SERVER_TOOL_READ_MEDIA_PREFIX_MIME, mime.c_str(), + data_uri.c_str())}, + {"path", path}, + {"mime", mime}, + {"size_bytes", (int)file_size}, + }; + } +}; + + struct server_tool_stream_result : server_task_result { std::string chunk; bool done = false; @@ -1114,6 +1216,7 @@ static server_tool & find_tool(std::vector> & tools throw std::invalid_argument(string_format("unknown tool \"%s\"", name.c_str())); } +// // // public API // @@ -1127,6 +1230,7 @@ static std::vector> build_tools() { tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); + tools.push_back(std::make_unique()); return tools; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index b1daedfc818c..56c571223fb4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -14,6 +14,7 @@ import ChatMessageToolCallBlockGetDatetime from './ChatMessageToolCallBlockGetDatetime.svelte'; import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte'; import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte'; + import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte'; import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte'; import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte'; import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte'; @@ -42,6 +43,8 @@ {:else if section.toolName === BuiltInTool.READ_FILE} +{:else if section.toolName === BuiltInTool.READ_MEDIA} + {:else if section.toolName === BuiltInTool.EDIT_FILE} {:else if section.toolName === BuiltInTool.WRITE_FILE} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index acf2de12ac64..92cd66149e52 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -10,7 +10,7 @@ import { classifyToolResult, formatJsonPretty, - parseToolResultWithImages, + parseToolResultWithMedia, type AgenticSection, type ToolResultLine } from '$lib/utils'; @@ -31,7 +31,7 @@ const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? ''); const parsedLines: ToolResultLine[] = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); const outputKind = $derived(classifyToolResult(section.toolResult)); @@ -105,14 +105,23 @@
{line.text}
- {#if line.image} - {line.image.name} - {/if} + {#if line.media} + {#if line.media.type === 'AUDIO'} +
+ +
+ {:else} + {line.media.name} + {/if} + {/if} {/each} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte index 5de801d39a48..00c422244f93 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte @@ -16,7 +16,7 @@ isExitCodeSummaryLine, parseExecShellCommandError, parseExecShellCommandExitStatus, - parseToolResultWithImages, + parseToolResultWithMedia, type AgenticSection, type ExecShellExitStatus, type ToolResultLine @@ -51,7 +51,7 @@ ); const parsedLines: ToolResultLine[] = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); // Drop the trailing "[exit code: N]" line - rendered as a colored @@ -200,10 +200,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.image} + {#if line.media} {line.image.name} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte new file mode 100644 index 000000000000..38498bacef12 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -0,0 +1,93 @@ + + + + {#snippet titleSnippet()} + Read media + {readMediaMeta?.fileName} + {/snippet} + + {#snippet children(_meta, _ctx)} + {#if section.toolResult} + {#if mediaAttachment} + {#if isAudio} +
+ +
+ {:else} +
+ {readMediaMeta?.fileName +
+ {/if} + {:else} +
+ Media attachment not found in message extras +
+ {/if} + + {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType} +
+ {#if readMediaMeta?.sizeBytes} + Size: {readMediaMeta.sizeBytes} bytes + {/if} + {#if readMediaMeta?.mimeType} + MIME: {readMediaMeta.mimeType} + {/if} +
+ {/if} + + {#if readMediaMeta?.path} +
{readMediaMeta.path}
+ {/if} + {:else} +
+ Waiting for media data... +
+ {/if} + {/snippet} + diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts new file mode 100644 index 000000000000..cc3fb634258c --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts @@ -0,0 +1,48 @@ +import type { AgenticSection } from '$lib/utils'; +import { NEWLINE } from '$lib/constants/code'; +import { PREFIX_FILE, PREFIX_SIZE, PREFIX_MIME } from '$lib/constants/read-media'; + +export interface ReadMediaMeta { + fileName: string; + path: string; + sizeBytes?: number; + mimeType?: string; +} + +/** + * Parse read_media tool result to extract metadata. + * Expected format (after extractBase64Attachments processing): + * File: /path/to/file.png + * Size: 12345 bytes + * MIME: image/png + * [Attachment saved: mcp-attachment-xxx.png] + * + * The data URI line is replaced by the attachment marker by + * agenticStore.extractBase64Attachments before storage. + */ +export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null { + if (!section.toolResult) return null; + + const lines = section.toolResult.split(NEWLINE); + let fileName = ''; + let path = ''; + let sizeBytes: number | undefined; + let mimeType: string | undefined; + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith(PREFIX_FILE)) { + path = trimmed.slice(PREFIX_FILE.length).trim(); + fileName = path.split('/').pop() ?? path; + } else if (trimmed.startsWith(PREFIX_SIZE)) { + const match = trimmed.match(new RegExp(`${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`)); + if (match) sizeBytes = parseInt(match[1], 10); + } else if (trimmed.startsWith(PREFIX_MIME)) { + mimeType = trimmed.slice(PREFIX_MIME.length).trim(); + } + } + + if (!path) return null; + + return { fileName, path, sizeBytes, mimeType }; +} diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index 0721e6484d62..da58923c2681 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -11,6 +11,7 @@ import type { Component } from 'svelte'; import { Braces, Clock, + Eye, FilePen, FilePlus, FileSearch, @@ -28,6 +29,7 @@ export interface BuiltinToolUiEntry { export const BUILTIN_TOOL_UI: Readonly> = { [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.BUILTIN }, [BuiltInTool.EDIT_FILE]: { icon: FilePen, label: 'Edit file', source: ToolSource.BUILTIN }, [BuiltInTool.WRITE_FILE]: { icon: FilePlus, label: 'Write file', source: ToolSource.BUILTIN }, [BuiltInTool.FILE_GLOB_SEARCH]: { diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index 44419012d16f..66f07c51120d 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -1,7 +1,8 @@ -import { MimeTypeImage } from '$lib/enums'; +import { MimeTypeAudio, MimeTypeImage } from '$lib/enums'; // File extension patterns for resource type detection export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i; +export const AUDIO_FILE_EXTENSION_REGEX = /\.(mp3|wav|ogg|flac|m4a|opus)$/i; export const CODE_FILE_EXTENSION_REGEX = /\.(js|ts|json|yaml|yml|xml|html|css|py|rs|go|java|cpp|c|h|rb|sh|toml)$/i; export const TEXT_FILE_EXTENSION_REGEX = /\.(txt|md|log)$/i; @@ -53,3 +54,21 @@ export const IMAGE_MIME_TO_EXTENSION: Record = { [MimeTypeImage.GIF]: 'gif', [MimeTypeImage.WEBP]: 'webp' } as const; + +/** + * Mapping from audio MIME types to file extensions. + * Used for generating attachment filenames from MIME types. + */ +export const AUDIO_MIME_TO_EXTENSION: Record = { + [MimeTypeAudio.MP3_MPEG]: 'mp3', + [MimeTypeAudio.MP3]: 'mp3', + [MimeTypeAudio.MP4]: 'm4a', + [MimeTypeAudio.WAV]: 'wav', + [MimeTypeAudio.WAVE]: 'wav', + [MimeTypeAudio.X_WAV]: 'wav', + [MimeTypeAudio.X_WAVE]: 'wav', + [MimeTypeAudio.VND_WAVE]: 'wav', + [MimeTypeAudio.X_PN_WAV]: 'wav', + [MimeTypeAudio.WEBM]: 'webm', + [MimeTypeAudio.WEBM_OPUS]: 'opus' +} as const; diff --git a/tools/ui/src/lib/constants/read-media.ts b/tools/ui/src/lib/constants/read-media.ts new file mode 100644 index 000000000000..7fbda2f1d729 --- /dev/null +++ b/tools/ui/src/lib/constants/read-media.ts @@ -0,0 +1,3 @@ +export const PREFIX_FILE = 'File: '; +export const PREFIX_SIZE = 'Size: '; +export const PREFIX_MIME = 'MIME: '; diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index 9985e1f4aa98..85ceea506e02 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -27,6 +27,7 @@ export enum ToolResponseField { */ export enum BuiltInTool { READ_FILE = 'read_file', + READ_MEDIA = 'read_media', EDIT_FILE = 'edit_file', WRITE_FILE = 'write_file', GET_DATETIME = 'get_datetime', diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 4ced749eac9a..c9495abed7e1 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -34,9 +34,11 @@ import { isAbortError } from '$lib/utils'; import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; import { IMAGE_MIME_TO_EXTENSION, + AUDIO_MIME_TO_EXTENSION, DATA_URI_BASE64_REGEX, MCP_ATTACHMENT_NAME_PREFIX, - DEFAULT_IMAGE_EXTENSION + DEFAULT_IMAGE_EXTENSION, + AUDIO_FILE_EXTENSION_REGEX } from '$lib/constants'; import { AttachmentType, @@ -1030,6 +1032,17 @@ class AgenticStore { return `[Attachment saved: ${name}]`; } + if (mimeType.startsWith('audio/')) { + attachments.push({ + type: AttachmentType.AUDIO, + name, + mimeType, + base64Url: trimmedLine + }); + + return `[Attachment saved: ${name}]`; + } + return line; }); @@ -1037,6 +1050,11 @@ class AgenticStore { } private buildAttachmentName(mimeType: string, index: number): string { + if (mimeType.startsWith('audio/')) { + const extension = AUDIO_MIME_TO_EXTENSION[mimeType] ?? 'mp3'; + return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + } + const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION; return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index aa8c287c3ab5..9d83b2ada038 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -47,11 +47,11 @@ export interface AgenticSection { } /** - * Represents a tool result line that may reference an image attachment + * Represents a tool result line that may reference a media attachment (image or audio) */ export type ToolResultLine = { text: string; - image?: DatabaseMessageExtraImageFile; + media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile; }; /** @@ -282,9 +282,9 @@ export function splitSearchSummaryList( } /** - * Parse tool result text into lines, matching image attachments by name. + * Parse tool result text into lines, matching media attachments (images and audio) by name. */ -export function parseToolResultWithImages( +export function parseToolResultWithMedia( toolResult: string, extras?: DatabaseMessageExtra[] ): ToolResultLine[] { @@ -294,12 +294,13 @@ export function parseToolResultWithImages( if (!match || !extras) return { text: line }; const attachmentName = match[1]; - const image = extras.find( - (e): e is DatabaseMessageExtraImageFile => - e.type === AttachmentType.IMAGE && e.name === attachmentName + const media = extras.find( + (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile => + (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) && + e.name === attachmentName ); - return { text: line, image }; + return { text: line, media }; }); } diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index f829793f1c7c..ccdcc722aa96 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -162,7 +162,7 @@ export { parseHeadersToArray, serializeHeaders } from './headers'; export { deriveAgenticSections, buildAssistantRawOutput, - parseToolResultWithImages, + parseToolResultWithMedia, splitSearchSummaryList, hasAgenticContent, classifyToolResult,