diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index a4c1059fff..fd0ff8ddd7 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1,6 +1,7 @@ #include "server-tools.h" #include "subproc.h" +#include "base64.hpp" #include #include @@ -864,6 +865,7 @@ static bool path_glob_match(const std::string & pattern, const std::string & rel // static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB +static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MB struct server_tool_read_file : server_tool { server_tool_read_file() { @@ -899,6 +901,8 @@ struct server_tool_read_file : server_tool { int start_line = json_value(params, "start_line", 1); int end_line = json_value(params, "end_line", -1); // -1 = no limit bool append_loc = json_value(params, "append_loc", false); + // comes from the x-resp-type header, the model cannot ask for it + bool as_base64 = json_value(params, "resp_type", std::string()) == "base64"; auto io = make_tools_io(params); @@ -906,6 +910,23 @@ struct server_tool_read_file : server_tool { if (!io->file_size(path, file_size)) { return {{"error", "cannot stat file: " + path}}; } + + if (as_base64) { + if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) { + return {{"error", string_format( + "file too large (%zu bytes, max %zu)", + (size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}}; + } + std::string content; + if (!io->read_file(path, content)) { + return {{"error", "failed to open file: " + path}}; + } + return { + {"base64", base64::encode(content.data(), content.size())}, + {"size_bytes", (size_t) content.size()}, + }; + } + if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) { return {{"error", string_format( "file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.", @@ -2135,6 +2156,15 @@ void server_tools::setup(const std::vector & enabled_tools, params["runtime"] = runtime->spec(); } + // x-resp-type header is only used by read_file for now + if (params.contains("resp_type")) { + params.erase("resp_type"); + } + auto resp_type = get_header(req.headers, "x-resp-type"); + if (!resp_type.empty()) { + params["resp_type"] = resp_type; + } + server_tool & tool = find_tool(tools, tool_name, stream); if (stream) { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index 5bbaa4ea31..7a00f1a8ea 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -7,6 +7,7 @@ import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte'; import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte'; import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte'; + import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte'; import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte'; import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte'; import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte'; @@ -45,6 +46,8 @@ {:else if section.toolName === BuiltInTool.READ_FILE} +{:else if section.toolName === BuiltInTool.READ_MEDIA} + {:else if section.toolName === BuiltInTool.EDIT_FILE} {:else if section.toolName === BuiltInTool.WRITE_FILE} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index 34dfde78b8..87208e2daa 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -8,14 +8,16 @@ import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app'; import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants'; import { getBuiltinToolUi } from '$lib/constants/built-in-tools'; - import { FileTypeText, ToolResultKind } from '$lib/enums'; + import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums'; import type { DatabaseMessageExtra } from '$lib/types'; import { type AgenticSection, classifyToolResult, formatJsonPretty, - parseToolResultWithImages + parseToolResultWithMedia, + type ToolResultLine } from '$lib/utils'; + import { createBase64DataUrl } from '$lib/utils/data-url'; interface Props { section: AgenticSection; @@ -29,8 +31,8 @@ const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? ''); const outputKind = $derived(classifyToolResult(section.toolResult)); - const parsedLines = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + const parsedLines: ToolResultLine[] = $derived( + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); @@ -103,13 +105,26 @@
{line.text}
- {#if line.image} - {line.image.name} + {#if line.media} + {#if line.media.type === AttachmentType.AUDIO} + {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG} +
+ +
+ {:else} + {line.media.name} + {/if} {/if} {/each} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte index b2fa8b331c..907a3cd12a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte @@ -23,7 +23,7 @@ isExitCodeSummaryLine, parseExecShellCommandError, parseExecShellCommandExitStatus, - parseToolResultWithImages, + parseToolResultWithMedia, type ToolResultLine } from '$lib/utils'; @@ -53,7 +53,7 @@ ); const parsedLines: ToolResultLine[] = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); // Drop the trailing "[exit code: N]" line - rendered as a colored @@ -223,10 +223,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.image} + {#if line.media} {line.image.name} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte new file mode 100644 index 0000000000..424c794d0b --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -0,0 +1,99 @@ + + + + {#snippet titleSnippet()} + Read media + {readMediaMeta?.fileName} + {/snippet} + + {#snippet children(_meta, _ctx)} + {#if section.toolResult} + {#if !mediaAttachment} +
+ Media attachment not found in message extras +
+ {:else if mediaAttachment.type === AttachmentType.AUDIO} +
+ +
+ {:else} +
+ {readMediaMeta?.fileName +
+ {/if} + + {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType} +
+ {#if readMediaMeta?.sizeBytes} + Size: {readMediaMeta.sizeBytes} bytes + {/if} + {#if readMediaMeta?.mimeType} + MIME: {readMediaMeta.mimeType} + {/if} +
+ {/if} + + {#if readMediaMeta?.path} +
{readMediaMeta.path}
+ {/if} + {:else} +
+ Waiting for media data... +
+ {/if} + {/snippet} + diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts new file mode 100644 index 0000000000..ab1a4d6aa0 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts @@ -0,0 +1,56 @@ +import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code'; +import { + PREFIX_FILE, + PREFIX_MIME, + PREFIX_SIZE, + READ_MEDIA_SIZE_REGEX +} from '$lib/constants/read-media'; +import type { AgenticSection } from '$lib/utils'; + +export interface ReadMediaMeta { + fileName: string; + path: string; + sizeBytes?: number; + mimeType?: string; +} + +/** + * Parse read_media tool result to extract metadata. + * Expected format (after extractBase64Attachments processing): + * File: /path/to/file.png + * Size: 12345 bytes + * MIME: image/png + * [Attachment saved: mcp-attachment-xxx.png] + * + * The data URI line is replaced by the attachment marker by + * agenticStore.extractBase64Attachments before storage. + */ +export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null { + if (!section.toolResult) return null; + + const lines = section.toolResult.split(NEWLINE); + + let fileName = ''; + let path = ''; + let sizeBytes: number | undefined; + let mimeType: string | undefined; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith(PREFIX_FILE)) { + path = trimmed.slice(PREFIX_FILE.length).trim(); + fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path; + } else if (trimmed.startsWith(PREFIX_SIZE)) { + const match = trimmed.match(READ_MEDIA_SIZE_REGEX); + + if (match) sizeBytes = Number(match[1]); + } else if (trimmed.startsWith(PREFIX_MIME)) { + mimeType = trimmed.slice(PREFIX_MIME.length).trim(); + } + } + + if (!path) return null; + + return { fileName, mimeType, path, sizeBytes }; +} diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index 89bcbecb8a..5bd24ffe83 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -10,6 +10,7 @@ import { Braces, Clock, + Eye, FilePen, FilePlus, FileSearch, @@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly> source: ToolSource.BUILTIN }, [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND }, [BuiltInTool.RUN_JAVASCRIPT]: { icon: Braces, label: 'Run JavaScript', diff --git a/tools/ui/src/lib/constants/code.ts b/tools/ui/src/lib/constants/code.ts index e57e1e6ec5..4b41142001 100644 --- a/tools/ui/src/lib/constants/code.ts +++ b/tools/ui/src/lib/constants/code.ts @@ -18,6 +18,9 @@ export const TRIM_TRAILING_PADDING_REGEX = /(?:\n[ \t]*)+$/; // `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path. export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/; +// Separates a file name from its extension, e.g. the '.' in `cover.png`. +export const FILE_EXTENSION_SEPARATOR = '.'; + // Matches the `text:` prefix that file-type identifiers use to denote a // plain-text language (e.g. `text:typescript`). Used by tool-call renderers // to recover the underlying highlight.js language. diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 09006cf1e9..357a33a624 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -49,6 +49,7 @@ export * from './sse'; export * from './precision'; export * from './processing-info'; export * from './pwa'; +export * from './read-media'; export * from './routes'; export * from './sandbox'; export * from './settings-keys'; diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index 0ef8d9ec13..c2639daa12 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -1,4 +1,4 @@ -import { MimeTypeImage } from '$lib/enums'; +import { MimeTypeAudio, MimeTypeImage } from '$lib/enums'; // File extension patterns for resource type detection export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i; @@ -27,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res'; // Default file extension for unknown image types export const DEFAULT_IMAGE_EXTENSION = 'img'; +// Default file extension for unknown audio types +export const DEFAULT_AUDIO_EXTENSION = 'mp3'; + // Default filename for resource content downloads export const DEFAULT_RESOURCE_FILENAME = 'resource.txt'; @@ -53,3 +56,18 @@ export const IMAGE_MIME_TO_EXTENSION: Record = { [MimeTypeImage.PNG]: 'png', [MimeTypeImage.WEBP]: 'webp' } as const; + +/** + * Mapping from audio MIME types to file extensions. + * Used for generating attachment filenames from MIME types. + */ +export const AUDIO_MIME_TO_EXTENSION: Record = { + [MimeTypeAudio.MP3]: 'mp3', + [MimeTypeAudio.MP3_MPEG]: 'mp3', + [MimeTypeAudio.VND_WAVE]: 'wav', + [MimeTypeAudio.WAV]: 'wav', + [MimeTypeAudio.WAVE]: 'wav', + [MimeTypeAudio.X_PN_WAV]: 'wav', + [MimeTypeAudio.X_WAV]: 'wav', + [MimeTypeAudio.X_WAVE]: 'wav' +} as const; diff --git a/tools/ui/src/lib/constants/read-media.ts b/tools/ui/src/lib/constants/read-media.ts new file mode 100644 index 0000000000..525c5e9029 --- /dev/null +++ b/tools/ui/src/lib/constants/read-media.ts @@ -0,0 +1,66 @@ +import { + BuiltInTool, + JsonSchemaType, + MimeTypeAudio, + MimeTypeImage, + ToolCallType +} from '$lib/enums'; +import type { OpenAIToolDefinition } from '$lib/types'; + +export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA; + +// header lines of the tool result, parsed back by the read_media renderer +export const PREFIX_FILE = 'File: '; +export const PREFIX_SIZE = 'Size: '; +export const PREFIX_MIME = 'MIME: '; + +/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */ +export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`); + +/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */ +export const READ_MEDIA_IMAGE_MIME: Record = { + gif: MimeTypeImage.GIF, + jpeg: MimeTypeImage.JPEG, + jpg: MimeTypeImage.JPEG, + png: MimeTypeImage.PNG +} as const; + +/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */ +export const READ_MEDIA_AUDIO_MIME: Record = { + mp3: MimeTypeAudio.MP3_MPEG, + wav: MimeTypeAudio.WAV +} as const; + +/** + * Build the read_media tool definition for the modalities the active model has. + * At least one of the two flags must be true, otherwise the tool is not offered + * at all - a model that cannot see or hear has nothing to do with the bytes. + */ +export function buildReadMediaToolDefinition( + supportsVision: boolean, + supportsAudio: boolean +): OpenAIToolDefinition { + const kinds: string[] = []; + + if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`); + + if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`); + + return { + function: { + description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`, + name: READ_MEDIA_TOOL_NAME, + parameters: { + properties: { + path: { + description: 'Path to the media file', + type: JsonSchemaType.STRING + } + }, + required: ['path'], + type: JsonSchemaType.OBJECT + } + }, + type: ToolCallType.FUNCTION + }; +} diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.ts index 65f4457c96..a467cda2b9 100644 --- a/tools/ui/src/lib/constants/tools.ts +++ b/tools/ui/src/lib/constants/tools.ts @@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums'; /** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */ export const X_TOOL_CWD_HEADER = 'x-tool-cwd'; +/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */ +export const X_RESP_TYPE_HEADER = 'x-resp-type'; + +/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */ +export const RESP_TYPE_BASE64 = 'base64'; + export const TOOL_GROUP_LABELS = { [ToolSource.BUILTIN]: 'Built-in', [ToolSource.CUSTOM]: 'JSON Schema', diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts index eecb36c23e..5785428cf0 100644 --- a/tools/ui/src/lib/enums/files.enums.ts +++ b/tools/ui/src/lib/enums/files.enums.ts @@ -163,6 +163,7 @@ export enum FileExtensionText { // MIME type prefixes and includes for content detection export enum MimeTypePrefix { IMAGE = 'image/', + AUDIO = 'audio/', TEXT = 'text' } diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index 7a9751ee79..31c992fef4 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -37,6 +37,7 @@ export enum GlobSearchType { */ export enum BuiltInTool { READ_FILE = 'read_file', + READ_MEDIA = 'read_media', EDIT_FILE = 'edit_file', WRITE_FILE = 'write_file', GET_DATETIME = 'get_datetime', diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 775a5d6e06..540d742393 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,4 +1,5 @@ import { settingsStore } from '../stores/settings.svelte'; +import { getAudioInputFormat } from '../utils/audio-format'; import { capImageDataURLSize } from '../utils/cap-img-size'; import { API_CHAT, @@ -20,18 +21,12 @@ import { import { AttachmentType, ContentPartType, - FileTypeAudio, MessageRole, - MimeTypeAudio, ReasoningFormat, StreamConnectionState } from '$lib/enums'; import { modelsStore } from '$lib/stores/models.svelte'; -import type { - AudioInputFormat, - DatabaseMessageExtraMcpPrompt, - DatabaseMessageExtraMcpResource -} from '$lib/types'; +import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types'; import type { ApiChatCompletionToolCall, ApiChatMessageContentPart, @@ -43,23 +38,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { streamIdentity } from '$lib/utils/stream-identity'; -function getAudioInputFormat(mimeType: string): AudioInputFormat { - const normalizedMimeType = mimeType.trim().toLowerCase(); - - if ( - normalizedMimeType === MimeTypeAudio.WAV || - normalizedMimeType === MimeTypeAudio.WAVE || - normalizedMimeType === MimeTypeAudio.X_WAV || - normalizedMimeType === MimeTypeAudio.X_WAVE || - normalizedMimeType === MimeTypeAudio.VND_WAVE || - normalizedMimeType === MimeTypeAudio.X_PN_WAV - ) { - return FileTypeAudio.WAV; - } - - return FileTypeAudio.MP3; -} - interface ResumableStreamState { bytesReceived: number; updatedAt: number; diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.ts new file mode 100644 index 0000000000..fd66350d00 --- /dev/null +++ b/tools/ui/src/lib/services/read-media.service.ts @@ -0,0 +1,112 @@ +import { ToolsService } from './tools.service'; +import { + FILE_EXTENSION_SEPARATOR, + FILE_PATH_SEPARATOR_REGEX, + NEWLINE, + PREFIX_FILE, + PREFIX_MIME, + PREFIX_SIZE, + READ_MEDIA_AUDIO_MIME, + READ_MEDIA_IMAGE_MIME, + RESP_TYPE_BASE64 +} from '$lib/constants'; +import { BuiltInTool, ToolResponseField } from '$lib/enums'; +import type { ToolExecutionResult } from '$lib/types'; + +/** Modalities of the model the tool call runs for. */ +export interface ReadMediaCapabilities { + audio: boolean; + vision: boolean; +} + +/** Lowercase extension of a path, without the dot. Empty when the file name has none. */ +function fileExtension(path: string): string { + const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? ''; + const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR); + + return dot > 0 ? name.slice(dot + 1).toLowerCase() : ''; +} + +/** + * **ReadMediaService** - frontend executor for the `read_media` tool + * + * The tool is synthetic: no such tool exists on the server. It reads the file + * through the built-in `read_file` tool with the `base64` response type, then + * turns the bytes into a data URI line. The agentic store lifts that line into + * an image or audio attachment on the tool result message, which is what makes + * the model perceive the file instead of reading a wall of base64. + * + * Living in the frontend is what lets it exist only for models that can + * actually use the result - the server has no idea which model is selected. + * + * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM + * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction + */ +export class ReadMediaService { + static async executeTool( + params: Record, + capabilities: ReadMediaCapabilities, + signal?: AbortSignal, + cwd?: string + ): Promise { + const path = typeof params.path === 'string' ? params.path : ''; + + if (!path) { + return { content: 'Error: missing "path" argument.', isError: true }; + } + + const extension = fileExtension(path); + const imageMime = READ_MEDIA_IMAGE_MIME[extension]; + const audioMime = READ_MEDIA_AUDIO_MIME[extension]; + + let resolvedMime: string | undefined; + + if (imageMime && capabilities.vision) resolvedMime = imageMime; + else if (audioMime && capabilities.audio) resolvedMime = audioMime; + + if (!resolvedMime) { + const supported = [ + ...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []), + ...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : []) + ]; + // an unreadable-by-this-model file is a dead end, so say why instead of failing silently + const reason = + imageMime || audioMime + ? `the current model cannot perceive ".${extension}" files` + : `".${extension}" is not a supported media type`; + + return { + content: `Error: ${reason}. Supported: ${supported.join(', ')}.`, + isError: true + }; + } + + const raw = await ToolsService.executeToolRaw( + BuiltInTool.READ_FILE, + { path }, + signal, + cwd, + RESP_TYPE_BASE64 + ); + + if (ToolResponseField.ERROR in raw) { + return { content: String(raw[ToolResponseField.ERROR]), isError: true }; + } + + const base64 = typeof raw.base64 === 'string' ? raw.base64 : ''; + + if (!base64) { + return { content: `Error: no data returned for ${path}.`, isError: true }; + } + + const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0; + const content = [ + `${PREFIX_FILE}${path}`, + `${PREFIX_SIZE}${sizeBytes} bytes`, + `${PREFIX_MIME}${resolvedMime}`, + `data:${resolvedMime};base64,${base64}` + ].join(NEWLINE); + + return { content, isError: false }; + } +} diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.ts index cd6b12cebe..9cb5ecb46d 100644 --- a/tools/ui/src/lib/services/tools.service.ts +++ b/tools/ui/src/lib/services/tools.service.ts @@ -1,5 +1,5 @@ import { base } from '$app/paths'; -import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants'; +import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants'; import { ToolResponseField } from '$lib/enums'; import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types'; import { apiFetch } from '$lib/utils'; @@ -51,16 +51,26 @@ export class ToolsService { * Execute a built-in tool and return the raw JSON response. Unlike * executeTool, this preserves structured fields (e.g. file_glob_search's * `entries` and `base`) that the flattened ToolExecutionResult drops. + * + * @param respType - sent as the x-resp-type request header. Only read_file + * honors it, with `base64` to get the raw bytes instead of decoded text. */ static async executeToolRaw( toolName: string, params: Record, signal?: AbortSignal, - cwd?: string + cwd?: string, + respType?: string ): Promise> { + const headers: Record = {}; + + if (cwd) headers[X_TOOL_CWD_HEADER] = cwd; + + if (respType) headers[X_RESP_TYPE_HEADER] = respType; + return apiFetch>(API_TOOLS.EXECUTE, { body: JSON.stringify({ params, tool: toolName }), - headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined, + headers: Object.keys(headers).length > 0 ? headers : undefined, method: 'POST', signal }); diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 080ed7c89a..a84fa34dc2 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -22,7 +22,9 @@ import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; import { + AUDIO_MIME_TO_EXTENSION, DATA_URI_BASE64_REGEX, + DEFAULT_AUDIO_EXTENSION, DEFAULT_IMAGE_EXTENSION, IMAGE_MIME_TO_EXTENSION, MCP_ATTACHMENT_NAME_PREFIX @@ -36,6 +38,7 @@ import { ToolCallType } from '$lib/enums'; import { ChatService } from '$lib/services'; +import { ReadMediaService } from '$lib/services/read-media.service'; import { SandboxService } from '$lib/services/sandbox.service'; import { ToolsService } from '$lib/services/tools.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; @@ -75,9 +78,10 @@ import type { import type { DatabaseMessage, DatabaseMessageExtra, + DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types/database'; -import { isAbortError } from '$lib/utils'; +import { getAudioInputFormat, isAbortError } from '$lib/utils'; import { SvelteMap } from 'svelte/reactivity'; function createDefaultSession(): AgenticSession { @@ -900,7 +904,18 @@ class AgenticStore { if (executionResult.isError) toolSuccess = false; } else if (toolSource === ToolSource.FRONTEND) { const args = this.parseToolArguments(toolCall.function.arguments); - const executionResult = await SandboxService.executeTool(toolName, args, signal); + const executionResult = + toolName === BuiltInTool.READ_MEDIA + ? await ReadMediaService.executeTool( + args, + { + audio: modelsStore.modelSupportsAudio(effectiveModel), + vision: modelsStore.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ) + : await SandboxService.executeTool(toolName, args, signal); result = executionResult.content; @@ -990,7 +1005,19 @@ class AgenticStore { ]; for (const attachment of attachments) { - if (attachment.type === AttachmentType.IMAGE) { + if (attachment.type === AttachmentType.AUDIO) { + if (modelsStore.modelSupportsAudio(effectiveModel)) { + contentParts.push({ + input_audio: { + data: (attachment as DatabaseMessageExtraAudioFile).base64Data, + format: getAudioInputFormat( + (attachment as DatabaseMessageExtraAudioFile).mimeType + ) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + } else if (attachment.type === AttachmentType.IMAGE) { if (modelsStore.modelSupportsVision(effectiveModel)) { contentParts.push({ image_url: { @@ -1101,6 +1128,18 @@ class AgenticStore { return `[Attachment saved: ${name}]`; } + if (mimeType.startsWith(MimeTypePrefix.AUDIO)) { + // audio extras hold the bare base64, the input_audio part has no room for a data URI + attachments.push({ + base64Data, + mimeType, + name, + type: AttachmentType.AUDIO + }); + + return `[Attachment saved: ${name}]`; + } + return line; }); @@ -1108,7 +1147,9 @@ class AgenticStore { } private buildAttachmentName(mimeType: string, index: number): string { - const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION; + const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) + ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) + : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; } diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts index 145de119ce..150fb3800f 100644 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ b/tools/ui/src/lib/stores/models.svelte.ts @@ -18,7 +18,10 @@ import { ModelsService } from '$lib/services/models.service'; import { PropsService } from '$lib/services/props.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { isRouterMode, serverStore } from '$lib/stores/server.svelte'; -import { getAuthHeaders, TTLCache } from '$lib/utils'; +// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back +// into the stores, and going through it here would read a half-built module +import { getAuthHeaders } from '$lib/utils/api-headers'; +import { TTLCache } from '$lib/utils/cache-ttl'; import { detectThinkingSupport, detectThinkingSupportWithReason diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index b1a946391a..3984dd2bc8 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,4 +1,5 @@ import { + buildReadMediaToolDefinition, buildSandboxToolDefinition, DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, HOME_TILDE, @@ -15,6 +16,7 @@ import { } from '$lib/enums'; import { ToolsService } from '$lib/services/tools.service'; import { mcpStore } from '$lib/stores/mcp.svelte'; +import { modelsStore, selectedModelName } from '$lib/stores/models.svelte'; import { config } from '$lib/stores/settings.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -168,9 +170,42 @@ class ToolsStore { } get frontendTools(): OpenAIToolDefinition[] { - return config().jsSandboxEnabled - ? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)] - : []; + const tools: OpenAIToolDefinition[] = []; + + if (config().jsSandboxEnabled) { + tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled)); + } + + const readMedia = this.readMediaTool(); + + if (readMedia) tools.push(readMedia); + + return tools; + } + + /** + * `read_media` runs in the frontend on top of the server's `read_file`, so it + * exists only when that tool is served and the active model can perceive the + * bytes. The server cannot make this call - it does not know which model the + * conversation uses. + */ + private readMediaTool(): OpenAIToolDefinition | null { + const hasReadFile = this._builtinTools.some( + (def) => def.function.name === BuiltInTool.READ_FILE + ); + + if (!hasReadFile) return null; + + const model = selectedModelName() ?? modelsStore.models[0]?.model ?? ''; + + if (!model) return null; + + const vision = modelsStore.modelSupportsVision(model); + const audio = modelsStore.modelSupportsAudio(model); + + if (!vision && !audio) return null; + + return buildReadMediaToolDefinition(vision, audio); } get customTools(): OpenAIToolDefinition[] { diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index 9c92a42d40..b2dd2cd9ed 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -50,11 +50,11 @@ export interface AgenticSection { } /** - * Represents a tool result line that may reference an image attachment + * Represents a tool result line that may reference a media attachment (image or audio) */ export type ToolResultLine = { text: string; - image?: DatabaseMessageExtraImageFile; + media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile; }; /** @@ -301,16 +301,16 @@ export function splitSearchSummaryList( return { lines }; } -/** Bounded cache for parseToolResultWithImages results. */ +/** Bounded cache for parseToolResultWithMedia results. */ const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32; const toolResultLinesCache = new Map(); /** - * Parse tool result text into lines, matching image attachments by name. + * Parse tool result text into lines, matching media attachments (images and audio) by name. * Memoized: called per render during streaming on unchanged tool result * strings with unchanged extras. */ -export function parseToolResultWithImages( +export function parseToolResultWithMedia( toolResult: string, extras?: DatabaseMessageExtra[] ): ToolResultLine[] { @@ -332,12 +332,13 @@ export function parseToolResultWithImages( if (!match || !extras) return { text: line }; const attachmentName = match[1]; - const image = extras.find( - (e): e is DatabaseMessageExtraImageFile => - e.type === AttachmentType.IMAGE && e.name === attachmentName + const media = extras.find( + (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile => + (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) && + e.name === attachmentName ); - return { image, text: line }; + return { media, text: line }; }); if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) { diff --git a/tools/ui/src/lib/utils/audio-format.ts b/tools/ui/src/lib/utils/audio-format.ts new file mode 100644 index 0000000000..4f597aad34 --- /dev/null +++ b/tools/ui/src/lib/utils/audio-format.ts @@ -0,0 +1,22 @@ +import { FileTypeAudio, MimeTypeAudio } from '$lib/enums'; +import type { AudioInputFormat } from '$lib/types/api'; + +/** + * Map a MIME type to the AudioInputFormat expected by the API. + */ +export function getAudioInputFormat(mimeType: string): AudioInputFormat { + const normalizedMimeType = mimeType.trim().toLowerCase(); + + if ( + normalizedMimeType === MimeTypeAudio.WAV || + normalizedMimeType === MimeTypeAudio.WAVE || + normalizedMimeType === MimeTypeAudio.X_WAV || + normalizedMimeType === MimeTypeAudio.X_WAVE || + normalizedMimeType === MimeTypeAudio.VND_WAVE || + normalizedMimeType === MimeTypeAudio.X_PN_WAV + ) { + return FileTypeAudio.WAV; + } + + return FileTypeAudio.MP3; +} diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 5762e8e4bb..c3585bd5f8 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -248,7 +248,7 @@ export { export { deriveAgenticSections, buildAssistantRawOutput, - parseToolResultWithImages, + parseToolResultWithMedia, splitSearchSummaryList, hasAgenticContent, classifyToolResult, @@ -325,3 +325,6 @@ export { uuid } from './uuid'; // CSS utilities export { remToPx } from './css'; + +// Audio format helper (used by agentic store and chat service) +export { getAudioInputFormat } from './audio-format'; diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.md index a4e3e4c644..198b219568 100644 --- a/tools/ui/tests/client/README-perf.md +++ b/tools/ui/tests/client/README-perf.md @@ -33,12 +33,12 @@ npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.t The point of the harness is the _scaling curve_, not any single number. -| Knob | Reads on | -| --------------------------- | ---------------------------------------------------------------------------------------------------- | -| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | -| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). | -| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | -| `openCodeFence` | `hljs.highlightAuto` on partial code. | +| Knob | Reads on | +| --------------------------- | --------------------------------------------------------------------------------------------------- | +| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | +| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). | +| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | +| `openCodeFence` | `hljs.highlightAuto` on partial code. | Deliberately no hard assertions: CI timing is noisy and the value here is the before/after delta, not a gate. diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.ts index 8bdaa095a8..24dcb48911 100644 --- a/tools/ui/tests/unit/agentic-hotpath.bench.ts +++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts @@ -6,7 +6,7 @@ // // Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts -import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic'; +import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic'; import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code'; import { computeLineDiff } from '$lib/utils/compute-line-diff'; import { preprocessLaTeX } from '$lib/utils/latex-protection'; @@ -200,17 +200,17 @@ describe('exit-code regex', () => { // --- per-line result parsers ---------------------------------------------- -describe('parseToolResultWithImages', () => { +describe('parseToolResultWithMedia', () => { bench('1KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_1KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_1KB, []); }); bench('200KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_200KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_200KB, []); }); bench('2MB', () => { - parseToolResultWithImages(SHELL_OUTPUT_2MB, []); + parseToolResultWithMedia(SHELL_OUTPUT_2MB, []); }); });