ui: add read_media tool (#25877)

* server: add read_image tool (#25875)

Adds a server-tool that allows vision models to analyze server-side images.
This tool is reading a single file for now:
The image data is base64 encoded and passed to the UI, which
decodes it, fills the <img> tag and removes the data URI before
passing the tool result back to the model.

* cleanup read_image tool: move magic strings to constants

* Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts
  with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants
* Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte
* Use NEWLINE constant from code.ts instead of hardcoded '\n'
* Use PREFIX_SIZE in regex pattern for size parsing
* Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp
  to match the TypeScript PREFIX_* constants for consistency

* server: rename read_image tool to read_media for images and audio

* Rename server_tool_read_image to server_tool_read_media in C++
* Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA
* Rename UI constants, parser, and Svelte component files
* Update display label from 'Read image' to 'Read media'

* ui: consolidate audio data URI handling into shared utility

* Extract getAudioInputFormat to a shared utility (was duplicated inline)
* Store raw base64 in base64Data on the message object
* Use base64Data to construct data URIs for audio rendering
* Update agentic store to build INPUT_AUDIO parts from base64Data

* server: read_media: restrict audio to wav/mp3 and minor fixes

* Server get_mime_from_extension now only advertises audio/wav and
  audio/mpeg (the only formats the model's input_audio API accepts)
* Case-insensitive extension matching (fixes .MP3, .Wav, etc.)
* Unknown extensions return an error instead of a multi-MB data URI
  that inflates model context with garbage
* Updated tool description to document supported formats
* Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server
* fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts

* server: read_media: add to --tools help text and README tool list

* ui: fix indentation in ChatMessageToolCallBlockDefault.svelte

* server: read_media tool: fix a cast to use the correct type

* server: read_media: multiple fixes

* server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file
* ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts

* server: make read_media inherit from read_file and add uses_cwd

* ui: fix formating issues

* rm from server

* move it to frontend-only tool

* correct partial commit

* rm unused

* ui: address review from allozaur

Replace the magic strings, regexes and number in the read_media parser
and service with named constants. Path splitting reuses
FILE_PATH_SEPARATOR_REGEX, the size header regex moves to
READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and
FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts.

---------

Co-authored-by: ckrafft <ckrafft@epyc>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Pascal <admin@serveurperso.com>

tools/server/server-tools.cpp

@@ -1,6 +1,7 @@
 #include "server-tools.h"
 
 #include "subproc.h"
+#include "base64.hpp"
 
 #include <filesystem>
 #include <fstream>
@@ -864,6 +865,7 @@ static bool path_glob_match(const std::string & pattern, const std::string & rel
 //
 
 static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB
+static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MB
 
 struct server_tool_read_file : server_tool {
     server_tool_read_file() {
@@ -899,6 +901,8 @@ struct server_tool_read_file : server_tool {
         int  start_line   = json_value(params, "start_line", 1);
         int  end_line     = json_value(params, "end_line",  -1); // -1 = no limit
         bool append_loc   = json_value(params, "append_loc", false);
+        // comes from the x-resp-type header, the model cannot ask for it
+        bool as_base64    = json_value(params, "resp_type", std::string()) == "base64";
 
         auto io = make_tools_io(params);
 
@@ -906,6 +910,23 @@ struct server_tool_read_file : server_tool {
         if (!io->file_size(path, file_size)) {
             return {{"error", "cannot stat file: " + path}};
         }
+
+        if (as_base64) {
+            if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) {
+                return {{"error", string_format(
+                    "file too large (%zu bytes, max %zu)",
+                    (size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}};
+            }
+            std::string content;
+            if (!io->read_file(path, content)) {
+                return {{"error", "failed to open file: " + path}};
+            }
+            return {
+                {"base64",     base64::encode(content.data(), content.size())},
+                {"size_bytes", (size_t) content.size()},
+            };
+        }
+
         if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) {
             return {{"error", string_format(
                 "file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.",
@@ -2135,6 +2156,15 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,
                 params["runtime"] = runtime->spec();
             }
 
+            // x-resp-type header is only used by read_file for now
+            if (params.contains("resp_type")) {
+                params.erase("resp_type");
+            }
+            auto resp_type = get_header(req.headers, "x-resp-type");
+            if (!resp_type.empty()) {
+                params["resp_type"] = resp_type;
+            }
+
             server_tool & tool = find_tool(tools, tool_name, stream);
 
             if (stream) {

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte

@@ -7,6 +7,7 @@
 	import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte';
 	import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte';
 	import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte';
+	import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte';
 	import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte';
 	import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';
 	import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';
@@ -45,6 +46,8 @@
 	<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />
 {:else if section.toolName === BuiltInTool.READ_FILE}
 	<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />
+{:else if section.toolName === BuiltInTool.READ_MEDIA}
+	<ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.EDIT_FILE}
 	<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.WRITE_FILE}

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte

@@ -8,14 +8,16 @@
 	import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app';
 	import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
 	import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
-	import { FileTypeText, ToolResultKind } from '$lib/enums';
+	import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums';
 	import type { DatabaseMessageExtra } from '$lib/types';
 	import {
 		type AgenticSection,
 		classifyToolResult,
 		formatJsonPretty,
-		parseToolResultWithImages
+		parseToolResultWithMedia,
+		type ToolResultLine
 	} from '$lib/utils';
+	import { createBase64DataUrl } from '$lib/utils/data-url';
 
 	interface Props {
 		section: AgenticSection;
@@ -29,8 +31,8 @@
 
 	const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
 	const outputKind = $derived(classifyToolResult(section.toolResult));
-	const parsedLines = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+	const parsedLines: ToolResultLine[] = $derived(
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);
 </script>
 
@@ -103,13 +105,26 @@
 							<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">
 								{line.text}
 							</div>
-							{#if line.image}
-								<img
-									src={line.image.base64Url}
-									alt={line.image.name}
-									class="mt-2 mb-2 h-auto max-w-full rounded-lg"
-									loading="lazy"
-								/>
+							{#if line.media}
+								{#if line.media.type === AttachmentType.AUDIO}
+									{@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG}
+									<div class="mt-2 mb-2">
+										<audio controls class="w-full rounded-lg">
+											<source
+												src={createBase64DataUrl(audioMimeType, line.media.base64Data)}
+												type={audioMimeType}
+											/>
+											Your browser does not support the audio element.
+										</audio>
+									</div>
+								{:else}
+									<img
+										src={line.media.base64Url}
+										alt={line.media.name}
+										class="mt-2 mb-2 h-auto max-w-full rounded-lg"
+										loading="lazy"
+									/>
+								{/if}
 							{/if}
 						{/each}
 					</div>

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte

@@ -23,7 +23,7 @@
 		isExitCodeSummaryLine,
 		parseExecShellCommandError,
 		parseExecShellCommandExitStatus,
-		parseToolResultWithImages,
+		parseToolResultWithMedia,
 		type ToolResultLine
 	} from '$lib/utils';
 
@@ -53,7 +53,7 @@
 	);
 
 	const parsedLines: ToolResultLine[] = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);
 
 	// Drop the trailing "[exit code: N]" line - rendered as a colored
@@ -223,10 +223,10 @@
 			>
 				{#each outputLines as line, i (i)}
 					<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>
-					{#if line.image}
+					{#if line.media}
 						<img
-							src={line.image.base64Url}
-							alt={line.image.name}
+							src={line.media.base64Url}
+							alt={line.media.name}
 							class="mt-2 mb-2 h-auto max-w-full rounded-lg"
 							loading="lazy"
 						/>

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte

@@ -0,0 +1,99 @@
+<script lang="ts">
+	import { parseReadMediaMeta } from './parsers/read-media';
+	import ToolCallBlock from './ToolCallBlock.svelte';
+	import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic';
+	import { AttachmentType, MimeTypeAudio } from '$lib/enums';
+	import type { DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types';
+	import { type AgenticSection } from '$lib/utils';
+	import { createBase64DataUrl } from '$lib/utils/data-url';
+
+	interface Props {
+		section: AgenticSection;
+		open: boolean;
+		isStreaming: boolean;
+		onToggle?: () => void;
+	}
+
+	let { isStreaming, onToggle, open, section }: Props = $props();
+
+	const readMediaMeta = $derived(parseReadMediaMeta(section));
+
+	// extractBase64Attachments swapped the data URI line for [Attachment saved: name]
+	// and moved the bytes to the message extras, so the name is the only link back
+	const mediaAttachment = $derived.by(() => {
+		const extras = section.toolResultExtras;
+
+		if (!extras || extras.length === 0) return null;
+
+		const match = section.toolResult?.match(ATTACHMENT_SAVED_REGEX);
+
+		if (!match) return null;
+
+		const attachmentName = match[1];
+
+		return (
+			extras.find(
+				(e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+					(e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+					e.name === attachmentName
+			) ?? null
+		);
+	});
+
+	const audioMimeType = $derived(readMediaMeta?.mimeType ?? MimeTypeAudio.MP3_MPEG);
+</script>
+
+<ToolCallBlock {section} {open} {isStreaming} meta={readMediaMeta} {onToggle}>
+	{#snippet titleSnippet()}
+		<span class="text-muted-foreground">Read media </span>
+		<span class="font-mono">{readMediaMeta?.fileName}</span>
+	{/snippet}
+
+	{#snippet children(_meta, _ctx)}
+		{#if section.toolResult}
+			{#if !mediaAttachment}
+				<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+					Media attachment not found in message extras
+				</div>
+			{:else if mediaAttachment.type === AttachmentType.AUDIO}
+				<div class="mt-2">
+					<audio controls class="w-full rounded-lg">
+						<source
+							src={createBase64DataUrl(audioMimeType, mediaAttachment.base64Data)}
+							type={audioMimeType}
+						/>
+						Your browser does not support the audio element.
+					</audio>
+				</div>
+			{:else}
+				<div class="mt-2">
+					<img
+						src={mediaAttachment.base64Url}
+						alt={readMediaMeta?.fileName ?? 'media'}
+						class="max-h-[60vh] max-w-full rounded-lg object-contain shadow-lg"
+						loading="lazy"
+					/>
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}
+				<div class="mt-2 flex gap-4 text-xs text-muted-foreground">
+					{#if readMediaMeta?.sizeBytes}
+						<span>Size: {readMediaMeta.sizeBytes} bytes</span>
+					{/if}
+					{#if readMediaMeta?.mimeType}
+						<span>MIME: {readMediaMeta.mimeType}</span>
+					{/if}
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.path}
+				<div class="mt-1 font-mono text-xs text-muted-foreground/60">{readMediaMeta.path}</div>
+			{/if}
+		{:else}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+				Waiting for media data...
+			</div>
+		{/if}
+	{/snippet}
+</ToolCallBlock>

tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts

@@ -0,0 +1,56 @@
+import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code';
+import {
+	PREFIX_FILE,
+	PREFIX_MIME,
+	PREFIX_SIZE,
+	READ_MEDIA_SIZE_REGEX
+} from '$lib/constants/read-media';
+import type { AgenticSection } from '$lib/utils';
+
+export interface ReadMediaMeta {
+	fileName: string;
+	path: string;
+	sizeBytes?: number;
+	mimeType?: string;
+}
+
+/**
+ * Parse read_media tool result to extract metadata.
+ * Expected format (after extractBase64Attachments processing):
+ *   File: /path/to/file.png
+ *   Size: 12345 bytes
+ *   MIME: image/png
+ *   [Attachment saved: mcp-attachment-xxx.png]
+ *
+ * The data URI line is replaced by the attachment marker by
+ * agenticStore.extractBase64Attachments before storage.
+ */
+export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null {
+	if (!section.toolResult) return null;
+
+	const lines = section.toolResult.split(NEWLINE);
+
+	let fileName = '';
+	let path = '';
+	let sizeBytes: number | undefined;
+	let mimeType: string | undefined;
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+
+		if (trimmed.startsWith(PREFIX_FILE)) {
+			path = trimmed.slice(PREFIX_FILE.length).trim();
+			fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path;
+		} else if (trimmed.startsWith(PREFIX_SIZE)) {
+			const match = trimmed.match(READ_MEDIA_SIZE_REGEX);
+
+			if (match) sizeBytes = Number(match[1]);
+		} else if (trimmed.startsWith(PREFIX_MIME)) {
+			mimeType = trimmed.slice(PREFIX_MIME.length).trim();
+		}
+	}
+
+	if (!path) return null;
+
+	return { fileName, mimeType, path, sizeBytes };
+}

tools/ui/src/lib/constants/built-in-tools.ts

@@ -10,6 +10,7 @@
 import {
 	Braces,
 	Clock,
+	Eye,
 	FilePen,
 	FilePlus,
 	FileSearch,
@@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly<Record<BuiltInTool, BuiltinToolUiEntry>>
 		source: ToolSource.BUILTIN
 	},
 	[BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN },
+	[BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND },
 	[BuiltInTool.RUN_JAVASCRIPT]: {
 		icon: Braces,
 		label: 'Run JavaScript',

tools/ui/src/lib/constants/code.ts

@@ -18,6 +18,9 @@ export const TRIM_TRAILING_PADDING_REGEX = /(?:\n[ \t]*)+$/;
 // `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path.
 export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/;
 
+// Separates a file name from its extension, e.g. the '.' in `cover.png`.
+export const FILE_EXTENSION_SEPARATOR = '.';
+
 // Matches the `text:` prefix that file-type identifiers use to denote a
 // plain-text language (e.g. `text:typescript`). Used by tool-call renderers
 // to recover the underlying highlight.js language.

tools/ui/src/lib/constants/index.ts

@@ -49,6 +49,7 @@ export * from './sse';
 export * from './precision';
 export * from './processing-info';
 export * from './pwa';
+export * from './read-media';
 export * from './routes';
 export * from './sandbox';
 export * from './settings-keys';

tools/ui/src/lib/constants/mcp-resource.ts

@@ -1,4 +1,4 @@
-import { MimeTypeImage } from '$lib/enums';
+import { MimeTypeAudio, MimeTypeImage } from '$lib/enums';
 
 // File extension patterns for resource type detection
 export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i;
@@ -27,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res';
 // Default file extension for unknown image types
 export const DEFAULT_IMAGE_EXTENSION = 'img';
 
+// Default file extension for unknown audio types
+export const DEFAULT_AUDIO_EXTENSION = 'mp3';
+
 // Default filename for resource content downloads
 export const DEFAULT_RESOURCE_FILENAME = 'resource.txt';
 
@@ -53,3 +56,18 @@ export const IMAGE_MIME_TO_EXTENSION: Record<string, string> = {
 	[MimeTypeImage.PNG]: 'png',
 	[MimeTypeImage.WEBP]: 'webp'
 } as const;
+
+/**
+ * Mapping from audio MIME types to file extensions.
+ * Used for generating attachment filenames from MIME types.
+ */
+export const AUDIO_MIME_TO_EXTENSION: Record<string, string> = {
+	[MimeTypeAudio.MP3]: 'mp3',
+	[MimeTypeAudio.MP3_MPEG]: 'mp3',
+	[MimeTypeAudio.VND_WAVE]: 'wav',
+	[MimeTypeAudio.WAV]: 'wav',
+	[MimeTypeAudio.WAVE]: 'wav',
+	[MimeTypeAudio.X_PN_WAV]: 'wav',
+	[MimeTypeAudio.X_WAV]: 'wav',
+	[MimeTypeAudio.X_WAVE]: 'wav'
+} as const;

tools/ui/src/lib/constants/read-media.ts

@@ -0,0 +1,66 @@
+import {
+	BuiltInTool,
+	JsonSchemaType,
+	MimeTypeAudio,
+	MimeTypeImage,
+	ToolCallType
+} from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA;
+
+// header lines of the tool result, parsed back by the read_media renderer
+export const PREFIX_FILE = 'File: ';
+export const PREFIX_SIZE = 'Size: ';
+export const PREFIX_MIME = 'MIME: ';
+
+/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */
+export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`);
+
+/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */
+export const READ_MEDIA_IMAGE_MIME: Record<string, string> = {
+	gif: MimeTypeImage.GIF,
+	jpeg: MimeTypeImage.JPEG,
+	jpg: MimeTypeImage.JPEG,
+	png: MimeTypeImage.PNG
+} as const;
+
+/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */
+export const READ_MEDIA_AUDIO_MIME: Record<string, string> = {
+	mp3: MimeTypeAudio.MP3_MPEG,
+	wav: MimeTypeAudio.WAV
+} as const;
+
+/**
+ * Build the read_media tool definition for the modalities the active model has.
+ * At least one of the two flags must be true, otherwise the tool is not offered
+ * at all - a model that cannot see or hear has nothing to do with the bytes.
+ */
+export function buildReadMediaToolDefinition(
+	supportsVision: boolean,
+	supportsAudio: boolean
+): OpenAIToolDefinition {
+	const kinds: string[] = [];
+
+	if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`);
+
+	if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`);
+
+	return {
+		function: {
+			description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`,
+			name: READ_MEDIA_TOOL_NAME,
+			parameters: {
+				properties: {
+					path: {
+						description: 'Path to the media file',
+						type: JsonSchemaType.STRING
+					}
+				},
+				required: ['path'],
+				type: JsonSchemaType.OBJECT
+			}
+		},
+		type: ToolCallType.FUNCTION
+	};
+}

tools/ui/src/lib/constants/tools.ts

@@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums';
 /** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */
 export const X_TOOL_CWD_HEADER = 'x-tool-cwd';
 
+/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */
+export const X_RESP_TYPE_HEADER = 'x-resp-type';
+
+/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */
+export const RESP_TYPE_BASE64 = 'base64';
+
 export const TOOL_GROUP_LABELS = {
 	[ToolSource.BUILTIN]: 'Built-in',
 	[ToolSource.CUSTOM]: 'JSON Schema',

tools/ui/src/lib/enums/files.enums.ts

@@ -163,6 +163,7 @@ export enum FileExtensionText {
 // MIME type prefixes and includes for content detection
 export enum MimeTypePrefix {
 	IMAGE = 'image/',
+	AUDIO = 'audio/',
 	TEXT = 'text'
 }
 

tools/ui/src/lib/enums/tools.enums.ts

@@ -37,6 +37,7 @@ export enum GlobSearchType {
  */
 export enum BuiltInTool {
 	READ_FILE = 'read_file',
+	READ_MEDIA = 'read_media',
 	EDIT_FILE = 'edit_file',
 	WRITE_FILE = 'write_file',
 	GET_DATETIME = 'get_datetime',

tools/ui/src/lib/services/chat.service.ts

@@ -1,4 +1,5 @@
 import { settingsStore } from '../stores/settings.svelte';
+import { getAudioInputFormat } from '../utils/audio-format';
 import { capImageDataURLSize } from '../utils/cap-img-size';
 import {
 	API_CHAT,
@@ -20,18 +21,12 @@ import {
 import {
 	AttachmentType,
 	ContentPartType,
-	FileTypeAudio,
 	MessageRole,
-	MimeTypeAudio,
 	ReasoningFormat,
 	StreamConnectionState
 } from '$lib/enums';
 import { modelsStore } from '$lib/stores/models.svelte';
-import type {
-	AudioInputFormat,
-	DatabaseMessageExtraMcpPrompt,
-	DatabaseMessageExtraMcpResource
-} from '$lib/types';
+import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';
 import type {
 	ApiChatCompletionToolCall,
 	ApiChatMessageContentPart,
@@ -43,23 +38,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers';
 import { formatAttachmentText } from '$lib/utils/formatters';
 import { streamIdentity } from '$lib/utils/stream-identity';
 
-function getAudioInputFormat(mimeType: string): AudioInputFormat {
-	const normalizedMimeType = mimeType.trim().toLowerCase();
-
-	if (
-		normalizedMimeType === MimeTypeAudio.WAV ||
-		normalizedMimeType === MimeTypeAudio.WAVE ||
-		normalizedMimeType === MimeTypeAudio.X_WAV ||
-		normalizedMimeType === MimeTypeAudio.X_WAVE ||
-		normalizedMimeType === MimeTypeAudio.VND_WAVE ||
-		normalizedMimeType === MimeTypeAudio.X_PN_WAV
-	) {
-		return FileTypeAudio.WAV;
-	}
-
-	return FileTypeAudio.MP3;
-}
-
 interface ResumableStreamState {
 	bytesReceived: number;
 	updatedAt: number;

tools/ui/src/lib/services/read-media.service.ts

@@ -0,0 +1,112 @@
+import { ToolsService } from './tools.service';
+import {
+	FILE_EXTENSION_SEPARATOR,
+	FILE_PATH_SEPARATOR_REGEX,
+	NEWLINE,
+	PREFIX_FILE,
+	PREFIX_MIME,
+	PREFIX_SIZE,
+	READ_MEDIA_AUDIO_MIME,
+	READ_MEDIA_IMAGE_MIME,
+	RESP_TYPE_BASE64
+} from '$lib/constants';
+import { BuiltInTool, ToolResponseField } from '$lib/enums';
+import type { ToolExecutionResult } from '$lib/types';
+
+/** Modalities of the model the tool call runs for. */
+export interface ReadMediaCapabilities {
+	audio: boolean;
+	vision: boolean;
+}
+
+/** Lowercase extension of a path, without the dot. Empty when the file name has none. */
+function fileExtension(path: string): string {
+	const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? '';
+	const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR);
+
+	return dot > 0 ? name.slice(dot + 1).toLowerCase() : '';
+}
+
+/**
+ * **ReadMediaService** - frontend executor for the `read_media` tool
+ *
+ * The tool is synthetic: no such tool exists on the server. It reads the file
+ * through the built-in `read_file` tool with the `base64` response type, then
+ * turns the bytes into a data URI line. The agentic store lifts that line into
+ * an image or audio attachment on the tool result message, which is what makes
+ * the model perceive the file instead of reading a wall of base64.
+ *
+ * Living in the frontend is what lets it exist only for models that can
+ * actually use the result - the server has no idea which model is selected.
+ *
+ * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM
+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction
+ */
+export class ReadMediaService {
+	static async executeTool(
+		params: Record<string, unknown>,
+		capabilities: ReadMediaCapabilities,
+		signal?: AbortSignal,
+		cwd?: string
+	): Promise<ToolExecutionResult> {
+		const path = typeof params.path === 'string' ? params.path : '';
+
+		if (!path) {
+			return { content: 'Error: missing "path" argument.', isError: true };
+		}
+
+		const extension = fileExtension(path);
+		const imageMime = READ_MEDIA_IMAGE_MIME[extension];
+		const audioMime = READ_MEDIA_AUDIO_MIME[extension];
+
+		let resolvedMime: string | undefined;
+
+		if (imageMime && capabilities.vision) resolvedMime = imageMime;
+		else if (audioMime && capabilities.audio) resolvedMime = audioMime;
+
+		if (!resolvedMime) {
+			const supported = [
+				...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []),
+				...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : [])
+			];
+			// an unreadable-by-this-model file is a dead end, so say why instead of failing silently
+			const reason =
+				imageMime || audioMime
+					? `the current model cannot perceive ".${extension}" files`
+					: `".${extension}" is not a supported media type`;
+
+			return {
+				content: `Error: ${reason}. Supported: ${supported.join(', ')}.`,
+				isError: true
+			};
+		}
+
+		const raw = await ToolsService.executeToolRaw(
+			BuiltInTool.READ_FILE,
+			{ path },
+			signal,
+			cwd,
+			RESP_TYPE_BASE64
+		);
+
+		if (ToolResponseField.ERROR in raw) {
+			return { content: String(raw[ToolResponseField.ERROR]), isError: true };
+		}
+
+		const base64 = typeof raw.base64 === 'string' ? raw.base64 : '';
+
+		if (!base64) {
+			return { content: `Error: no data returned for ${path}.`, isError: true };
+		}
+
+		const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0;
+		const content = [
+			`${PREFIX_FILE}${path}`,
+			`${PREFIX_SIZE}${sizeBytes} bytes`,
+			`${PREFIX_MIME}${resolvedMime}`,
+			`data:${resolvedMime};base64,${base64}`
+		].join(NEWLINE);
+
+		return { content, isError: false };
+	}
+}

tools/ui/src/lib/services/tools.service.ts

@@ -1,5 +1,5 @@
 import { base } from '$app/paths';
-import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants';
+import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants';
 import { ToolResponseField } from '$lib/enums';
 import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types';
 import { apiFetch } from '$lib/utils';
@@ -51,16 +51,26 @@ export class ToolsService {
 	 * Execute a built-in tool and return the raw JSON response. Unlike
 	 * executeTool, this preserves structured fields (e.g. file_glob_search's
 	 * `entries` and `base`) that the flattened ToolExecutionResult drops.
+	 *
+	 * @param respType - sent as the x-resp-type request header. Only read_file
+	 * honors it, with `base64` to get the raw bytes instead of decoded text.
 	 */
 	static async executeToolRaw(
 		toolName: string,
 		params: Record<string, unknown>,
 		signal?: AbortSignal,
-		cwd?: string
+		cwd?: string,
+		respType?: string
 	): Promise<Record<string, unknown>> {
+		const headers: Record<string, string> = {};
+
+		if (cwd) headers[X_TOOL_CWD_HEADER] = cwd;
+
+		if (respType) headers[X_RESP_TYPE_HEADER] = respType;
+
 		return apiFetch<Record<string, unknown>>(API_TOOLS.EXECUTE, {
 			body: JSON.stringify({ params, tool: toolName }),
-			headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined,
+			headers: Object.keys(headers).length > 0 ? headers : undefined,
 			method: 'POST',
 			signal
 		});

tools/ui/src/lib/stores/agentic.svelte.ts

@@ -22,7 +22,9 @@
 
 import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants';
 import {
+	AUDIO_MIME_TO_EXTENSION,
 	DATA_URI_BASE64_REGEX,
+	DEFAULT_AUDIO_EXTENSION,
 	DEFAULT_IMAGE_EXTENSION,
 	IMAGE_MIME_TO_EXTENSION,
 	MCP_ATTACHMENT_NAME_PREFIX
@@ -36,6 +38,7 @@ import {
 	ToolCallType
 } from '$lib/enums';
 import { ChatService } from '$lib/services';
+import { ReadMediaService } from '$lib/services/read-media.service';
 import { SandboxService } from '$lib/services/sandbox.service';
 import { ToolsService } from '$lib/services/tools.service';
 import { conversationsStore } from '$lib/stores/conversations.svelte';
@@ -75,9 +78,10 @@ import type {
 import type {
 	DatabaseMessage,
 	DatabaseMessageExtra,
+	DatabaseMessageExtraAudioFile,
 	DatabaseMessageExtraImageFile
 } from '$lib/types/database';
-import { isAbortError } from '$lib/utils';
+import { getAudioInputFormat, isAbortError } from '$lib/utils';
 import { SvelteMap } from 'svelte/reactivity';
 
 function createDefaultSession(): AgenticSession {
@@ -900,7 +904,18 @@ class AgenticStore {
 							if (executionResult.isError) toolSuccess = false;
 						} else if (toolSource === ToolSource.FRONTEND) {
 							const args = this.parseToolArguments(toolCall.function.arguments);
-							const executionResult = await SandboxService.executeTool(toolName, args, signal);
+							const executionResult =
+								toolName === BuiltInTool.READ_MEDIA
+									? await ReadMediaService.executeTool(
+											args,
+											{
+												audio: modelsStore.modelSupportsAudio(effectiveModel),
+												vision: modelsStore.modelSupportsVision(effectiveModel)
+											},
+											signal,
+											conversationsStore.activeConversation?.cwd
+										)
+									: await SandboxService.executeTool(toolName, args, signal);
 
 							result = executionResult.content;
 
@@ -990,7 +1005,19 @@ class AgenticStore {
 				];
 
 				for (const attachment of attachments) {
-					if (attachment.type === AttachmentType.IMAGE) {
+					if (attachment.type === AttachmentType.AUDIO) {
+						if (modelsStore.modelSupportsAudio(effectiveModel)) {
+							contentParts.push({
+								input_audio: {
+									data: (attachment as DatabaseMessageExtraAudioFile).base64Data,
+									format: getAudioInputFormat(
+										(attachment as DatabaseMessageExtraAudioFile).mimeType
+									)
+								},
+								type: ContentPartType.INPUT_AUDIO
+							});
+						}
+					} else if (attachment.type === AttachmentType.IMAGE) {
 						if (modelsStore.modelSupportsVision(effectiveModel)) {
 							contentParts.push({
 								image_url: {
@@ -1101,6 +1128,18 @@ class AgenticStore {
 				return `[Attachment saved: ${name}]`;
 			}
 
+			if (mimeType.startsWith(MimeTypePrefix.AUDIO)) {
+				// audio extras hold the bare base64, the input_audio part has no room for a data URI
+				attachments.push({
+					base64Data,
+					mimeType,
+					name,
+					type: AttachmentType.AUDIO
+				});
+
+				return `[Attachment saved: ${name}]`;
+			}
+
 			return line;
 		});
 
@@ -1108,7 +1147,9 @@ class AgenticStore {
 	}
 
 	private buildAttachmentName(mimeType: string, index: number): string {
-		const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION;
+		const extension = mimeType.startsWith(MimeTypePrefix.AUDIO)
+			? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION)
+			: (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION);
 
 		return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`;
 	}

tools/ui/src/lib/stores/models.svelte.ts

@@ -18,7 +18,10 @@ import { ModelsService } from '$lib/services/models.service';
 import { PropsService } from '$lib/services/props.service';
 import { conversationsStore } from '$lib/stores/conversations.svelte';
 import { isRouterMode, serverStore } from '$lib/stores/server.svelte';
-import { getAuthHeaders, TTLCache } from '$lib/utils';
+// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back
+// into the stores, and going through it here would read a half-built module
+import { getAuthHeaders } from '$lib/utils/api-headers';
+import { TTLCache } from '$lib/utils/cache-ttl';
 import {
 	detectThinkingSupport,
 	detectThinkingSupportWithReason

tools/ui/src/lib/stores/tools.svelte.ts

@@ -1,4 +1,5 @@
 import {
+	buildReadMediaToolDefinition,
 	buildSandboxToolDefinition,
 	DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
 	HOME_TILDE,
@@ -15,6 +16,7 @@ import {
 } from '$lib/enums';
 import { ToolsService } from '$lib/services/tools.service';
 import { mcpStore } from '$lib/stores/mcp.svelte';
+import { modelsStore, selectedModelName } from '$lib/stores/models.svelte';
 import { config } from '$lib/stores/settings.svelte';
 import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types';
 import { SvelteMap, SvelteSet } from 'svelte/reactivity';
@@ -168,9 +170,42 @@ class ToolsStore {
 	}
 
 	get frontendTools(): OpenAIToolDefinition[] {
-		return config().jsSandboxEnabled
-			? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)]
-			: [];
+		const tools: OpenAIToolDefinition[] = [];
+
+		if (config().jsSandboxEnabled) {
+			tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled));
+		}
+
+		const readMedia = this.readMediaTool();
+
+		if (readMedia) tools.push(readMedia);
+
+		return tools;
+	}
+
+	/**
+	 * `read_media` runs in the frontend on top of the server's `read_file`, so it
+	 * exists only when that tool is served and the active model can perceive the
+	 * bytes. The server cannot make this call - it does not know which model the
+	 * conversation uses.
+	 */
+	private readMediaTool(): OpenAIToolDefinition | null {
+		const hasReadFile = this._builtinTools.some(
+			(def) => def.function.name === BuiltInTool.READ_FILE
+		);
+
+		if (!hasReadFile) return null;
+
+		const model = selectedModelName() ?? modelsStore.models[0]?.model ?? '';
+
+		if (!model) return null;
+
+		const vision = modelsStore.modelSupportsVision(model);
+		const audio = modelsStore.modelSupportsAudio(model);
+
+		if (!vision && !audio) return null;
+
+		return buildReadMediaToolDefinition(vision, audio);
 	}
 
 	get customTools(): OpenAIToolDefinition[] {

tools/ui/src/lib/utils/agentic.ts

@@ -50,11 +50,11 @@ export interface AgenticSection {
 }
 
 /**
- * Represents a tool result line that may reference an image attachment
+ * Represents a tool result line that may reference a media attachment (image or audio)
  */
 export type ToolResultLine = {
 	text: string;
-	image?: DatabaseMessageExtraImageFile;
+	media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile;
 };
 
 /**
@@ -301,16 +301,16 @@ export function splitSearchSummaryList(
 	return { lines };
 }
 
-/** Bounded cache for parseToolResultWithImages results. */
+/** Bounded cache for parseToolResultWithMedia results. */
 const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32;
 const toolResultLinesCache = new Map<string, ToolResultLine[]>();
 
 /**
- * Parse tool result text into lines, matching image attachments by name.
+ * Parse tool result text into lines, matching media attachments (images and audio) by name.
  * Memoized: called per render during streaming on unchanged tool result
  * strings with unchanged extras.
  */
-export function parseToolResultWithImages(
+export function parseToolResultWithMedia(
 	toolResult: string,
 	extras?: DatabaseMessageExtra[]
 ): ToolResultLine[] {
@@ -332,12 +332,13 @@ export function parseToolResultWithImages(
 		if (!match || !extras) return { text: line };
 
 		const attachmentName = match[1];
-		const image = extras.find(
-			(e): e is DatabaseMessageExtraImageFile =>
-				e.type === AttachmentType.IMAGE && e.name === attachmentName
+		const media = extras.find(
+			(e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+				(e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+				e.name === attachmentName
 		);
 
-		return { image, text: line };
+		return { media, text: line };
 	});
 
 	if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) {

tools/ui/src/lib/utils/audio-format.ts

@@ -0,0 +1,22 @@
+import { FileTypeAudio, MimeTypeAudio } from '$lib/enums';
+import type { AudioInputFormat } from '$lib/types/api';
+
+/**
+ * Map a MIME type to the AudioInputFormat expected by the API.
+ */
+export function getAudioInputFormat(mimeType: string): AudioInputFormat {
+	const normalizedMimeType = mimeType.trim().toLowerCase();
+
+	if (
+		normalizedMimeType === MimeTypeAudio.WAV ||
+		normalizedMimeType === MimeTypeAudio.WAVE ||
+		normalizedMimeType === MimeTypeAudio.X_WAV ||
+		normalizedMimeType === MimeTypeAudio.X_WAVE ||
+		normalizedMimeType === MimeTypeAudio.VND_WAVE ||
+		normalizedMimeType === MimeTypeAudio.X_PN_WAV
+	) {
+		return FileTypeAudio.WAV;
+	}
+
+	return FileTypeAudio.MP3;
+}

tools/ui/src/lib/utils/index.ts

@@ -248,7 +248,7 @@ export {
 export {
 	deriveAgenticSections,
 	buildAssistantRawOutput,
-	parseToolResultWithImages,
+	parseToolResultWithMedia,
 	splitSearchSummaryList,
 	hasAgenticContent,
 	classifyToolResult,
@@ -325,3 +325,6 @@ export { uuid } from './uuid';
 
 // CSS utilities
 export { remToPx } from './css';
+
+// Audio format helper (used by agentic store and chat service)
+export { getAudioInputFormat } from './audio-format';

tools/ui/tests/client/README-perf.md

@@ -33,12 +33,12 @@ npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.t
 
 The point of the harness is the _scaling curve_, not any single number.
 
-| Knob                        | Reads on                                                                                             |
-| --------------------------- | ---------------------------------------------------------------------------------------------------- |
-| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed.                                       |
-| `toolResultBytes`           | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). |
-| `editFileEdits`             | `computeLineDiff`, the O(m\*n) LCS.                                                                  |
-| `openCodeFence`             | `hljs.highlightAuto` on partial code.                                                                |
+| Knob                        | Reads on                                                                                            |
+| --------------------------- | --------------------------------------------------------------------------------------------------- |
+| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed.                                      |
+| `toolResultBytes`           | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). |
+| `editFileEdits`             | `computeLineDiff`, the O(m\*n) LCS.                                                                 |
+| `openCodeFence`             | `hljs.highlightAuto` on partial code.                                                               |
 
 Deliberately no hard assertions: CI timing is noisy and the value here is the
 before/after delta, not a gate.

tools/ui/tests/unit/agentic-hotpath.bench.ts

@@ -6,7 +6,7 @@
 //
 // Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts
 
-import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic';
+import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic';
 import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code';
 import { computeLineDiff } from '$lib/utils/compute-line-diff';
 import { preprocessLaTeX } from '$lib/utils/latex-protection';
@@ -200,17 +200,17 @@ describe('exit-code regex', () => {
 
 // --- per-line result parsers ----------------------------------------------
 
-describe('parseToolResultWithImages', () => {
+describe('parseToolResultWithMedia', () => {
 	bench('1KB', () => {
-		parseToolResultWithImages(SHELL_OUTPUT_1KB, []);
+		parseToolResultWithMedia(SHELL_OUTPUT_1KB, []);
 	});
 
 	bench('200KB', () => {
-		parseToolResultWithImages(SHELL_OUTPUT_200KB, []);
+		parseToolResultWithMedia(SHELL_OUTPUT_200KB, []);
 	});
 
 	bench('2MB', () => {
-		parseToolResultWithImages(SHELL_OUTPUT_2MB, []);
+		parseToolResultWithMedia(SHELL_OUTPUT_2MB, []);
 	});
 });