mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-08-12 22:31:11 +04:00
4dd127584b
* server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the <img> tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft <ckrafft@epyc> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
74 lines
2.4 KiB
TypeScript
74 lines
2.4 KiB
TypeScript
import { MimeTypeAudio, MimeTypeImage } from '$lib/enums';
|
|
|
|
// File extension patterns for resource type detection
|
|
export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i;
|
|
export const CODE_FILE_EXTENSION_REGEX =
|
|
/\.(js|ts|json|yaml|yml|xml|html|css|py|rs|go|java|cpp|c|h|rb|sh|toml)$/i;
|
|
export const TEXT_FILE_EXTENSION_REGEX = /\.(txt|md|log)$/i;
|
|
|
|
// URI protocol prefix pattern
|
|
export const PROTOCOL_PREFIX_REGEX = /^[a-z]+:\/\//;
|
|
|
|
// File extension regex for display name extraction
|
|
export const FILE_EXTENSION_REGEX = /\.[^.]+$/;
|
|
|
|
// Separator regex for splitting display names (kebab-case/snake_case)
|
|
export const DISPLAY_NAME_SEPARATOR_REGEX = /[-_]/;
|
|
|
|
// Regex for matching base64-encoded data URIs
|
|
export const DATA_URI_BASE64_REGEX = /^data:([^;]+);base64,([A-Za-z0-9+/]+=*)$/;
|
|
|
|
// Prefix for MCP attachment filenames
|
|
export const MCP_ATTACHMENT_NAME_PREFIX = 'mcp-attachment';
|
|
|
|
// Prefix for MCP resource attachment IDs
|
|
export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res';
|
|
|
|
// Default file extension for unknown image types
|
|
export const DEFAULT_IMAGE_EXTENSION = 'img';
|
|
|
|
// Default file extension for unknown audio types
|
|
export const DEFAULT_AUDIO_EXTENSION = 'mp3';
|
|
|
|
// Default filename for resource content downloads
|
|
export const DEFAULT_RESOURCE_FILENAME = 'resource.txt';
|
|
|
|
// Path separator for resource URI parsing
|
|
export const PATH_SEPARATOR = '/';
|
|
|
|
// Separator for joining text content from multiple resource parts
|
|
export const RESOURCE_TEXT_CONTENT_SEPARATOR = '\n\n';
|
|
|
|
// Fallback text for unknown content types
|
|
export const RESOURCE_UNKNOWN_TYPE = 'unknown type';
|
|
|
|
// Label prefix for binary blob content
|
|
export const BINARY_CONTENT_LABEL = 'Binary content';
|
|
|
|
/**
|
|
* Mapping from image MIME types to file extensions.
|
|
* Used for generating attachment filenames from MIME types.
|
|
*/
|
|
export const IMAGE_MIME_TO_EXTENSION: Record<string, string> = {
|
|
[MimeTypeImage.GIF]: 'gif',
|
|
[MimeTypeImage.JPEG]: 'jpg',
|
|
[MimeTypeImage.JPG]: 'jpg',
|
|
[MimeTypeImage.PNG]: 'png',
|
|
[MimeTypeImage.WEBP]: 'webp'
|
|
} as const;
|
|
|
|
/**
|
|
* Mapping from audio MIME types to file extensions.
|
|
* Used for generating attachment filenames from MIME types.
|
|
*/
|
|
export const AUDIO_MIME_TO_EXTENSION: Record<string, string> = {
|
|
[MimeTypeAudio.MP3]: 'mp3',
|
|
[MimeTypeAudio.MP3_MPEG]: 'mp3',
|
|
[MimeTypeAudio.VND_WAVE]: 'wav',
|
|
[MimeTypeAudio.WAV]: 'wav',
|
|
[MimeTypeAudio.WAVE]: 'wav',
|
|
[MimeTypeAudio.X_PN_WAV]: 'wav',
|
|
[MimeTypeAudio.X_WAV]: 'wav',
|
|
[MimeTypeAudio.X_WAVE]: 'wav'
|
|
} as const;
|