LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: fix longest_edge ignoring min/max pixels (#26638)

Browse Source 
* mtmd: fix longest_edge ignoring min/max pixels

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-08-07 18:05:15 +02:00
committed by GitHub
parent 4cf5cab65d
commit 4cb22cd537
3 changed files with 65 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-model.h
+11
View File
@@ -170,6 +170,17 @@ struct clip_hparams {
warmup_image_size = static_cast<int>(std::sqrt(image_max_pixels));
}
// used by longest_edge preprocessor (no model-specific value for min/max tokens)
void set_limit_image_tokens() {
const int patch_area = patch_size * patch_size * n_merge * n_merge;
if (custom_image_min_tokens > 0) {
image_min_pixels = custom_image_min_tokens * patch_area;
}
if (custom_image_max_tokens > 0) {
image_max_pixels = custom_image_max_tokens * patch_area;
}
}
void set_warmup_n_tokens(int n_tokens) {
int n_tok_per_side = static_cast<int>(std::sqrt(n_tokens));
GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n");
tools/mtmd/clip.cpp
+3
View File
@@ -1434,6 +1434,7 @@ struct clip_model_loader {
// use default llava-uhd preprocessing params
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
hparams.set_limit_image_tokens();
} break;
case PROJECTOR_TYPE_LFM2:
{
@@ -1471,6 +1472,7 @@ struct clip_model_loader {
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.image_longest_edge = hparams.image_size;
get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
hparams.set_limit_image_tokens();
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_DOTS_OCR:
@@ -1595,6 +1597,7 @@ struct clip_model_loader {
if (hparams.image_longest_edge == 0) {
hparams.image_longest_edge = 3024;
}
// note: the step3vl preprocessor slices based on a fixed window grid, so it does not support custom min/max image tokens
hparams.warmup_image_size = hparams.image_size;
} break;
case PROJECTOR_TYPE_YOUTUVL:
tools/mtmd/mtmd-image.cpp
+51 -47
View File
@@ -139,50 +139,46 @@ struct img_tool {
}
}
// calculate the size of the **resized** image, while preserving the aspect ratio
// the calculated size will be aligned to the nearest multiple of align_size
// if H or W size is larger than longest_edge, it will be resized to longest_edge
static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int longest_edge) {
GGML_ASSERT(align_size > 0);
if (inp_size.width <= 0 || inp_size.height <= 0 || longest_edge <= 0) {
struct calc_size_opt {
int align_size = 1;
int min_pixels = 0; // 0 = disabled
int max_pixels = 0; // 0 = disabled
// applied before min/max_pixels, so min_pixels can push an edge back above longest_edge
int longest_edge = 0; // 0 = disabled
};
// calculate the size of the **resized** image, while preserving the aspect ratio and
// aligning to the nearest multiple of align_size ("smart_resize" in transformers code)
static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const calc_size_opt & opts) {
GGML_ASSERT(opts.align_size > 0);
const int width = inp_size.width;
const int height = inp_size.height;
if (width <= 0 || height <= 0) {
return {0, 0};
}
float scale = std::min(static_cast<float>(longest_edge) / inp_size.width,
static_cast<float>(longest_edge) / inp_size.height);
auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
float target_width_f = static_cast<float>(inp_size.width) * scale;
float target_height_f = static_cast<float>(inp_size.height) * scale;
int w_bar, h_bar;
if (opts.longest_edge > 0) {
const float scale = std::min(static_cast<float>(opts.longest_edge) / width,
static_cast<float>(opts.longest_edge) / height);
w_bar = ceil_by_factor(width * scale);
h_bar = ceil_by_factor(height * scale);
} else {
// always align up first
w_bar = std::max(opts.align_size, round_by_factor(width));
h_bar = std::max(opts.align_size, round_by_factor(height));
}
auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
int aligned_width = ceil_by_factor(target_width_f);
int aligned_height = ceil_by_factor(target_height_f);
return {aligned_width, aligned_height};
}
// calculate the size of the **resized** image, while preserving the aspect ratio
// the calculated size will have min_pixels <= W*H <= max_pixels
// this is referred as "smart_resize" in transformers code
static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int min_pixels, const int max_pixels) {
GGML_ASSERT(align_size > 0);
const int width = inp_size.width;
const int height = inp_size.height;
auto round_by_factor = [f = align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
auto floor_by_factor = [f = align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
// always align up first
int h_bar = std::max(align_size, round_by_factor(height));
int w_bar = std::max(align_size, round_by_factor(width));
if (h_bar * w_bar > max_pixels) {
const auto beta = std::sqrt(static_cast<float>(height * width) / max_pixels);
h_bar = std::max(align_size, floor_by_factor(height / beta));
w_bar = std::max(align_size, floor_by_factor(width / beta));
} else if (h_bar * w_bar < min_pixels) {
const auto beta = std::sqrt(static_cast<float>(min_pixels) / (height * width));
if (opts.max_pixels > 0 && h_bar * w_bar > opts.max_pixels) {
const auto beta = std::sqrt(static_cast<float>(height) * width / opts.max_pixels);
h_bar = std::max(opts.align_size, floor_by_factor(height / beta));
w_bar = std::max(opts.align_size, floor_by_factor(width / beta));
} else if (opts.min_pixels > 0 && h_bar * w_bar < opts.min_pixels) {
const auto beta = std::sqrt(static_cast<float>(opts.min_pixels) / (static_cast<float>(height) * width));
h_bar = ceil_by_factor(height * beta);
w_bar = ceil_by_factor(width * beta);
}
@@ -937,9 +933,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
const int cur_merge = hparams.n_merge;
const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
original_size,
hparams.patch_size * cur_merge,
hparams.image_min_pixels,
hparams.image_max_pixels);
{
/* align_size */ hparams.patch_size * cur_merge,
/* min_pixels */ hparams.image_min_pixels,
/* max_pixels */ hparams.image_max_pixels,
/* longest_edge */ 0,
});
img_tool::resize(img, resized_image, target_size,
hparams.image_resize_algo,
hparams.image_resize_pad,
@@ -961,8 +960,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const cl
const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;
const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
original_size,
hparams.patch_size * cur_merge,
hparams.image_longest_edge);
{
/* align_size */ hparams.patch_size * cur_merge,
/* min_pixels */ std::max(0, hparams.image_min_pixels),
/* max_pixels */ std::max(0, hparams.image_max_pixels),
/* longest_edge */ hparams.image_longest_edge,
});
img_tool::resize(img, resized_image, target_size,
hparams.image_resize_algo,
hparams.image_resize_pad,
@@ -1000,8 +1003,8 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
inst.overview_size = img_tool::calc_size_preserved_ratio(
original_size, align_size,
hparams.image_min_pixels, hparams.image_max_pixels);
original_size,
{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
// tile if either dimension exceeds tile_size with tolerance
const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
@@ -1109,7 +1112,8 @@ mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_i
// CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L737
const clip_image_size original_size = img.get_size();
const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(
original_size, hparams.image_size, hparams.image_longest_edge);
original_size,
{ hparams.image_size, std::max(0, hparams.image_min_pixels), std::max(0, hparams.image_max_pixels), hparams.image_longest_edge });
// LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n",
// __func__, original_size.width, original_size.height,
// refined_size.width, refined_size.height);