diff --git a/conversion/__init__.py b/conversion/__init__.py index c5ecc68cfd..b2bb7e5161 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -288,6 +288,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "LlavaForConditionalGeneration": "llava", "MERaLiON2ForConditionalGeneration": "ultravox", "MiMoV2ForCausalLM": "mimo", + "MiniMaxM3SparseForConditionalGeneration": "minimax", "MiniCPMV4_6ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "NemotronH_Nano_VL_V2": "nemotron", diff --git a/conversion/minimax.py b/conversion/minimax.py index e82e393a38..c2175cc932 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -7,7 +7,7 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf +from .base import ModelBase, TextModel, MmprojModel, gguf @ModelBase.register("MiniMaxM2ForCausalLM") @@ -92,3 +92,78 @@ class MiniMaxM3Model(MiniMaxM2Model): data_torch = data_torch + 1.0 yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration") +class MiniMaxM3VisionModel(MmprojModel): + @classmethod + def filter_tensors(cls, item): + name, gen = item + # keep only the vision-side tensors; text / mtp / sparse-index are dropped + if not name.startswith(("vision_tower.", "multi_modal_projector.", "patch_merge_mlp.")): + return None + return super().filter_tensors((name, gen)) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINIMAXM3) + self.gguf_writer.add_vision_use_gelu(True) + + # the ViT carries its own LayerNorm eps (text tower uses a different one) + self.gguf_writer.add_vision_attention_layernorm_eps( + self.hparams_vision.get("layer_norm_eps", 1e-5) + ) + + comp = self.hparams_vision.get("img_token_compression_config", {}) + merge_size = comp.get("spatial_merge_size", 2) + self.gguf_writer.add_vision_spatial_merge_size(int(merge_size)) + + def modify_tensors(self, data_torch, name, bid): + assert self.hparams_vision is not None + + # Conv3d patch embed -> Conv2d slices + if name == "vision_tower.vision_model.embeddings.patch_embedding.weight": + if data_torch.ndim != 5: + raise ValueError(f"unexpected patch_embedding rank {data_torch.ndim} for {name}") + kt = data_torch.shape[2] + base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + for t in range(kt): + suffix = ".weight" if t == 0 else f".weight.{t}" + yield (base + suffix, data_torch[:, :, t, ...]) + return + + # Permute ViT q/k. HF [Ta Ha Wa | Tb Hb Wb | pad] reorder to [Ta Tb | Ha Hb | Wa Wb | pad]. + for new_name, tensor in super().modify_tensors(data_torch, name, bid): + if ".attn_q." in new_name or ".attn_k." in new_name: + tensor = self._permute_vit_qk(tensor, new_name) + yield new_name, tensor + + def _permute_vit_qk(self, t: "Tensor", new_name: str) -> "Tensor": + assert self.hparams_vision is not None + n_head = self.hparams_vision["num_attention_heads"] + d_head = t.shape[0] // n_head + axis_dim = 2 * ((2 * (d_head // 2) // 3) // 2) + ah = axis_dim // 2 + half = 3 * ah + perm = [] + perm += list(range(0, ah)) + perm += list(range(half, half + ah)) + perm += list(range(ah, 2 * ah)) + perm += list(range(half + ah, half + 2 * ah)) + perm += list(range(2 * ah, 3 * ah)) + perm += list(range(half + 2 * ah, half + 3 * ah)) + perm += list(range(2 * half, d_head)) + + assert axis_dim % 2 == 0 + assert 3 * axis_dim <= d_head + assert len(perm) == d_head + assert sorted(perm) == list(range(d_head)), "perm is not a bijection of d_head" + assert t.shape[0] == n_head * d_head, f"{new_name}: {t.shape[0]} != {n_head}*{d_head}" + assert d_head == 80 + + idx = torch.tensor(perm, dtype=torch.long) + if t.ndim == 2: + return t.reshape(n_head, d_head, t.shape[1])[:, idx, :].reshape(t.shape) + return t.reshape(n_head, d_head)[:, idx].reshape(t.shape) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 66d50cca26..2071e3eaa8 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -857,6 +857,8 @@ class MODEL_TENSOR(IntEnum): V_MM_UP = auto() # cogvlm V_MM_DOWN = auto() # cogvlm V_MM_GATE = auto() # cogvlm + V_MM_MERGER_FC1 = auto() # minimax-m3 (patch-merge MLP) + V_MM_MERGER_FC2 = auto() # minimax-m3 (patch-merge MLP) V_TOK_BOI = auto() # cogvlm V_TOK_EOI = auto() # cogvlm V_TOK_IMG_BEGIN = auto() # hunyuanvl @@ -1441,6 +1443,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_MM_UP: "mm.up", MODEL_TENSOR.V_MM_DOWN: "mm.down", MODEL_TENSOR.V_MM_GATE: "mm.gate", + MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1", + MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2", MODEL_TENSOR.V_TOK_BOI: "v.boi", MODEL_TENSOR.V_TOK_EOI: "v.eoi", MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm", @@ -1637,6 +1641,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.V_RESMPL_QUERY, MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK, MODEL_TENSOR.V_MM_PATCH_MERGER, + MODEL_TENSOR.V_MM_MERGER_FC1, + MODEL_TENSOR.V_MM_MERGER_FC2, MODEL_TENSOR.V_DS_NORM, MODEL_TENSOR.V_DS_FC1, MODEL_TENSOR.V_DS_FC2, @@ -4771,6 +4777,7 @@ class VisionProjectorType: YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" HUNYUANVL = "hunyuanvl" + MINIMAXM3 = "minimax_m3" MINICPMV4_6 = "minicpmv4_6" GRANITE_SPEECH = "granite_speech" # audio MIMOVL = "mimovl" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 59623accfd..62d7a827e3 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1838,6 +1838,14 @@ class TensorNameMap: "visual.downsample", # glm4v ), + MODEL_TENSOR.V_MM_MERGER_FC1: ( + "patch_merge_mlp.linear_1", # minimax-m3 + ), + + MODEL_TENSOR.V_MM_MERGER_FC2: ( + "patch_merge_mlp.linear_2", # minimax-m3 + ), + MODEL_TENSOR.V_DS_NORM: ( "model.visual.deepstack_merger_list.{bid}.norm", # deepstack in qwen3vl ), diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index 84075fea32..ba76c48115 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -110,6 +110,15 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re - Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design. - Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests. +## Multimodal (`tools/mtmd/`) + +- Tensor names must be prefixed by `v.`, `a.`, `mm.` or `a.mm.` (legacy naming doesn't follow this convention - this is expected, but new code should follow it). +- Do not use explicit sin/cos for RoPE; use `ggml_rope_ext` instead, see `HOWTO-add-model.md`. If it can't express the needed behavior, that's a design discussion, not a PR. +- New GGML ops must not be introduced in the same PR, you must push it as a separate PR. +- In most cases, `build_vit` should be enough to build the transformer graph for vision models. Do not add a loop to build the transformer graph manually, unless you have a very good reason to do so. If you do, please explain why in the PR description. +- If you need a dedicated preprocessor, there is a high chance that it can be a derived class from one of the existing preprocessors. Check carefully before adding a new preprocessor class. +- If the model need a new public API in `mtmd.h`, open a discussion first. + ## General (always) Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on every changed line - this is a distinct pass from checking that the code works, and matters just as much for review speed: diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index d0329ca567..fd7ddceb0b 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -47,6 +47,7 @@ add_library(mtmd models/paddleocr.cpp models/pixtral.cpp models/qwen2vl.cpp + models/minimax-m3.cpp models/qwen3vl.cpp models/mimovl.cpp models/qwen3a.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 5b413681f0..42374311ce 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -131,6 +131,8 @@ #define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3 #define TN_MM_PROJECTOR "mm.model.fc.%s" // idefics3, deepseekocr #define TN_MM_PATCH_MERGER "mm.patch_merger.%s" // mistral small 3.1, glm4v +#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP +#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s" #define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral #define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model) #define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model) @@ -370,6 +372,7 @@ enum projector_type { PROJECTOR_TYPE_MINICPMV4_6, PROJECTOR_TYPE_GRANITE_SPEECH, PROJECTOR_TYPE_MIMOVL, + PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_UNKNOWN, }; @@ -424,6 +427,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, + { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, }; diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 6d4336c401..850957d7de 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -397,6 +397,10 @@ struct clip_model { ggml_tensor * mm_0_b = nullptr; ggml_tensor * mm_2_w = nullptr; ggml_tensor * mm_2_b = nullptr; + ggml_tensor * mm_merger_fc1_w = nullptr; // minimax-m3 + ggml_tensor * mm_merger_fc1_b = nullptr; + ggml_tensor * mm_merger_fc2_w = nullptr; + ggml_tensor * mm_merger_fc2_b = nullptr; ggml_tensor * image_newline = nullptr; ggml_tensor * view_seperator = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b886650649..e0e2107a0b 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -915,6 +915,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_STEP3VL: { builder = std::make_unique(ctx, img); @@ -1469,6 +1473,17 @@ struct clip_model_loader { LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + hparams.n_merge = 2; // spatial_merge_size + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.rope_theta = 10000.0f; // vision_config.rope_theta + // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length) + hparams.set_limit_image_tokens(8, 576); + hparams.set_warmup_n_tokens(16*16); + } break; case PROJECTOR_TYPE_MIMOVL: { hparams.n_merge = 2; // spatial_merge_size @@ -2089,6 +2104,19 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + // per-patch MLP: mm.1 -> gelu -> mm.2 + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2 + model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight")); + model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias")); + model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight")); + model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias")); + } break; case PROJECTOR_TYPE_STEP3VL: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); @@ -3360,6 +3388,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_MINIMAX_M3: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_YOUTUVL: { @@ -3866,6 +3895,24 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 set_input_i32("positions", positions); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + const int n_merge = hparams.n_merge; + const int gh = image_size_height / patch_size; + const int gw = image_size_width / patch_size; + std::vector pos_h, pos_w; + pos_h.reserve(gh * gw); + pos_w.reserve(gh * gw); + for (int bh = 0; bh < gh / n_merge; bh++) + for (int bw = 0; bw < gw / n_merge; bw++) + for (int mh = 0; mh < n_merge; mh++) + for (int mw = 0; mw < n_merge; mw++) { + pos_h.push_back(bh * n_merge + mh); + pos_w.push_back(bw * n_merge + mw); + } + set_input_i32("minimax_pos_h", pos_h); + set_input_i32("minimax_pos_w", pos_w); + } break; case PROJECTOR_TYPE_DOTS_OCR: { const int pw = image_size_width / patch_size; @@ -4569,6 +4616,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_GLM_EDGE: return ctx->model.mm_model_mlp_3_w->ne[1]; + case PROJECTOR_TYPE_MINIMAX_M3: + return ctx->model.mm_merger_fc2_b->ne[0]; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_EXAONE4_5: diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp new file mode 100644 index 0000000000..447621754e --- /dev/null +++ b/tools/mtmd/models/minimax-m3.cpp @@ -0,0 +1,84 @@ +#include "models.h" + +ggml_tensor * clip_graph_minimax_m3::apply_rope( + ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { + const int64_t Hn = x->ne[1]; + const int64_t P = x->ne[2]; + const size_t es = ggml_element_size(x); + const int dh = (int) x->ne[0]; + const int axd = 2 * ((2 * (dh / 2) / 3) / 2); + + GGML_ASSERT(x->nb[0] == es); + GGML_ASSERT(3 * axd <= dh); + + const float th = hparams.rope_theta; + + // layout of x is [t, h, w, pad] + // t is unrotated, h and w are rotated, pad is unrotated + // note: everything from n_dims onward untouched, so w and pad are rotated in one call. + auto sl = [&](int off, int n) { + return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es)); + }; + ggml_tensor * t = sl(0, axd); + ggml_tensor * h = sl(axd, axd); + ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad + + h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0); +} + +ggml_cgraph * clip_graph_minimax_m3::build() { + GGML_ASSERT(model.patch_bias == nullptr); + GGML_ASSERT(model.class_embedding == nullptr); + GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1); + GGML_ASSERT(model.mm_1_w && model.mm_2_w); + GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w); + + const int batch_size = 1; + const int n_pos = n_patches; + const int merge = hparams.n_merge; + + // patch embedding + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * inp = ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + + // spatial merge + { + inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); + inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size); + inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge)); + inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); + inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size); + } + + // t (time axis) is always 0 for now, so we leave it unrotated + ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h); + ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w); + + ggml_tensor * inpL = build_vit( + inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, + [&](ggml_tensor * c, const clip_layer &) { + return apply_rope(c, pos_h, pos_w); + }); + + // projector + ggml_tensor * emb = inpL; + emb = build_ffn(emb, model.mm_1_w, model.mm_1_b, + nullptr, nullptr, + model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1); + + const int64_t proj = emb->ne[0]; + emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge)); + + emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b, + nullptr, nullptr, + model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1); + + ggml_build_forward_expand(gf, emb); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 5f1493fa60..2d7555da41 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -40,6 +40,12 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl { ggml_cgraph * build() override; }; +struct clip_graph_minimax_m3 : clip_graph { + clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w); +}; + struct clip_graph_mimovl : clip_graph { clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5915b4cba9..bb49b211ef 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -463,6 +463,13 @@ struct mtmd_context { img_end = "<|vision_end|>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[ + img_beg = "]<]start of image[>["; + img_end = "]<]end of image[>["; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_YOUTUVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|>