mtmd: add MiniMax M3 vision support (#2086)

Co-authored-by: Smart <smart@augmented-special.services>

examples/mtmd/clip-impl.h

@@ -40,6 +40,7 @@
 #define KEY_FEATURE_LAYER       "clip.vision.feature_layer"
 #define KEY_PROJ_SCALE_FACTOR   "clip.vision.projector.scale_factor"
 #define KEY_SPATIAL_MERGE_SIZE  "clip.vision.spatial_merge_size"
+#define KEY_TEMPORAL_PATCH_SIZE "clip.vision.temporal_patch_size"
 #define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"
 
 #define KEY_MM_PATCH_MERGE_TYPE   "clip.vision.mm_patch_merge_type"
@@ -167,6 +168,7 @@ enum projector_type {
     PROJECTOR_TYPE_LIGHTONOCR,
     PROJECTOR_TYPE_COGVLM,
     PROJECTOR_TYPE_JANUS_PRO,
+    PROJECTOR_TYPE_MINIMAX_M3_VL,
     PROJECTOR_TYPE_UNKNOWN,
 
 };
@@ -197,6 +199,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},
     { PROJECTOR_TYPE_COGVLM,    "cogvlm"},
     { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
+    { PROJECTOR_TYPE_MINIMAX_M3_VL, "minimax_m3_vl"},
 };
 
 static projector_type clip_projector_type_from_string(const std::string & str) {

examples/mtmd/clip.cpp

@@ -192,6 +192,7 @@ struct clip_hparams {
     int32_t image_min_pixels = -1;
     int32_t image_max_pixels = -1;
     int32_t n_merge = 0; // number of patch merges **per-side**
+    int32_t temporal_patch_size = 1;
 
     float image_mean[3];
     float image_std[3];
@@ -758,6 +759,101 @@ struct clip_graph {
         return gf;
     }
 
+    ggml_cgraph * build_minimax_m3_vl() {
+        GGML_ASSERT(model.patch_bias == nullptr);
+        GGML_ASSERT(model.class_embedding == nullptr);
+        GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+        GGML_ASSERT(model.patch_embeddings_1 != nullptr);
+        GGML_ASSERT(hparams.n_merge == 2);
+        GGML_ASSERT(img.nx % (patch_size * hparams.n_merge) == 0);
+        GGML_ASSERT(img.ny % (patch_size * hparams.n_merge) == 0);
+
+        const int batch_size = 1;
+
+        // MiniMax-M3 uses 3-axis NEOX RoPE. Each axis gets an even slice of the
+        // head dim; any remainder is left unrotated.
+        const int rope_dims = 2 * (d_head / 2);
+        const int axis_dim  = 2 * ((rope_dims / 3) / 2);
+        const int rot_dim   = 3 * axis_dim;
+
+        ggml_tensor * rope_cos = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, d_head, 1, n_patches);
+        ggml_set_name(rope_cos, "rope_cos");
+        ggml_set_input(rope_cos);
+
+        ggml_tensor * rope_sin = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, d_head, 1, n_patches);
+        ggml_set_name(rope_sin, "rope_sin");
+        ggml_set_input(rope_sin);
+
+        ggml_tensor * inp_raw = build_inp_raw();
+
+        // The generic convolution path converts im2col output to f16. MiniMax-M3's
+        // patch embedding is sensitive to that loss, so keep the accumulation in f32.
+        auto conv_2d_f32 = [&](ggml_tensor * kernel) {
+            ggml_tensor * kernel_f32 = ggml_cast(ctx0, kernel, GGML_TYPE_F32);
+            ggml_tensor * col = ggml_im2col(
+                ctx0, kernel_f32, inp_raw,
+                patch_size, patch_size, 0, 0, 1, 1,
+                true, GGML_TYPE_F32);
+            ggml_tensor * cur = ggml_mul_mat(
+                ctx0,
+                ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[3] * col->ne[2] * col->ne[1]),
+                ggml_reshape_2d(ctx0, kernel_f32, kernel_f32->ne[0] * kernel_f32->ne[1] * kernel_f32->ne[2], kernel_f32->ne[3]));
+            cur = ggml_reshape_4d(ctx0, cur, col->ne[1], col->ne[2], col->ne[3], kernel->ne[3]);
+            return ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2));
+        };
+
+        ggml_tensor * inp = conv_2d_f32(model.patch_embeddings_0);
+        inp = ggml_add(ctx0, inp, conv_2d_f32(model.patch_embeddings_1));
+        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
+        inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches, batch_size);
+        cb(inp, "patch_embd", -1);
+
+        auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
+            return build_rope_vision_neox(ctx0, cur, rope_cos, rope_sin, rot_dim);
+        };
+
+        ggml_tensor * cur = build_vit(
+            inp,
+            n_patches,
+            NORM_TYPE_NORMAL,
+            hparams.ffn_op,
+            nullptr,
+            add_pos);
+
+        cur = ggml_mul_mat(ctx0, model.mm_0_w, cur);
+        cur = ggml_add(ctx0, cur, model.mm_0_b);
+        cur = ggml_gelu_erf(ctx0, cur);
+
+        cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
+        cur = ggml_add(ctx0, cur, model.mm_1_b);
+
+        // The reference processor groups patches by 2x2 merge blocks. We keep the
+        // ViT tokens in raster order, then reorder here before concatenating each
+        // merge block for patch_merge_mlp.
+        cur = ggml_reshape_4d(ctx0, cur,
+            hparams.projection_dim * hparams.n_merge,
+            n_patches_x / hparams.n_merge,
+            hparams.n_merge,
+            batch_size * (n_patches_y / hparams.n_merge));
+        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
+        cur = ggml_cont(ctx0, cur);
+
+        cur = ggml_reshape_3d(ctx0, cur, hparams.projection_dim * hparams.n_merge * hparams.n_merge,
+            n_patches / (hparams.n_merge * hparams.n_merge), batch_size);
+
+        cur = ggml_mul_mat(ctx0, model.mm_2_w, cur);
+        cur = ggml_add(ctx0, cur, model.mm_2_b);
+        cur = ggml_gelu_erf(ctx0, cur);
+
+        cur = ggml_mul_mat(ctx0, model.mm_3_w, cur);
+        cur = ggml_add(ctx0, cur, model.mm_3_b);
+        cur = ggml_reshape_3d(ctx0, cur, hparams.projection_dim, n_patches / (hparams.n_merge * hparams.n_merge), batch_size);
+
+        ggml_build_forward_expand(gf, cur);
+
+        return gf;
+    }
+
     // Qwen2VL and Qwen2.5VL use M-RoPE
     ggml_cgraph * build_qwen2vl() {
         GGML_ASSERT(model.patch_bias == nullptr);
@@ -2788,6 +2884,29 @@ private:
         return cur;
     }
 
+    // MiniMax-M3 vision RoPE uses rotate_half semantics over host-computed
+    // cos/sin tables. cur is [d_head, n_head, n_pos]; cos/sin broadcast over heads.
+    static ggml_tensor * build_rope_vision_neox(
+            ggml_context * ctx0, ggml_tensor * cur,
+            ggml_tensor * cos, ggml_tensor * sin, int rot_dim) {
+        const int64_t d_head = cur->ne[0];
+        const int64_t n_head = cur->ne[1];
+        const int64_t n_pos  = cur->ne[2];
+        const int64_t half   = rot_dim / 2;
+        const size_t  es     = ggml_element_size(cur);
+
+        ggml_tensor * first  = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, half, n_head, n_pos, cur->nb[1], cur->nb[2], 0));
+        ggml_tensor * second = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, half, n_head, n_pos, cur->nb[1], cur->nb[2], half * es));
+        ggml_tensor * rotated = ggml_concat(ctx0, ggml_neg(ctx0, second), first, 0);
+
+        if (rot_dim < d_head) {
+            ggml_tensor * tail = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, d_head - rot_dim, n_head, n_pos, cur->nb[1], cur->nb[2], rot_dim * es));
+            rotated = ggml_concat(ctx0, rotated, tail, 0);
+        }
+
+        return ggml_add(ctx0, ggml_mul(ctx0, cur, cos), ggml_mul(ctx0, rotated, sin));
+    }
+
     // aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
     // support dynamic resolution
     ggml_tensor * build_patch_merge_permute(ggml_tensor * cur, int scale_factor) {
@@ -2878,6 +2997,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
             {
                 res = graph.build_qwen3vl();
             } break;
+        case PROJECTOR_TYPE_MINIMAX_M3_VL:
+            {
+                res = graph.build_minimax_m3_vl();
+            } break;
         case PROJECTOR_TYPE_GEMMA4V:
             {
                 res = graph.build_gemma4();
@@ -3242,6 +3365,24 @@ struct clip_model_loader {
                             LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
                         }
                     } break;
+                case PROJECTOR_TYPE_MINIMAX_M3_VL:
+                    {
+                        hparams.rope_theta = 10000.0f;
+                        hparams.n_merge = 2;
+                        hparams.temporal_patch_size = 2;
+                        hparams.ffn_op = FFN_GELU_ERF;
+                        log_ffn_op = "gelu_erf";
+                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+                        get_u32(KEY_TEMPORAL_PATCH_SIZE, hparams.temporal_patch_size, false);
+                        get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false);
+                        get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false);
+                        if (hparams.image_min_pixels <= 0 || hparams.image_max_pixels <= 0) {
+                            hparams.set_limit_image_tokens(8, 576);
+                        } else {
+                            hparams.warmup_image_size = static_cast<int>(std::sqrt(hparams.image_max_pixels));
+                        }
+                        hparams.set_warmup_n_tokens(256);
+                    } break;
                 case PROJECTOR_TYPE_GEMMA4V:
                     {
                         hparams.rope_theta = 100.0f;
@@ -3296,6 +3437,7 @@ struct clip_model_loader {
                 LOG_INF("%s: has_llava_proj:     %d\n", __func__, hparams.has_llava_projector);
                 LOG_INF("%s: minicpmv_version:   %d\n", __func__, hparams.minicpmv_version);
                 LOG_INF("%s: n_merge:            %d\n", __func__, hparams.n_merge);
+                LOG_INF("%s: temporal_patch_size:%d\n", __func__, hparams.temporal_patch_size);
                 LOG_INF("%s: n_wa_pattern:       %d\n", __func__, hparams.n_wa_pattern);
                 if (hparams.image_min_pixels > 0) {
                     LOG_INF("%s: image_min_pixels:   %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : "");
@@ -3577,6 +3719,17 @@ struct clip_model_loader {
                     model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
                     model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
                 } break;
+            case PROJECTOR_TYPE_MINIMAX_M3_VL:
+                {
+                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 4, "weight"));
+                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 4, "bias"));
+                    model.mm_3_w = get_tensor(string_format(TN_LLAVA_PROJ, 6, "weight"));
+                    model.mm_3_b = get_tensor(string_format(TN_LLAVA_PROJ, 6, "bias"));
+                } break;
             case PROJECTOR_TYPE_GEMMA3:
                 {
                     model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
@@ -4590,8 +4743,6 @@ private:
     }
 };
 
-// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector
-// res_imgs memory is being allocated here, previous allocations will be freed if found
 bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) {
     clip_image_size original_size{img->nx, img->ny};
     auto & params = ctx->model.hparams;
@@ -4617,6 +4768,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_GEMMA4V:
+        case PROJECTOR_TYPE_MINIMAX_M3_VL:
             {
                 GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
                 clip_image_u8 resized;
@@ -4876,7 +5028,7 @@ const char * clip_patch_merge_type(const struct clip_ctx * ctx) {
 int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img) {
     const auto & params = ctx->model.hparams;
     const int n_total = clip_n_output_tokens(ctx, img);
-    if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL) {
+    if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL || ctx->proj_type() == PROJECTOR_TYPE_MINIMAX_M3_VL) {
         return img->nx / (params.patch_size * 2);
     }
     return n_total;
@@ -4884,7 +5036,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
 
 int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img) {
     const auto & params = ctx->model.hparams;
-    if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL) {
+    if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL || ctx->proj_type() == PROJECTOR_TYPE_MINIMAX_M3_VL) {
         return img->ny / (params.patch_size * 2);
     }
     return 1;
@@ -4942,6 +5094,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_QWEN3VL:
+        case PROJECTOR_TYPE_MINIMAX_M3_VL:
             {
                 // dynamic size (2 conv, so double patch size)
                 int x_patch = img->nx / (params.patch_size * 2);
@@ -5390,6 +5543,41 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
                 }
                 set_input_i32("pos_w", pos_data);
             } break;
+        case PROJECTOR_TYPE_MINIMAX_M3_VL:
+            {
+                // 3-axis (T|H|W) NEOX RoPE. Still images use t=0, so the T
+                // band is identity; trailing dims keep the identity defaults.
+                const int n_head    = hparams.n_head;
+                const int d_head    = hparams.n_embd / n_head;
+                const int rope_dims = 2 * (d_head / 2);
+                const int axis_dim  = 2 * ((rope_dims / 3) / 2);
+                const int n_freq    = axis_dim / 2;
+                const int rot_dim   = 3 * axis_dim;
+                const int half      = rot_dim / 2;
+                const int grid_w    = image_size_width / patch_size;
+
+                std::vector<float> inv_freq(n_freq);
+                for (int i = 0; i < n_freq; i++) {
+                    inv_freq[i] = std::pow(hparams.rope_theta, -2.0f * i / (float) axis_dim);
+                }
+
+                std::vector<float> cos_data((size_t) d_head * n_pos, 1.0f);
+                std::vector<float> sin_data((size_t) d_head * n_pos, 0.0f);
+                for (int p = 0; p < n_pos; p++) {
+                    const int pos_axis[3] = { 0, p / grid_w, p % grid_w };
+                    float * cptr = cos_data.data() + (size_t) p * d_head;
+                    float * sptr = sin_data.data() + (size_t) p * d_head;
+                    for (int d = 0; d < half; d++) {
+                        const float ang = pos_axis[d / n_freq] * inv_freq[d % n_freq];
+                        const float c = std::cos(ang);
+                        const float s = std::sin(ang);
+                        cptr[d] = cptr[d + half] = c;
+                        sptr[d] = sptr[d + half] = s;
+                    }
+                }
+                set_input_f32("rope_cos", cos_data);
+                set_input_f32("rope_sin", sin_data);
+            } break;
         case PROJECTOR_TYPE_GLM_EDGE:
         {
             // llava and other models
@@ -5512,6 +5700,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_JANUS_PRO:
             return ctx->model.mm_1_b->ne[0];
+        case PROJECTOR_TYPE_MINIMAX_M3_VL:
+            return ctx->model.mm_3_b->ne[0];
         case PROJECTOR_TYPE_QWEN3VL:
             // main path + deepstack paths
             return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);

examples/mtmd/legacy-models/minimaxm3-convert-image-encoder-to-gguf.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+
+import torch
+from gguf import GGUFEndian, GGUFWriter
+from safetensors.torch import load_file
+
+
+VISION_PREFIX = "vision_tower.vision_model."
+
+
+def load_index(model_dir: Path) -> dict[str, str]:
+    index_path = model_dir / "model.safetensors.index.json"
+    if index_path.exists():
+        with index_path.open("r", encoding="utf-8") as f:
+            return json.load(f)["weight_map"]
+
+    shards = sorted(model_dir.glob("*.safetensors"))
+    if len(shards) == 1:
+        tensors = load_file(str(shards[0]), device="cpu")
+        return {name: shards[0].name for name in tensors}
+
+    raise FileNotFoundError(f"unable to find safetensors index in {model_dir}")
+
+
+def rename_tensor(name: str) -> str | None:
+    if name == "vision_tower.vision_model.embeddings.patch_embedding.weight":
+        return "v.patch_embd.weight"
+    if name in (
+        "vision_tower.vision_model.pre_layrnorm.weight",
+        "vision_tower.vision_model.pre_layrnorm.bias",
+    ):
+        return name.replace("vision_tower.vision_model.pre_layrnorm", "v.pre_ln")
+
+    if name.startswith("multi_modal_projector."):
+        name = name.replace("multi_modal_projector.linear_1", "mm.0")
+        name = name.replace("multi_modal_projector.linear_2", "mm.2")
+        return name
+    if name.startswith("patch_merge_mlp."):
+        name = name.replace("patch_merge_mlp.linear_1", "mm.4")
+        name = name.replace("patch_merge_mlp.linear_2", "mm.6")
+        return name
+
+    if not name.startswith(VISION_PREFIX + "encoder.layers."):
+        return None
+
+    name = name[len(VISION_PREFIX):]
+    name = name.replace("encoder.layers", "blk")
+    name = name.replace("layer_norm1", "ln1")
+    name = name.replace("layer_norm2", "ln2")
+    name = name.replace("self_attn.q_proj", "attn_q")
+    name = name.replace("self_attn.k_proj", "attn_k")
+    name = name.replace("self_attn.v_proj", "attn_v")
+    name = name.replace("self_attn.out_proj", "attn_out")
+    name = name.replace("mlp.fc1", "ffn_up")
+    name = name.replace("mlp.fc2", "ffn_down")
+    return "v." + name
+
+
+def read_json(path: Path) -> dict:
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Convert MiniMax-M3 vision encoder/projector to GGUF")
+    parser.add_argument("-m", "--model-dir", required=True, help="Path to MiniMax-M3 HF model directory")
+    parser.add_argument("-o", "--output", default=None, help="Output GGUF path")
+    parser.add_argument("--use-f32", action="store_true", help="Write tensors as f32 instead of f16")
+    parser.add_argument("--bigendian", action="store_true", help="Write big-endian GGUF")
+    args = parser.parse_args()
+
+    model_dir = Path(args.model_dir)
+    config = read_json(model_dir / "config.json")
+    vision_config = config["vision_config"]
+
+    preprocessor_path = model_dir / "preprocessor_config.json"
+    preprocessor = read_json(preprocessor_path) if preprocessor_path.exists() else {}
+    compression = vision_config.get("img_token_compression_config", {})
+
+    output = Path(args.output) if args.output else model_dir / "mmproj-minimax-m3-vl.gguf"
+    ftype = 0 if args.use_f32 else 1
+
+    writer = GGUFWriter(
+        path=str(output),
+        arch="clip",
+        endianess=GGUFEndian.BIG if args.bigendian else GGUFEndian.LITTLE,
+    )
+    writer.add_bool("clip.has_text_encoder", False)
+    writer.add_bool("clip.has_vision_encoder", True)
+    writer.add_bool("clip.has_audio_encoder", False)
+    writer.add_string("clip.projector_type", "minimax_m3_vl")
+    writer.add_string("general.name", "MiniMax-M3 vision projector")
+    writer.add_uint32("general.file_type", ftype)
+
+    writer.add_uint32("clip.vision.image_size", vision_config["image_size"])
+    writer.add_uint32("clip.vision.patch_size", vision_config["patch_size"])
+    writer.add_uint32("clip.vision.embedding_length", vision_config["hidden_size"])
+    writer.add_uint32("clip.vision.feed_forward_length", vision_config["intermediate_size"])
+    writer.add_uint32("clip.vision.projection_dim", vision_config["projection_dim"])
+    writer.add_uint32("clip.vision.attention.head_count", vision_config["num_attention_heads"])
+    writer.add_uint32("clip.vision.block_count", vision_config["num_hidden_layers"])
+    writer.add_float32("clip.vision.attention.layer_norm_epsilon", vision_config.get("layer_norm_eps", 1e-5))
+    writer.add_uint32("clip.vision.spatial_merge_size", compression.get("spatial_merge_size", 2))
+    writer.add_uint32("clip.vision.temporal_patch_size", compression.get("temporal_patch_size", 2))
+    writer.add_uint32("clip.vision.image_min_pixels", preprocessor.get("min_pixels", 4 * 28 * 28))
+    writer.add_uint32("clip.vision.image_max_pixels", preprocessor.get("max_pixels", 451584))
+    writer.add_array("clip.vision.image_mean", preprocessor.get("image_mean", [0.48145466, 0.4578275, 0.40821073]))
+    writer.add_array("clip.vision.image_std", preprocessor.get("image_std", [0.26862954, 0.26130258, 0.27577711]))
+    writer.add_bool("clip.use_gelu", True)
+
+    weight_map = load_index(model_dir)
+    shard_cache: dict[str, dict[str, torch.Tensor]] = {}
+
+    for src_name in sorted(weight_map):
+        dst_name = rename_tensor(src_name)
+        if dst_name is None:
+            continue
+
+        shard_name = weight_map[src_name]
+        if shard_name not in shard_cache:
+            shard_cache[shard_name] = load_file(str(model_dir / shard_name), device="cpu")
+
+        data = shard_cache[shard_name][src_name]
+        if src_name.endswith("patch_embedding.weight") and data.ndim == 5:
+            if data.shape[2] != 2:
+                raise ValueError(f"expected temporal_patch_size 2, got {data.shape[2]}")
+            for i in range(data.shape[2]):
+                patch_name = dst_name if i == 0 else f"{dst_name}.{i}"
+                patch_data = data[:, :, i]
+                if args.use_f32:
+                    patch_data = patch_data.float()
+                else:
+                    patch_data = patch_data.half()
+                writer.add_tensor(patch_name, patch_data.numpy())
+            continue
+        if args.use_f32:
+            data = data.float()
+        elif data.ndim == 2 and dst_name.endswith(".weight"):
+            data = data.half()
+        else:
+            data = data.float()
+        writer.add_tensor(dst_name, data.numpy())
+
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file()
+    writer.close()
+    print(f"Wrote {output}")
+
+
+if __name__ == "__main__":
+    main()

examples/mtmd/mtmd.cpp

@@ -283,6 +283,11 @@ struct mtmd_context {
             img_beg = "<|vision_start|>";
             img_end = "<|vision_end|>";
 
+        } else if (proj == PROJECTOR_TYPE_MINIMAX_M3_VL) {
+            // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[
+            img_beg = "]<]start of image[>[";
+            img_end = "]<]end of image[>[";
+
         } else if (proj == PROJECTOR_TYPE_LLAMA4) {
             // (more details in mtmd_context constructor)
             img_beg = "<|image_start|>";
@@ -1208,4 +1213,3 @@ void mtmd_input_chunk_to_json(mtmd_input_chunk * chunk, json & j) {
 
 
 
-

ggml/src/ggml-cuda.cu

@@ -3772,6 +3772,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
                 case GGML_UNARY_OP_GELU:
                     ggml_cuda_op_gelu(ctx, dst);
                     break;
+                case GGML_UNARY_OP_GELU_ERF:
+                    ggml_cuda_op_gelu_erf(ctx, dst);
+                    break;
                 case GGML_UNARY_OP_SILU:
                     ggml_cuda_op_silu(ctx, dst);
                     break;
@@ -4692,6 +4695,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons
         case GGML_OP_UNARY:
             switch (ggml_get_unary_op(op)) {
                 case GGML_UNARY_OP_GELU:
+                case GGML_UNARY_OP_GELU_ERF:
                 case GGML_UNARY_OP_SILU:
                 case GGML_UNARY_OP_SWIGLU:
                 case GGML_UNARY_OP_SWIGLU_OAI:

ggml/src/ggml.c

@@ -3317,11 +3317,16 @@ inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float
 static const float GELU_QUICK_COEF = -1.702f;
 static const float GELU_COEF_A     = 0.044715f;
 static const float SQRT_2_OVER_PI  = 0.79788456080286535587989211986876f;
+static const float SQRT_2_INV      = 0.70710678118654752440084436210484f;
 
 inline static float ggml_gelu_f32(float x) {
     return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));
 }
 
+inline static float ggml_gelu_erf_f32(float x) {
+    return 0.5f*x*(1.0f + erff(x*SQRT_2_INV));
+}
+
 inline static float ggml_gelu_quick_f32(float x) {
     return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x)));
 }
@@ -3943,6 +3948,13 @@ inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
     }
 #endif
 }
+
+inline static void ggml_vec_gelu_erf_f32(const int n, float * y, const float * x) {
+    for (int i = 0; i < n; ++i) {
+        y[i] = ggml_gelu_erf_f32(x[i]);
+    }
+}
+
 inline static void ggml_vec_mul_gelu_f32(const int n, float * z, const float * x, const float * y) {
     int i = 0;
 #if defined(__AVX512F__) && defined(__AVX512DQ__)
@@ -4159,8 +4171,6 @@ inline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_f
     }
 }
 
-static const float SQRT_2_INV = 0.70710678118654752440084436210484f;
-
 inline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) {
     for (int i = 0; i < n; ++i) {
         float xi = x[i];
@@ -15609,6 +15619,65 @@ static void ggml_compute_forward_gelu(
     }
 }
 
+// ggml_compute_forward_gelu_erf
+
+static void ggml_compute_forward_gelu_erf_f32(
+        const struct ggml_compute_params * params,
+        struct ggml_tensor * dst) {
+
+    const struct ggml_tensor * src0 = dst->src[0];
+
+    assert(ggml_is_contiguous_1(src0));
+    assert(ggml_is_contiguous_1(dst));
+    assert(ggml_are_same_shape(src0, dst));
+
+    const int ith = params->ith;
+    const int nth = params->nth;
+
+    const int nc = src0->ne[0];
+    const int nr = ggml_nrows(src0);
+
+    // rows per thread
+    const int dr = (nr + nth - 1)/nth;
+
+    // row range for this thread
+    const int ir0 = dr*ith;
+    const int ir1 = MIN(ir0 + dr, nr);
+
+    for (int i1 = ir0; i1 < ir1; i1++) {
+        ggml_vec_gelu_erf_f32(nc,
+                (float *) ((char *) dst->data  + i1*( dst->nb[1])),
+                (float *) ((char *) src0->data + i1*(src0->nb[1])));
+
+#ifndef NDEBUG
+        for (int k = 0; k < nc; k++) {
+            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
+            UNUSED(x);
+            assert(!isnan(x));
+            assert(!isinf(x));
+        }
+#endif
+    }
+}
+
+static void ggml_compute_forward_gelu_erf(
+        const struct ggml_compute_params * params,
+        struct ggml_tensor * dst) {
+
+    const struct ggml_tensor * src0 = dst->src[0];
+
+    switch (src0->type) {
+        case GGML_TYPE_F32:
+            {
+                ggml_compute_forward_gelu_erf_f32(params, dst);
+            } break;
+        default:
+            {
+                GGML_ABORT("fatal error");
+            }
+    }
+}
+
 // ggml_compute_forward_fill
 
 static void ggml_compute_forward_fill_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst) {
@@ -23186,6 +23255,10 @@ static void ggml_compute_forward_unary(
             {
                 ggml_compute_forward_gelu(params, dst);
             } break;
+        case GGML_UNARY_OP_GELU_ERF:
+            {
+                ggml_compute_forward_gelu_erf(params, dst);
+            } break;
         case GGML_UNARY_OP_GELU_QUICK:
             {
                 ggml_compute_forward_gelu_quick(params, dst);
@@ -26505,6 +26578,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
                 case GGML_UNARY_OP_SIGMOID:
                 case GGML_UNARY_OP_NEG:
                 case GGML_UNARY_OP_GELU:
+                case GGML_UNARY_OP_GELU_ERF:
                 case GGML_UNARY_OP_GELU_QUICK:
                 case GGML_UNARY_OP_SILU:
                 case GGML_UNARY_OP_EXP: