diff --git a/examples/mtmd/clip-impl.h b/examples/mtmd/clip-impl.h index 1253b93d9..b149d533f 100644 --- a/examples/mtmd/clip-impl.h +++ b/examples/mtmd/clip-impl.h @@ -40,6 +40,7 @@ #define KEY_FEATURE_LAYER "clip.vision.feature_layer" #define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor" #define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size" +#define KEY_TEMPORAL_PATCH_SIZE "clip.vision.temporal_patch_size" #define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers" #define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type" @@ -167,6 +168,7 @@ enum projector_type { PROJECTOR_TYPE_LIGHTONOCR, PROJECTOR_TYPE_COGVLM, PROJECTOR_TYPE_JANUS_PRO, + PROJECTOR_TYPE_MINIMAX_M3_VL, PROJECTOR_TYPE_UNKNOWN, }; @@ -197,6 +199,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"}, { PROJECTOR_TYPE_COGVLM, "cogvlm"}, { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"}, + { PROJECTOR_TYPE_MINIMAX_M3_VL, "minimax_m3_vl"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/examples/mtmd/clip.cpp b/examples/mtmd/clip.cpp index 02fc2a89e..a17a0a2d4 100644 --- a/examples/mtmd/clip.cpp +++ b/examples/mtmd/clip.cpp @@ -192,6 +192,7 @@ struct clip_hparams { int32_t image_min_pixels = -1; int32_t image_max_pixels = -1; int32_t n_merge = 0; // number of patch merges **per-side** + int32_t temporal_patch_size = 1; float image_mean[3]; float image_std[3]; @@ -758,6 +759,101 @@ struct clip_graph { return gf; } + ggml_cgraph * build_minimax_m3_vl() { + GGML_ASSERT(model.patch_bias == nullptr); + GGML_ASSERT(model.class_embedding == nullptr); + GGML_ASSERT(model.patch_embeddings_0 != nullptr); + GGML_ASSERT(model.patch_embeddings_1 != nullptr); + GGML_ASSERT(hparams.n_merge == 2); + GGML_ASSERT(img.nx % (patch_size * hparams.n_merge) == 0); + GGML_ASSERT(img.ny % (patch_size * hparams.n_merge) == 0); + + const int batch_size = 1; + + // MiniMax-M3 uses 3-axis NEOX RoPE. Each axis gets an even slice of the + // head dim; any remainder is left unrotated. + const int rope_dims = 2 * (d_head / 2); + const int axis_dim = 2 * ((rope_dims / 3) / 2); + const int rot_dim = 3 * axis_dim; + + ggml_tensor * rope_cos = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, d_head, 1, n_patches); + ggml_set_name(rope_cos, "rope_cos"); + ggml_set_input(rope_cos); + + ggml_tensor * rope_sin = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, d_head, 1, n_patches); + ggml_set_name(rope_sin, "rope_sin"); + ggml_set_input(rope_sin); + + ggml_tensor * inp_raw = build_inp_raw(); + + // The generic convolution path converts im2col output to f16. MiniMax-M3's + // patch embedding is sensitive to that loss, so keep the accumulation in f32. + auto conv_2d_f32 = [&](ggml_tensor * kernel) { + ggml_tensor * kernel_f32 = ggml_cast(ctx0, kernel, GGML_TYPE_F32); + ggml_tensor * col = ggml_im2col( + ctx0, kernel_f32, inp_raw, + patch_size, patch_size, 0, 0, 1, 1, + true, GGML_TYPE_F32); + ggml_tensor * cur = ggml_mul_mat( + ctx0, + ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[3] * col->ne[2] * col->ne[1]), + ggml_reshape_2d(ctx0, kernel_f32, kernel_f32->ne[0] * kernel_f32->ne[1] * kernel_f32->ne[2], kernel_f32->ne[3])); + cur = ggml_reshape_4d(ctx0, cur, col->ne[1], col->ne[2], col->ne[3], kernel->ne[3]); + return ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 1, 3, 2)); + }; + + ggml_tensor * inp = conv_2d_f32(model.patch_embeddings_0); + inp = ggml_add(ctx0, inp, conv_2d_f32(model.patch_embeddings_1)); + inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); + inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches, batch_size); + cb(inp, "patch_embd", -1); + + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return build_rope_vision_neox(ctx0, cur, rope_cos, rope_sin, rot_dim); + }; + + ggml_tensor * cur = build_vit( + inp, + n_patches, + NORM_TYPE_NORMAL, + hparams.ffn_op, + nullptr, + add_pos); + + cur = ggml_mul_mat(ctx0, model.mm_0_w, cur); + cur = ggml_add(ctx0, cur, model.mm_0_b); + cur = ggml_gelu_erf(ctx0, cur); + + cur = ggml_mul_mat(ctx0, model.mm_1_w, cur); + cur = ggml_add(ctx0, cur, model.mm_1_b); + + // The reference processor groups patches by 2x2 merge blocks. We keep the + // ViT tokens in raster order, then reorder here before concatenating each + // merge block for patch_merge_mlp. + cur = ggml_reshape_4d(ctx0, cur, + hparams.projection_dim * hparams.n_merge, + n_patches_x / hparams.n_merge, + hparams.n_merge, + batch_size * (n_patches_y / hparams.n_merge)); + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + cur = ggml_cont(ctx0, cur); + + cur = ggml_reshape_3d(ctx0, cur, hparams.projection_dim * hparams.n_merge * hparams.n_merge, + n_patches / (hparams.n_merge * hparams.n_merge), batch_size); + + cur = ggml_mul_mat(ctx0, model.mm_2_w, cur); + cur = ggml_add(ctx0, cur, model.mm_2_b); + cur = ggml_gelu_erf(ctx0, cur); + + cur = ggml_mul_mat(ctx0, model.mm_3_w, cur); + cur = ggml_add(ctx0, cur, model.mm_3_b); + cur = ggml_reshape_3d(ctx0, cur, hparams.projection_dim, n_patches / (hparams.n_merge * hparams.n_merge), batch_size); + + ggml_build_forward_expand(gf, cur); + + return gf; + } + // Qwen2VL and Qwen2.5VL use M-RoPE ggml_cgraph * build_qwen2vl() { GGML_ASSERT(model.patch_bias == nullptr); @@ -2788,6 +2884,29 @@ private: return cur; } + // MiniMax-M3 vision RoPE uses rotate_half semantics over host-computed + // cos/sin tables. cur is [d_head, n_head, n_pos]; cos/sin broadcast over heads. + static ggml_tensor * build_rope_vision_neox( + ggml_context * ctx0, ggml_tensor * cur, + ggml_tensor * cos, ggml_tensor * sin, int rot_dim) { + const int64_t d_head = cur->ne[0]; + const int64_t n_head = cur->ne[1]; + const int64_t n_pos = cur->ne[2]; + const int64_t half = rot_dim / 2; + const size_t es = ggml_element_size(cur); + + ggml_tensor * first = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, half, n_head, n_pos, cur->nb[1], cur->nb[2], 0)); + ggml_tensor * second = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, half, n_head, n_pos, cur->nb[1], cur->nb[2], half * es)); + ggml_tensor * rotated = ggml_concat(ctx0, ggml_neg(ctx0, second), first, 0); + + if (rot_dim < d_head) { + ggml_tensor * tail = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, d_head - rot_dim, n_head, n_pos, cur->nb[1], cur->nb[2], rot_dim * es)); + rotated = ggml_concat(ctx0, rotated, tail, 0); + } + + return ggml_add(ctx0, ggml_mul(ctx0, cur, cos), ggml_mul(ctx0, rotated, sin)); + } + // aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL) // support dynamic resolution ggml_tensor * build_patch_merge_permute(ggml_tensor * cur, int scale_factor) { @@ -2878,6 +2997,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { res = graph.build_qwen3vl(); } break; + case PROJECTOR_TYPE_MINIMAX_M3_VL: + { + res = graph.build_minimax_m3_vl(); + } break; case PROJECTOR_TYPE_GEMMA4V: { res = graph.build_gemma4(); @@ -3242,6 +3365,24 @@ struct clip_model_loader { LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; + case PROJECTOR_TYPE_MINIMAX_M3_VL: + { + hparams.rope_theta = 10000.0f; + hparams.n_merge = 2; + hparams.temporal_patch_size = 2; + hparams.ffn_op = FFN_GELU_ERF; + log_ffn_op = "gelu_erf"; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + get_u32(KEY_TEMPORAL_PATCH_SIZE, hparams.temporal_patch_size, false); + get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels, false); + if (hparams.image_min_pixels <= 0 || hparams.image_max_pixels <= 0) { + hparams.set_limit_image_tokens(8, 576); + } else { + hparams.warmup_image_size = static_cast(std::sqrt(hparams.image_max_pixels)); + } + hparams.set_warmup_n_tokens(256); + } break; case PROJECTOR_TYPE_GEMMA4V: { hparams.rope_theta = 100.0f; @@ -3296,6 +3437,7 @@ struct clip_model_loader { LOG_INF("%s: has_llava_proj: %d\n", __func__, hparams.has_llava_projector); LOG_INF("%s: minicpmv_version: %d\n", __func__, hparams.minicpmv_version); LOG_INF("%s: n_merge: %d\n", __func__, hparams.n_merge); + LOG_INF("%s: temporal_patch_size:%d\n", __func__, hparams.temporal_patch_size); LOG_INF("%s: n_wa_pattern: %d\n", __func__, hparams.n_wa_pattern); if (hparams.image_min_pixels > 0) { LOG_INF("%s: image_min_pixels: %d%s\n", __func__, hparams.image_min_pixels, hparams.custom_image_min_tokens > 0 ? " (custom value)" : ""); @@ -3577,6 +3719,17 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break; + case PROJECTOR_TYPE_MINIMAX_M3_VL: + { + model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); + model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias")); + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 4, "weight")); + model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 4, "bias")); + model.mm_3_w = get_tensor(string_format(TN_LLAVA_PROJ, 6, "weight")); + model.mm_3_b = get_tensor(string_format(TN_LLAVA_PROJ, 6, "bias")); + } break; case PROJECTOR_TYPE_GEMMA3: { model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); @@ -4590,8 +4743,6 @@ private: } }; -// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector -// res_imgs memory is being allocated here, previous allocations will be freed if found bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) { clip_image_size original_size{img->nx, img->ny}; auto & params = ctx->model.hparams; @@ -4617,6 +4768,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_GEMMA4V: + case PROJECTOR_TYPE_MINIMAX_M3_VL: { GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0); clip_image_u8 resized; @@ -4876,7 +5028,7 @@ const char * clip_patch_merge_type(const struct clip_ctx * ctx) { int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img) { const auto & params = ctx->model.hparams; const int n_total = clip_n_output_tokens(ctx, img); - if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL) { + if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL || ctx->proj_type() == PROJECTOR_TYPE_MINIMAX_M3_VL) { return img->nx / (params.patch_size * 2); } return n_total; @@ -4884,7 +5036,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img) { const auto & params = ctx->model.hparams; - if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL) { + if (ctx->proj_type() == PROJECTOR_TYPE_QWEN2VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL || ctx->proj_type() == PROJECTOR_TYPE_QWEN3VL || ctx->proj_type() == PROJECTOR_TYPE_MINIMAX_M3_VL) { return img->ny / (params.patch_size * 2); } return 1; @@ -4942,6 +5094,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_MINIMAX_M3_VL: { // dynamic size (2 conv, so double patch size) int x_patch = img->nx / (params.patch_size * 2); @@ -5390,6 +5543,41 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima } set_input_i32("pos_w", pos_data); } break; + case PROJECTOR_TYPE_MINIMAX_M3_VL: + { + // 3-axis (T|H|W) NEOX RoPE. Still images use t=0, so the T + // band is identity; trailing dims keep the identity defaults. + const int n_head = hparams.n_head; + const int d_head = hparams.n_embd / n_head; + const int rope_dims = 2 * (d_head / 2); + const int axis_dim = 2 * ((rope_dims / 3) / 2); + const int n_freq = axis_dim / 2; + const int rot_dim = 3 * axis_dim; + const int half = rot_dim / 2; + const int grid_w = image_size_width / patch_size; + + std::vector inv_freq(n_freq); + for (int i = 0; i < n_freq; i++) { + inv_freq[i] = std::pow(hparams.rope_theta, -2.0f * i / (float) axis_dim); + } + + std::vector cos_data((size_t) d_head * n_pos, 1.0f); + std::vector sin_data((size_t) d_head * n_pos, 0.0f); + for (int p = 0; p < n_pos; p++) { + const int pos_axis[3] = { 0, p / grid_w, p % grid_w }; + float * cptr = cos_data.data() + (size_t) p * d_head; + float * sptr = sin_data.data() + (size_t) p * d_head; + for (int d = 0; d < half; d++) { + const float ang = pos_axis[d / n_freq] * inv_freq[d % n_freq]; + const float c = std::cos(ang); + const float s = std::sin(ang); + cptr[d] = cptr[d + half] = c; + sptr[d] = sptr[d + half] = s; + } + } + set_input_f32("rope_cos", cos_data); + set_input_f32("rope_sin", sin_data); + } break; case PROJECTOR_TYPE_GLM_EDGE: { // llava and other models @@ -5512,6 +5700,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_JANUS_PRO: return ctx->model.mm_1_b->ne[0]; + case PROJECTOR_TYPE_MINIMAX_M3_VL: + return ctx->model.mm_3_b->ne[0]; case PROJECTOR_TYPE_QWEN3VL: // main path + deepstack paths return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); diff --git a/examples/mtmd/legacy-models/minimaxm3-convert-image-encoder-to-gguf.py b/examples/mtmd/legacy-models/minimaxm3-convert-image-encoder-to-gguf.py new file mode 100644 index 000000000..2efba06a8 --- /dev/null +++ b/examples/mtmd/legacy-models/minimaxm3-convert-image-encoder-to-gguf.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +from pathlib import Path + +import torch +from gguf import GGUFEndian, GGUFWriter +from safetensors.torch import load_file + + +VISION_PREFIX = "vision_tower.vision_model." + + +def load_index(model_dir: Path) -> dict[str, str]: + index_path = model_dir / "model.safetensors.index.json" + if index_path.exists(): + with index_path.open("r", encoding="utf-8") as f: + return json.load(f)["weight_map"] + + shards = sorted(model_dir.glob("*.safetensors")) + if len(shards) == 1: + tensors = load_file(str(shards[0]), device="cpu") + return {name: shards[0].name for name in tensors} + + raise FileNotFoundError(f"unable to find safetensors index in {model_dir}") + + +def rename_tensor(name: str) -> str | None: + if name == "vision_tower.vision_model.embeddings.patch_embedding.weight": + return "v.patch_embd.weight" + if name in ( + "vision_tower.vision_model.pre_layrnorm.weight", + "vision_tower.vision_model.pre_layrnorm.bias", + ): + return name.replace("vision_tower.vision_model.pre_layrnorm", "v.pre_ln") + + if name.startswith("multi_modal_projector."): + name = name.replace("multi_modal_projector.linear_1", "mm.0") + name = name.replace("multi_modal_projector.linear_2", "mm.2") + return name + if name.startswith("patch_merge_mlp."): + name = name.replace("patch_merge_mlp.linear_1", "mm.4") + name = name.replace("patch_merge_mlp.linear_2", "mm.6") + return name + + if not name.startswith(VISION_PREFIX + "encoder.layers."): + return None + + name = name[len(VISION_PREFIX):] + name = name.replace("encoder.layers", "blk") + name = name.replace("layer_norm1", "ln1") + name = name.replace("layer_norm2", "ln2") + name = name.replace("self_attn.q_proj", "attn_q") + name = name.replace("self_attn.k_proj", "attn_k") + name = name.replace("self_attn.v_proj", "attn_v") + name = name.replace("self_attn.out_proj", "attn_out") + name = name.replace("mlp.fc1", "ffn_up") + name = name.replace("mlp.fc2", "ffn_down") + return "v." + name + + +def read_json(path: Path) -> dict: + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Convert MiniMax-M3 vision encoder/projector to GGUF") + parser.add_argument("-m", "--model-dir", required=True, help="Path to MiniMax-M3 HF model directory") + parser.add_argument("-o", "--output", default=None, help="Output GGUF path") + parser.add_argument("--use-f32", action="store_true", help="Write tensors as f32 instead of f16") + parser.add_argument("--bigendian", action="store_true", help="Write big-endian GGUF") + args = parser.parse_args() + + model_dir = Path(args.model_dir) + config = read_json(model_dir / "config.json") + vision_config = config["vision_config"] + + preprocessor_path = model_dir / "preprocessor_config.json" + preprocessor = read_json(preprocessor_path) if preprocessor_path.exists() else {} + compression = vision_config.get("img_token_compression_config", {}) + + output = Path(args.output) if args.output else model_dir / "mmproj-minimax-m3-vl.gguf" + ftype = 0 if args.use_f32 else 1 + + writer = GGUFWriter( + path=str(output), + arch="clip", + endianess=GGUFEndian.BIG if args.bigendian else GGUFEndian.LITTLE, + ) + writer.add_bool("clip.has_text_encoder", False) + writer.add_bool("clip.has_vision_encoder", True) + writer.add_bool("clip.has_audio_encoder", False) + writer.add_string("clip.projector_type", "minimax_m3_vl") + writer.add_string("general.name", "MiniMax-M3 vision projector") + writer.add_uint32("general.file_type", ftype) + + writer.add_uint32("clip.vision.image_size", vision_config["image_size"]) + writer.add_uint32("clip.vision.patch_size", vision_config["patch_size"]) + writer.add_uint32("clip.vision.embedding_length", vision_config["hidden_size"]) + writer.add_uint32("clip.vision.feed_forward_length", vision_config["intermediate_size"]) + writer.add_uint32("clip.vision.projection_dim", vision_config["projection_dim"]) + writer.add_uint32("clip.vision.attention.head_count", vision_config["num_attention_heads"]) + writer.add_uint32("clip.vision.block_count", vision_config["num_hidden_layers"]) + writer.add_float32("clip.vision.attention.layer_norm_epsilon", vision_config.get("layer_norm_eps", 1e-5)) + writer.add_uint32("clip.vision.spatial_merge_size", compression.get("spatial_merge_size", 2)) + writer.add_uint32("clip.vision.temporal_patch_size", compression.get("temporal_patch_size", 2)) + writer.add_uint32("clip.vision.image_min_pixels", preprocessor.get("min_pixels", 4 * 28 * 28)) + writer.add_uint32("clip.vision.image_max_pixels", preprocessor.get("max_pixels", 451584)) + writer.add_array("clip.vision.image_mean", preprocessor.get("image_mean", [0.48145466, 0.4578275, 0.40821073])) + writer.add_array("clip.vision.image_std", preprocessor.get("image_std", [0.26862954, 0.26130258, 0.27577711])) + writer.add_bool("clip.use_gelu", True) + + weight_map = load_index(model_dir) + shard_cache: dict[str, dict[str, torch.Tensor]] = {} + + for src_name in sorted(weight_map): + dst_name = rename_tensor(src_name) + if dst_name is None: + continue + + shard_name = weight_map[src_name] + if shard_name not in shard_cache: + shard_cache[shard_name] = load_file(str(model_dir / shard_name), device="cpu") + + data = shard_cache[shard_name][src_name] + if src_name.endswith("patch_embedding.weight") and data.ndim == 5: + if data.shape[2] != 2: + raise ValueError(f"expected temporal_patch_size 2, got {data.shape[2]}") + for i in range(data.shape[2]): + patch_name = dst_name if i == 0 else f"{dst_name}.{i}" + patch_data = data[:, :, i] + if args.use_f32: + patch_data = patch_data.float() + else: + patch_data = patch_data.half() + writer.add_tensor(patch_name, patch_data.numpy()) + continue + if args.use_f32: + data = data.float() + elif data.ndim == 2 and dst_name.endswith(".weight"): + data = data.half() + else: + data = data.float() + writer.add_tensor(dst_name, data.numpy()) + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + print(f"Wrote {output}") + + +if __name__ == "__main__": + main() diff --git a/examples/mtmd/mtmd.cpp b/examples/mtmd/mtmd.cpp index 04cf77f69..bb95c39ec 100644 --- a/examples/mtmd/mtmd.cpp +++ b/examples/mtmd/mtmd.cpp @@ -283,6 +283,11 @@ struct mtmd_context { img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; + } else if (proj == PROJECTOR_TYPE_MINIMAX_M3_VL) { + // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[ + img_beg = "]<]start of image[>["; + img_end = "]<]end of image[>["; + } else if (proj == PROJECTOR_TYPE_LLAMA4) { // (more details in mtmd_context constructor) img_beg = "<|image_start|>"; @@ -1208,4 +1213,3 @@ void mtmd_input_chunk_to_json(mtmd_input_chunk * chunk, json & j) { - diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu index e006e661c..dc2c86b17 100644 --- a/ggml/src/ggml-cuda.cu +++ b/ggml/src/ggml-cuda.cu @@ -3772,6 +3772,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_UNARY_OP_GELU: ggml_cuda_op_gelu(ctx, dst); break; + case GGML_UNARY_OP_GELU_ERF: + ggml_cuda_op_gelu_erf(ctx, dst); + break; case GGML_UNARY_OP_SILU: ggml_cuda_op_silu(ctx, dst); break; @@ -4692,6 +4695,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_ERF: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_SWIGLU: case GGML_UNARY_OP_SWIGLU_OAI: diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index c105ea37f..79f89b575 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -3317,11 +3317,16 @@ inline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float static const float GELU_QUICK_COEF = -1.702f; static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; +static const float SQRT_2_INV = 0.70710678118654752440084436210484f; inline static float ggml_gelu_f32(float x) { return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x))); } +inline static float ggml_gelu_erf_f32(float x) { + return 0.5f*x*(1.0f + erff(x*SQRT_2_INV)); +} + inline static float ggml_gelu_quick_f32(float x) { return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x))); } @@ -3943,6 +3948,13 @@ inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) { } #endif } + +inline static void ggml_vec_gelu_erf_f32(const int n, float * y, const float * x) { + for (int i = 0; i < n; ++i) { + y[i] = ggml_gelu_erf_f32(x[i]); + } +} + inline static void ggml_vec_mul_gelu_f32(const int n, float * z, const float * x, const float * y) { int i = 0; #if defined(__AVX512F__) && defined(__AVX512DQ__) @@ -4159,8 +4171,6 @@ inline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_f } } -static const float SQRT_2_INV = 0.70710678118654752440084436210484f; - inline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) { for (int i = 0; i < n; ++i) { float xi = x[i]; @@ -15609,6 +15619,65 @@ static void ggml_compute_forward_gelu( } } +// ggml_compute_forward_gelu_erf + +static void ggml_compute_forward_gelu_erf_f32( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + assert(ggml_is_contiguous_1(src0)); + assert(ggml_is_contiguous_1(dst)); + assert(ggml_are_same_shape(src0, dst)); + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src0->ne[0]; + const int nr = ggml_nrows(src0); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_vec_gelu_erf_f32(nc, + (float *) ((char *) dst->data + i1*( dst->nb[1])), + (float *) ((char *) src0->data + i1*(src0->nb[1]))); + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k]; + UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif + } +} + +static void ggml_compute_forward_gelu_erf( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_gelu_erf_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_fill static void ggml_compute_forward_fill_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst) { @@ -23186,6 +23255,10 @@ static void ggml_compute_forward_unary( { ggml_compute_forward_gelu(params, dst); } break; + case GGML_UNARY_OP_GELU_ERF: + { + ggml_compute_forward_gelu_erf(params, dst); + } break; case GGML_UNARY_OP_GELU_QUICK: { ggml_compute_forward_gelu_quick(params, dst); @@ -26505,6 +26578,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_UNARY_OP_SIGMOID: case GGML_UNARY_OP_NEG: case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_ERF: case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_EXP: