LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

mtmd: add n_embd_head (#26342)

Browse Source 
Co-authored-by: Daniel Han <unslothai@gmail.com>
This commit is contained in:
Xuan-Son Nguyen
2026-07-31 15:30:19 +02:00
committed by GitHub
co-authored by Daniel Han
parent a09d8abf8c
commit db7d8b24b5
5 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+1
View File
@@ -353,6 +353,7 @@ class Keys:
class Attention:
HEAD_COUNT = "clip.vision.attention.head_count"
HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA)
HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embd
LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon"
class Projector:
gguf-py/gguf/gguf_writer.py
+3
View File
@@ -1226,6 +1226,9 @@ class GGUFWriter:
def add_vision_head_count_kv(self, value: int) -> None:
self.add_uint32(Keys.ClipVision.Attention.HEAD_COUNT_KV, value)
def add_vision_head_dim(self, value: int) -> None:
self.add_uint32(Keys.ClipVision.Attention.HEAD_DIM, value)
def add_vision_attention_layernorm_eps(self, value: float) -> None:
self.add_float32(Keys.ClipVision.Attention.LAYERNORM_EPS, value)
tools/mtmd/clip-impl.h
+1
View File
@@ -41,6 +41,7 @@
#define KEY_PROJ_DIM "clip.%s.projection_dim"
#define KEY_N_HEAD "clip.%s.attention.head_count"
#define KEY_N_HEAD_KV "clip.%s.attention.head_count_kv"
#define KEY_N_EMBD_HEAD "clip.%s.attention.head_dim"
#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"
#define KEY_FEATURE_LAYERS "clip.%s.feature_layer"
tools/mtmd/clip-model.h
+2
View File
@@ -54,6 +54,8 @@ struct clip_hparams {
int32_t projection_dim = 0;
int32_t n_head = 0;
int32_t n_head_kv = 0;
// 0 = derive from n_embd; set when qkv width != n_embd
int32_t n_embd_head = 0;
int32_t n_layer = 0;
int32_t n_merge = 1; // number of patch merges **per-side**
tools/mtmd/clip.cpp
+4 -3
View File
@@ -253,7 +253,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
n_embd(hparams.n_embd),
n_head(hparams.n_head),
n_head_kv(hparams.n_head_kv),
d_head(n_head > 0 ? n_embd / n_head : 0),
d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)),
n_layer(hparams.n_layer),
n_mmproj_embd(clip_n_mmproj_embd(ctx)),
eps(hparams.eps),
@@ -372,13 +372,13 @@ ggml_tensor * clip_graph::build_vit(
/* nb1 */ ggml_row_size(cur->type, d_head),
/* nb2 */ cur->nb[1],
/* nb3 */ cur->nb[1] * n_pos,
/* offset */ ggml_row_size(cur->type, n_embd));
/* offset */ ggml_row_size(cur->type, n_head * d_head));
Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B,
/* nb1 */ ggml_row_size(cur->type, d_head),
/* nb2 */ cur->nb[1],
/* nb3 */ cur->nb[1] * n_pos,
/* offset */ ggml_row_size(cur->type, 2 * n_embd));
/* offset */ ggml_row_size(cur->type, 2 * n_head * d_head));
if (layer.q_norm) {
GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
@@ -1190,6 +1190,7 @@ struct clip_model_loader {
const char * prefix = is_vision ? "vision" : "audio";
get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);
get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);
get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);
get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);
get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer);
get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);