From 0377426cef5d211b397f72b1b37adbcd480fbb65 Mon Sep 17 00:00:00 2001 From: Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:32:01 -0700 Subject: [PATCH] model-saver : fix expert shared/chunk FFN length key clobber (#26693) The saver called add_kv with LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH twice, the second time passing n_ff_chexp. gguf_set_val_u32 removes-then-appends, so the second call clobbers the first: the saved shared_feed_forward_length ends up as n_ff_chexp (0 for every arch except GroveMoE), and expert_chunk_feed_forward_length is never written at all. So a save->load roundtrip of any MoE model with a shared expert loses n_ff_shexp. On reload the arch falls back to n_ff for the shexp tensor shape, that no longer matches the saved tensor, and the model FAILS to load. Hits qwen2moe, qwen3-next, granite-moe, hunyuan-moe, ernie4.5, bailingmoe2, nemotron-h, and the other shared-expert MoEs. Fix: the second call writes LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH. test-llama-archs: set expert_shared_feed_forward_length to a value distinct from n_ff in the MoE setup so the roundtrip exercises it. Without the fix the reload fails on a shexp tensor-shape mismatch; with it, every arch roundtrips clean. --- src/llama-model-saver.cpp | 2 +- tests/test-llama-archs.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 3812c594e7..248d1ed387 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -213,7 +213,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true); add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); - add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp); + add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp); add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp); add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp); add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0e29d221ba..6ff0d0ac1d 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -217,6 +217,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (moe) { ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff); + ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1));