LLM-Inference/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-08-12 22:29:39 +04:00
Code Issues Packages Projects Releases Wiki Activity

Do not quantize integer tensors

Browse Source
This commit is contained in:
Kawrakow
2026-08-03 07:36:27 +00:00
parent 87eeec9f74
commit 79176e3b32
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-quantize.cpp
+6 -1
View File
@@ -1452,8 +1452,13 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
llama_format_tensor_shape(tensor).c_str(),
ggml_type_name(tensor->type));
bool quantize = tensor->type != GGML_TYPE_I32 &&
tensor->type != GGML_TYPE_I64 &&
tensor->type != GGML_TYPE_I16 &&
tensor->type != GGML_TYPE_I8; // i.e., do not quantize tensors holding int values
// This used to be a regex, but <regex> has an extreme cost to compile times.
bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?
quantize &= name.rfind("weight") == name.size() - 6; // ends with 'weight'?
// quantize only 2D and 3D tensors (experts)
quantize &= (ggml_n_dims(tensor) >= 2);