LLM-Inference/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-08-12 22:29:39 +04:00
Code Issues Packages Projects Releases Wiki Activity

Add --prefetch-experts to stream mmap'd MoE experts into page cache (#2101)

Browse Source 
* Add --prefetch-experts to stream mmap'd MoE experts into page cache

* Drop fds, fault experts in with MADV_POPULATE_READ instead of pread

* Remove stale note about pread workers

* Move MoE prefetch behind ggml_backend_prefetch_* wrappers

* Cleanup stale comments

* Add --prefetch-experts-threads, drop GGML_MOE_PREFETCH_THREADS env var
This commit is contained in:
dmaivel
2026-07-11 10:43:42 +03:00
committed by GitHub
parent 70ea15226f
commit 6a909f4ff6
13 changed files with 709 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+16
View File
@@ -2141,6 +2141,15 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
params.warmup = false;
return true;
}
if (arg == "--prefetch-experts") {
params.prefetch_experts = true;
return true;
}
if (arg == "--prefetch-experts-threads") {
CHECK_ARG;
params.prefetch_experts_threads = std::stoi(argv[i]);
return true;
}
if (arg == "--fit-margin") {
CHECK_ARG;
int32_t margin = std::stoi(argv[i]);
@@ -3247,6 +3256,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
options.push_back({ "*", " --cpu-moe", "keep all MoE weights in CPU memory"});
options.push_back({ "*", " --n-cpu-moe N", "keep MoE weights of the first N layers in CPU memory"});
options.push_back({ "*", " --defer-experts", "defer expert mmap residency on Linux to reduce model load time"});
options.push_back({ "*", " --prefetch-experts", "stream mmap'd MoE expert weights into the page cache on Linux"});
options.push_back({ "*", " --prefetch-experts-threads N",
"number of expert prefetch workers, tune to drive speed/type (default: auto)"});
options.push_back({ "*", " --fit-margin N", "safety margin in MiB when auto-fitting model offloading"});
options.push_back({ "*", "-wgt, --worst-graph-tokens N", "number of tokens to use for worst-case graph"});
options.push_back({ "*", " --fit", "automatically determine which tensors to offload to the GPU(s)"});
@@ -4289,6 +4301,8 @@ struct llama_context_params common_context_params_to_llama(const gpt_params & pa
cparams.min_experts = params.min_experts;
cparams.thresh_experts = params.thresh_experts;
cparams.only_active_experts = params.only_active_exps;
cparams.prefetch_experts = params.prefetch_experts;
cparams.prefetch_experts_threads = params.prefetch_experts_threads;
cparams.max_extra_alloc = params.max_extra_alloc_MiB;
cparams.mtp = params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP);
cparams.mtp_op_type = MTP_OP_NONE;
@@ -5286,6 +5300,8 @@ void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const l
fprintf(stream, "merge_qkv: %s # default: false\n", params.merge_qkv ? "true" : "false");
fprintf(stream, "merge_up_gate_exps: %s # default: false\n", params.merge_up_gate_exps ? "true" : "false");
fprintf(stream, "defer_experts: %s # default: false\n", params.defer_experts ? "true" : "false");
fprintf(stream, "prefetch_experts: %s # default: false\n", params.prefetch_experts ? "true" : "false");
fprintf(stream, "prefetch_experts_threads: %d # default: 0 (auto)\n", params.prefetch_experts_threads);
fprintf(stream, "max_extra_alloc: %d # default: 256\n", params.max_extra_alloc_MiB);
fprintf(stream, "penalize_nl: %s # default: false\n", sparams.penalize_nl ? "true" : "false");
fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type);