diff --git a/common/arg.cpp b/common/arg.cpp
index 9753441313..84c19e0632 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -2508,7 +2508,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
add_opt(common_arg(
{"--mlock"},
- "DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",
+ "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing",
[](common_params & params) {
LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
params.load_mode = LLAMA_LOAD_MODE_MLOCK;
@@ -2537,13 +2537,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
"model loading mode (default: mmap)\n"
"- none: no special loading mode\n"
"- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"
- "- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
+ "- mlock: force system to keep model in RAM rather than swapping or compressing\n"
+ "- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
"- dio: use DirectIO if available\n",
[](common_params & params, const std::string & value) {
- /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
- else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
- else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
- else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }
+ /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
+ else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
+ else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
+ else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; }
+ else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
diff --git a/include/llama.h b/include/llama.h
index 9fab693170..3c6d22be89 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -203,10 +203,11 @@ extern "C" {
};
enum llama_load_mode {
- LLAMA_LOAD_MODE_NONE = 0, // no special loading mode
- LLAMA_LOAD_MODE_MMAP = 1, // memory map the model
- LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing
- LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available
+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode
+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model
+ LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing
+ LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing
+ LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available
};
LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 43447f57d3..510586e96c 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -542,7 +542,7 @@ llama_model_loader::llama_model_loader(
tensor_buft_overrides = param_tensor_buft_overrides_p;
- this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MLOCK;
+ this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;
this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;
if (!fname.empty()) {
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 5179692108..074acbe1fa 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1249,7 +1249,7 @@ void llama_model_base::load_vocab(llama_model_loader & ml) {
bool llama_model_base::load_tensors(llama_model_loader & ml) {
const auto & split_mode = params.split_mode;
- const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK;
+ const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK || params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;
const auto & tensor_split = params.tensor_split;
const int n_layer_all = hparams.n_layer_all;
diff --git a/src/llama.cpp b/src/llama.cpp
index 11ac9656d9..d22e4c81a9 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -54,6 +54,8 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) {
return "mmap";
case LLAMA_LOAD_MODE_MLOCK:
return "mlock";
+ case LLAMA_LOAD_MODE_MMAP_MLOCK:
+ return "mmap+mlock";
case LLAMA_LOAD_MODE_DIRECT_IO:
return "dio";
}
@@ -61,10 +63,11 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) {
}
enum llama_load_mode llama_load_mode_from_str(const char * str) {
- if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }
- if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }
- if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }
- if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }
+ if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }
+ if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }
+ if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }
+ if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; }
+ if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }
throw std::invalid_argument(std::string("unknown load mode: ") + str);
}
diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp
index 000ecd9aaa..1d3584f903 100644
--- a/tests/test-arg-parser.cpp
+++ b/tests/test-arg-parser.cpp
@@ -143,6 +143,10 @@ static void test(void) {
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
+ argv = {"binary_name", "-lm", "mmap+mlock"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
+
argv = {"binary_name", "-lm", "dio"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
@@ -187,6 +191,11 @@ static void test(void) {
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
+ setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);
+ argv = {"binary_name"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
+
setenv("LLAMA_ARG_LOAD_MODE", "dio", true);
argv = {"binary_name"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
diff --git a/tools/cli/README.md b/tools/cli/README.md
index 6ee447b073..972ea04dc7 100644
--- a/tools/cli/README.md
+++ b/tools/cli/README.md
@@ -55,10 +55,10 @@
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) |
-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) |
-| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) |
| `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
diff --git a/tools/completion/README.md b/tools/completion/README.md
index 17f7cd7659..bce71d68d9 100644
--- a/tools/completion/README.md
+++ b/tools/completion/README.md
@@ -138,10 +138,10 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) |
-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) |
-| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) |
| `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp
index 29ad352d0c..dc1c7caf4a 100644
--- a/tools/llama-bench/llama-bench.cpp
+++ b/tools/llama-bench/llama-bench.cpp
@@ -429,45 +429,45 @@ static void print_usage(int /* argc */, char ** argv) {
}
printf("\n");
printf("test parameters:\n");
- printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());
- printf(" -hf, -hfr, --hf-repo /[:quant] Hugging Face model repository; quant is optional, case-insensitive\n");
- printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n");
- printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n");
- printf(" (default: unused)\n");
- printf(" -hff, --hf-file Hugging Face model file. If specified, it will override the quant in --hf-repo\n");
- printf(" (default: unused)\n");
- printf(" -hft, --hf-token Hugging Face access token\n");
- printf(" (default: value from HF_TOKEN environment variable)\n");
- printf(" --offline Offline mode: forces use of cache, prevents network access\n");
- printf(" (default: disabled)\n");
- printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());
- printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());
- printf(" -pg (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());
- printf(" -d, --n-depth (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str());
- printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());
- printf(" -ub, --ubatch-size (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());
- printf(" -ctk, --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());
- printf(" -ctv, --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());
- printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());
- printf(" -C, --cpu-mask (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());
- printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());
- printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());
- printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());
- printf(" -ncmoe, --n-cpu-moe (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());
- printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
- printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());
- printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
- printf(" -fa, --flash-attn (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
- printf(" -dev, --device (default: auto)\n");
- printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
- printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
- printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
- printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
- printf(" -ts, --tensor-split (default: 0)\n");
+ printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());
+ printf(" -hf, -hfr, --hf-repo /[:quant] Hugging Face model repository; quant is optional, case-insensitive\n");
+ printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n");
+ printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n");
+ printf(" (default: unused)\n");
+ printf(" -hff, --hf-file Hugging Face model file. If specified, it will override the quant in --hf-repo\n");
+ printf(" (default: unused)\n");
+ printf(" -hft, --hf-token Hugging Face access token\n");
+ printf(" (default: value from HF_TOKEN environment variable)\n");
+ printf(" --offline Offline mode: forces use of cache, prevents network access\n");
+ printf(" (default: disabled)\n");
+ printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());
+ printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());
+ printf(" -pg (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());
+ printf(" -d, --n-depth (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str());
+ printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());
+ printf(" -ub, --ubatch-size (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());
+ printf(" -ctk, --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());
+ printf(" -ctv, --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());
+ printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());
+ printf(" -C, --cpu-mask (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());
+ printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());
+ printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());
+ printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());
+ printf(" -ncmoe, --n-cpu-moe (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());
+ printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());
+ printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());
+ printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
+ printf(" -fa, --flash-attn (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
+ printf(" -dev, --device (default: auto)\n");
+ printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
+ printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
+ printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
+ printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
+ printf(" -ts, --tensor-split (default: 0)\n");
printf(" -ot --override-tensor =;...\n");
- printf(" (default: disabled)\n");
- printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");
- printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());
+ printf(" (default: disabled)\n");
+ printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");
+ printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());
printf("\n");
printf(
"Multiple values can be given for each parameter by separating them with ','\n"
@@ -785,6 +785,8 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
mode = LLAMA_LOAD_MODE_MMAP;
} else if (m == "mlock") {
mode = LLAMA_LOAD_MODE_MLOCK;
+ } else if (m == "mmap+mlock") {
+ mode = LLAMA_LOAD_MODE_MMAP_MLOCK;
} else if (m == "dio") {
mode = LLAMA_LOAD_MODE_DIRECT_IO;
} else {
diff --git a/tools/server/README.md b/tools/server/README.md
index d345654554..25aacf9f51 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -72,10 +72,10 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) |
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) |
| `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) |
-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) |
-| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
+| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
(env: LLAMA_ARG_LOAD_MODE) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) |
| `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |