diff --git a/common/arg.cpp b/common/arg.cpp index 84c19e0632..3bc9574d81 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -539,6 +539,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }; + // an explicit draft file selection (e.g. -md with -hfd) disables the sidecar resolution of the draft repo + if (!params.speculative.draft.mparams.hf_file.empty()) { + plan_spec.mtp = {}; + plan_spec.dflash = {}; + plan_spec.eagle3 = {}; + } + // infer the speculative type from the sidecar shipped by the draft repo when none is requested if (spec_types_is_default(params)) { if (!plan_spec.mtp.local_path.empty()) { @@ -588,6 +595,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params }); } + // a wired draft sidecar counts as an explicit draft for the main plan fallback below + if (spec_sidecar_found) { + had_spec_url = true; + } + // handle plan_spec (e.g. --spec-draft-hf) if (!plan_spec.model_files.empty() && !had_spec_url && !spec_sidecar_found) { add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); diff --git a/common/download.cpp b/common/download.cpp index e8e938426f..3776c6c7eb 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -568,16 +568,30 @@ static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, } // pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"), -// preferring deeper shared directory prefix with the model, then closest quantization +// preferring deeper shared directory prefix with the model, then exact `tag` match, +// then closest quantization to the tag when given, or to the model otherwise static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, const std::string & model, - const std::string & keyword) { + const std::string & keyword, + const std::string & tag = "") { hf_cache::hf_file best; size_t best_depth = 0; int best_diff = 0; + bool best_exact = false; bool found = false; - auto model_bits = extract_quant_bits(model); + std::string tag_upper = tag; + for (char & c : tag_upper) { + c = (char) std::toupper((unsigned char) c); + } + + int model_bits = 0; + if (!tag_upper.empty()) { + auto pos = tag_upper.find_first_of("0123456789"); + model_bits = pos == std::string::npos ? 0 : std::stoi(tag_upper.substr(pos)); + } else { + model_bits = extract_quant_bits(model); + } auto model_parts = string_split(model, '/'); auto model_dir = model_parts.end() - 1; @@ -600,10 +614,19 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, auto bits = extract_quant_bits(f.path); auto diff = std::abs(bits - model_bits); - if (!found || depth > best_depth || (depth == best_depth && diff < best_diff)) { + std::string path_upper = f.path; + for (char & c : path_upper) { + c = (char) std::toupper((unsigned char) c); + } + bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos; + + if (!found || depth > best_depth || + (depth == best_depth && exact && !best_exact) || + (depth == best_depth && exact == best_exact && diff < best_diff)) { best = f; best_depth = depth; best_diff = diff; + best_exact = exact; found = true; } } @@ -616,18 +639,21 @@ static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, } static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "mtp-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "mtp-", tag); } static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "eagle3-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "eagle3-", tag); } static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "dflash-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "dflash-", tag); } static bool gguf_filename_is_model(const std::string & filepath) { @@ -736,27 +762,36 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } } else { primary = find_best_model(all, tag); - if (primary.path.empty()) { + // a requested sidecar can resolve on its own, without a full model of the same tag + if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; } } - plan.primary = primary; - plan.model_files = get_split_files(all, primary); + if (!primary.path.empty()) { + plan.primary = primary; + plan.model_files = get_split_files(all, primary); + } - if (opts.download_mmproj) { + if (opts.download_mmproj && !primary.path.empty()) { plan.mmproj = find_best_mmproj(all, primary.path); } if (opts.download_mtp) { - plan.mtp = find_best_mtp(all, primary.path); + plan.mtp = find_best_mtp(all, primary.path, tag); } if (opts.download_dflash) { - plan.dflash = find_best_dflash(all, primary.path); + plan.dflash = find_best_dflash(all, primary.path, tag); } if (opts.download_eagle3) { - plan.eagle3 = find_best_eagle3(all, primary.path); + plan.eagle3 = find_best_eagle3(all, primary.path, tag); + } + + if (primary.path.empty() && + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { + LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); + list_available_gguf_files(all); } return plan; diff --git a/common/speculative.cpp b/common/speculative.cpp index 3cb08767bd..ee94d7c376 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2284,7 +2284,7 @@ common_speculative_init_result::common_speculative_init_result( std::string model_path; if (has_draft) { model_path = params.speculative.draft.mparams.path; - LOG_TRC("%s: loading draft model '%s'\n", __func__, model_path.c_str()); + LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str()); llama_model * model_dft = llama_model_load_from_file(params.model.path.c_str(), mparams); if (model_dft == NULL) { @@ -2304,7 +2304,7 @@ common_speculative_init_result::common_speculative_init_result( } else if (spec_mtp) { model_path = params.model.path; - LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str()); + LOG_INF("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str()); llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams); if (ctx_dft == nullptr) {