diff --git a/common/speculative.cpp b/common/speculative.cpp index 7579a565f..fb712397c 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -159,6 +159,10 @@ struct common_speculative_state { size_t n_gen_tokens = 0; // number of tokens generated by this implementation. size_t n_acc_tokens = 0; // number of tokens accepted by the target model. + // Position zero represents speculative position 1. + std::vector drafted_by_position; + std::vector accepted_by_position; + // TODO: track performance of most recent calls const bool gen_perf = true; // whether to generate performance stats. @@ -1577,6 +1581,11 @@ llama_tokens common_speculative_draft( spec->curr_impl = impl.get(); impl->n_gen_drafts++; impl->n_gen_tokens += result.size(); + impl->drafted_by_position.resize(std::max(impl->drafted_by_position.size(), result.size())); + impl->accepted_by_position.resize(impl->drafted_by_position.size()); + for (size_t i = 0; i < result.size(); ++i) { + impl->drafted_by_position[i]++; + } break; // We have a draft, so break out of the loop and return it. } @@ -1610,6 +1619,12 @@ void common_speculative_accept(common_speculative * spec, uint16_t n_accepted) { if (n_accepted > 0) { impl->n_acc_drafts++; impl->n_acc_tokens += n_accepted; + + const size_t n_accepted_positions = std::min(n_accepted, impl->drafted_by_position.size()); + impl->accepted_by_position.resize(std::max(impl->accepted_by_position.size(), n_accepted_positions)); + for (size_t i = 0; i < n_accepted_positions; ++i) { + impl->accepted_by_position[i]++; + } } impl->accept(n_accepted); @@ -2083,7 +2098,12 @@ int32_t common_speculative_on_target_seq_batch( return 0; } - if (!common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_DFLASH)) { + // Self-speculative stages do not consume target hidden-state features. + if (!common_speculative_has_target_features(spec)) { + return 0; + } + + if (common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_MTP)) { llama_context * ctx_mtp = common_speculative_get_companion_ctx(spec); ctx_mtp = ctx_mtp ? ctx_mtp : ctx_tgt; if (ctx_mtp == nullptr) { @@ -2917,6 +2937,34 @@ common_speculative_type common_speculative_current_type(const common_speculative return spec->curr_impl->type; } +common_speculative_metrics_snapshot common_speculative_get_metrics_snapshot(const common_speculative * spec) { + common_speculative_metrics_snapshot snapshot; + if (spec == nullptr) { + return snapshot; + } + + snapshot.stages.reserve(spec->impls.size()); + for (const auto & impl : spec->impls) { + common_speculative_metrics_stage_snapshot stage; + stage.type = impl->type; + stage.n_call_begin = impl->n_call_begin; + stage.n_call_draft = impl->n_call_draft; + stage.n_call_accept = impl->n_call_accept; + stage.n_gen_drafts = impl->n_gen_drafts; + stage.n_acc_drafts = impl->n_acc_drafts; + stage.n_gen_tokens = impl->n_gen_tokens; + stage.n_acc_tokens = impl->n_acc_tokens; + stage.drafted_by_position = impl->drafted_by_position; + stage.accepted_by_position = impl->accepted_by_position; + stage.t_begin_us = impl->t_begin_us; + stage.t_draft_us = impl->t_draft_us; + stage.t_accept_us = impl->t_accept_us; + snapshot.stages.push_back(std::move(stage)); + } + + return snapshot; +} + void common_speculative_context_shift( common_speculative * spec, llama_seq_id seq_id, @@ -3101,3 +3149,153 @@ int32_t mtp_update_kv_cache(struct llama_context * ctx, const llama_batch& batch llama_set_mtp_op_type(ctx, MTP_OP_NONE); return ret; } +common_speculative_round_result common_speculative_run_round( + common_speculative * spec, + llama_model * model, + llama_context * ctx, + common_sampler * sampler, + llama_context * ctx_guidance, + common_params_speculative params, + const common_params_sampling & sparams, + llama_seq_id seq_id, + llama_pos n_past, + int n_predict_budget, + bool have_carry, + const llama_tokens & draft_history, + llama_token carry_token) { + common_speculative_round_result result; + + if (spec == nullptr || n_predict_budget == 1) { + return result; + } + + const int n_ctx = llama_n_ctx(ctx); + const int n_batch = llama_n_batch(ctx); + int max_usable_draft = params.get_max_stage_n_max(); + if (max_usable_draft <= 0) { + max_usable_draft = params.n_max; + } + const int configured_n_max = common_speculative_get_configured_n_max(spec); + if (configured_n_max > 0) { + max_usable_draft = std::min(max_usable_draft, configured_n_max); + } + if (n_predict_budget >= 0) { + max_usable_draft = std::min(max_usable_draft, n_predict_budget - 2); + } + max_usable_draft = std::min(max_usable_draft, n_ctx - (int) n_past - 2); + max_usable_draft = std::min(max_usable_draft, n_batch - 1); + + // A normal speculative round needs room for the sampled token, at least one + // draft position, and the verification carry. + if (max_usable_draft <= 0) { + return result; + } + + params.n_max = std::max(0, max_usable_draft); + params.n_min = std::min(std::max(0, params.n_min), params.n_max); + for (auto & stage : params.stages) { + if (stage.has_n_max_override()) { + stage.n_max = std::min(stage.n_max, params.n_max); + } + if (stage.has_n_min_override()) { + const int stage_max = stage.has_n_max_override() ? stage.n_max : params.n_max; + stage.n_min = std::min(stage.n_min, stage_max); + } + } + + result.attempted = true; + result.sampled_before_from_carry = have_carry; + if (have_carry) { + result.sampled_before = carry_token; + } else { + result.sampled_before = common_sampler_sample_legacy(sampler, ctx, ctx_guidance); + common_sampler_accept(sampler, ctx, result.sampled_before, true); + } + result.sampled_before_ready = true; + + auto draft_result = common_speculative_draft_ex( + spec, + ctx, + params, + draft_history, + result.sampled_before, + n_past, + seq_id); + auto & draft = draft_result.tokens; + + const int min_usable_draft = params.get_min_usable_stage_n_min(); + if ((int) draft.size() < min_usable_draft || (draft.empty() && !draft_result.target_only)) { + return result; + } + + if (llama_model_has_recurrent(model) || llama_model_is_openpangu(model)) { + if (!common_speculative_before_draft( + spec, + model, + ctx, + sampler, + sparams, + seq_id, + n_past, + result.sampled_before, + (int) draft.size() + 1, + params.recurrent_ckpt_mode)) { + draft.clear(); + } + } + + if (draft.empty() && !draft_result.target_only) { + return result; + } + + llama_batch verify_batch = llama_batch_init((int) draft.size() + 1, 0, 1); + std::vector verify_indices; + verify_indices.reserve(draft.size() + 1); + + common_batch_add(verify_batch, result.sampled_before, n_past, { seq_id }, true); + verify_indices.push_back(0); + for (size_t i = 0; i < draft.size(); ++i) { + common_batch_add(verify_batch, draft[i], n_past + 1 + (llama_pos) i, { seq_id }, true); + verify_indices.push_back((int) i + 1); + } + + if (llama_decode(ctx, verify_batch) != 0) { + llama_batch_free(verify_batch); + result.failed = true; + result.error = "speculative verify decode failed"; + return result; + } + + std::vector ids; + try { + ids = common_sampler_sample_and_accept_n(sampler, ctx, verify_indices, draft); + } catch (const std::exception & e) { + llama_batch_free(verify_batch); + result.failed = true; + result.error = e.what(); + return result; + } + + std::vector accepted_output_indices; + if (!ids.empty()) { + accepted_output_indices.assign(verify_indices.begin(), verify_indices.begin() + ids.size()); + } + + if (!ids.empty()) { + common_speculative_commit( + spec, + ctx, + sampler, + seq_id, + result.sampled_before, + ids, + (int) draft.size(), + n_past + 1, + accepted_output_indices); + result.ids = std::move(ids); + result.used_speculative = true; + } + + llama_batch_free(verify_batch); + return result; +} diff --git a/common/speculative.h b/common/speculative.h index 436ab719c..05ff28ae5 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -40,6 +40,31 @@ struct common_speculative_draft_result { bool target_only = false; }; +struct common_speculative_metrics_stage_snapshot { + common_speculative_type type = COMMON_SPECULATIVE_TYPE_NONE; + + uint64_t n_call_begin = 0; + uint64_t n_call_draft = 0; + uint64_t n_call_accept = 0; + + uint64_t n_gen_drafts = 0; + uint64_t n_acc_drafts = 0; + uint64_t n_gen_tokens = 0; + uint64_t n_acc_tokens = 0; + + // Position zero represents speculative position 1. + std::vector drafted_by_position; + std::vector accepted_by_position; + + int64_t t_begin_us = 0; + int64_t t_draft_us = 0; + int64_t t_accept_us = 0; +}; + +struct common_speculative_metrics_snapshot { + std::vector stages; +}; + // comma separated list of all types std::string common_speculative_type_name_str(); @@ -236,6 +261,8 @@ void common_speculative_print_stats(const common_speculative * spec, double slot common_speculative_type common_speculative_current_type(const common_speculative * spec); +common_speculative_metrics_snapshot common_speculative_get_metrics_snapshot(const common_speculative * spec); + // Context shift for MTP to match how server handle main model void common_speculative_context_shift( common_speculative * spec, @@ -243,3 +270,29 @@ void common_speculative_context_shift( llama_pos kv_keep, llama_pos kv_discard, llama_pos kv_past); + +struct common_speculative_round_result { + bool attempted = false; + bool sampled_before_ready = false; + bool sampled_before_from_carry = false; + bool used_speculative = false; + bool failed = false; + std::string error; + llama_token sampled_before = LLAMA_TOKEN_NULL; + llama_tokens ids; +}; + +common_speculative_round_result common_speculative_run_round( + common_speculative * spec, + llama_model * model, + llama_context * ctx, + common_sampler * sampler, + llama_context * ctx_guidance, + common_params_speculative params, + const common_params_sampling & sparams, + llama_seq_id seq_id, + llama_pos n_past, + int n_predict_budget, + bool have_carry, + const llama_tokens & draft_history, + llama_token carry_token); diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 759ad1b45..17cf00ce4 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -50,6 +50,7 @@ else() endif() add_subdirectory(save-load-state) add_subdirectory(simple) + add_subdirectory(spec-bench) add_subdirectory(speculative) add_subdirectory(sweep-bench) add_subdirectory(tokenize) diff --git a/examples/spec-bench/CMakeLists.txt b/examples/spec-bench/CMakeLists.txt new file mode 100644 index 000000000..2111a69ab --- /dev/null +++ b/examples/spec-bench/CMakeLists.txt @@ -0,0 +1,41 @@ +set(TARGET llama-spec-bench) +set(SPEC_BENCH_PROMPT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prompts") +set(SPEC_BENCH_PROMPT_FILES + "${SPEC_BENCH_PROMPT_DIR}/code.txt" + "${SPEC_BENCH_PROMPT_DIR}/extract.txt" + "${SPEC_BENCH_PROMPT_DIR}/story.txt" +) +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SPEC_BENCH_PROMPT_FILES}) + +function(spec_bench_escape_cpp_string output_var input_value) + string(REPLACE "\\" "\\\\" value "${input_value}") + string(REPLACE [=["]=] [=[\"]=] value "${value}") + string(REPLACE "\r" "\\r" value "${value}") + string(REPLACE "\n" "\\n" value "${value}") + set(${output_var} "${value}" PARENT_SCOPE) +endfunction() + +file(READ "${SPEC_BENCH_PROMPT_DIR}/code.txt" SPEC_BENCH_PROMPT_CODE_RAW) +file(READ "${SPEC_BENCH_PROMPT_DIR}/extract.txt" SPEC_BENCH_PROMPT_EXTRACT_RAW) +file(READ "${SPEC_BENCH_PROMPT_DIR}/story.txt" SPEC_BENCH_PROMPT_STORY_RAW) +string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_CODE_RAW "${SPEC_BENCH_PROMPT_CODE_RAW}") +string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_EXTRACT_RAW "${SPEC_BENCH_PROMPT_EXTRACT_RAW}") +string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_STORY_RAW "${SPEC_BENCH_PROMPT_STORY_RAW}") +string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_CODE_RAW "${SPEC_BENCH_PROMPT_CODE_RAW}") +string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_EXTRACT_RAW "${SPEC_BENCH_PROMPT_EXTRACT_RAW}") +string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_STORY_RAW "${SPEC_BENCH_PROMPT_STORY_RAW}") +spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_CODE "${SPEC_BENCH_PROMPT_CODE_RAW}") +spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_EXTRACT "${SPEC_BENCH_PROMPT_EXTRACT_RAW}") +spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_STORY "${SPEC_BENCH_PROMPT_STORY_RAW}") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/spec-bench-prompts.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/spec-bench-prompts.h" + @ONLY +) + + +add_executable(${TARGET} spec-bench.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) +target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") diff --git a/examples/spec-bench/README.md b/examples/spec-bench/README.md new file mode 100644 index 000000000..38cb35651 --- /dev/null +++ b/examples/spec-bench/README.md @@ -0,0 +1,81 @@ +# ik_llama.cpp/examples/spec-bench + +`llama-spec-bench` is a direct C++ speculative benchmark for prompt-driven tasks. +It reuses the normal `llama-common` startup and speculative lifecycle instead of +benchmarking through `llama-server`. + +## Scope + +- built-in canonical tasks: `code`, `extract`, `story` +- all three canonical built-in workloads by default, or one plain custom prompt via `-p` / `-f` +- optional strict JSONL prompt-file override for structured multi-prompt workloads +- baseline and speculative runs use the same binary and normal model/sampler args +- Markdown report by default; compact JSONL is available with `--output-format jsonl` +- per-stage drafted and accepted counts by speculative position + +## Benchmark-specific flags + +- `--prompts `: replace the built-in tasks with a strict JSONL prompt file +- `-p, --prompt `: run one inline custom prompt +- `-f, --file `: run one plain-text custom prompt file; the file is one prompt, not one task per line +- `--task `: select built-in tasks +- `--repeat `: repeat each task `n` times +- `--retry `: retry transient task failures up to `n` times +- `--output-format jsonl`: select the common JSONL output convention; output is written to `stdout` +- `--output-details`: print prompts and responses first, followed by normal and detailed Markdown metrics; JSONL includes complete details +- `--predict ` / `-n `: command-level generation budget for every task without a row override + +## Input modes + +Choose exactly one mode: built-ins (optionally narrowed with `--task`), one `-p` prompt, one `-f` file, or one `--prompts` JSONL dataset. +Examples: + +```bash +./build/bin/llama-spec-bench -m model.gguf -n 4 -p "Write a merge sort in C++." +./build/bin/llama-spec-bench -m model.gguf -n 4 -f examples/spec-bench/prompts/code.txt +``` + +## Dataset format + +Each JSONL line must be an object containing a non-empty `prompt` string. Optional fields are +`id`, `name`, `category`, and positive integer `max_tokens`: + +```json +{"id":"task-1","name":"math","category":"reasoning","prompt":"Solve 12*17.","max_tokens":64} +``` + +IDs default to the one-based input line number and must be unique. Unknown fields, +duplicate IDs, empty prompts, malformed JSON, and invalid `max_tokens` values are rejected. +The file replaces the built-in task set for that invocation. + +The canonical prompts are embedded into the executable at configure time from `prompts/code.txt`, +`prompts/extract.txt`, and `prompts/story.txt`; no source-tree or network access is needed at runtime. + +Compact JSONL includes raw `drafted_by_position` and `accepted_by_position` arrays for +every stage. Detailed JSONL additionally includes the derived +`acceptance_rate_by_position` and `conditional_acceptance_rate` arrays. Array element +zero is speculative position one; the first conditional rate is `null` because it has +no preceding position. + +Acceptance length is defined consistently as `1 + accepted_tokens / num_drafts` in detailed JSON, compact JSON, Markdown, and repeat summaries. + +Repeated attempts are executed in one process. Stateful drafting stages, including +adaptive n-gram stages and lookup caches, may therefore carry learned state from an +earlier task or repeat; use `--repeat 1` and separate invocations when independent +samples are required. Pin the chat-template mode (`--jinja` or `--no-jinja`) when +comparing runs because it changes the effective prompt. Speculative verification can +also diverge from a baseline after a near-tie because batched evaluation changes +floating-point reduction order, so this tool is a performance and acceptance benchmark, +not a bit-identical output checker. + +## Example + +```bash +./build/bin/llama-spec-bench \ + -m model.gguf \ + --seed 123 \ + --temp 0 \ + --predict 256 \ + --output-format jsonl \ + --task code,extract,story > results.jsonl +``` diff --git a/examples/spec-bench/prompts/code.txt b/examples/spec-bench/prompts/code.txt new file mode 100644 index 000000000..630911ced --- /dev/null +++ b/examples/spec-bench/prompts/code.txt @@ -0,0 +1 @@ +Write a quick sort python algorithm, answer only the code. diff --git a/examples/spec-bench/prompts/extract.txt b/examples/spec-bench/prompts/extract.txt new file mode 100644 index 000000000..54ed40570 --- /dev/null +++ b/examples/spec-bench/prompts/extract.txt @@ -0,0 +1,9 @@ +Extract all core events with their exact dates into a bulleted list + +YouTube is an American online video sharing platform owned by Google. YouTube was founded on February 14, 2005, by Chad Hurley, Jawed Karim, and Steve Chen, who were former employees of PayPal. Headquartered in San Bruno, California, it is the second-most-visited website in the world, after Google itself. In January 2024, YouTube had more than 2.7 billion monthly active users, who collectively consumed more than one billion hours of video content every day. As of May 2019, videos were being uploaded to the platform at a rate of more than 500 hours of content per minute, and as of mid-2024, there were approximately 14.8 billion videos in total. + +On November 13, 2006, YouTube was purchased by Google for US$1.65 billion (equivalent to $2.44 billion in 2025). Google expanded YouTube's business model from generating revenue through advertisements alone to offering paid content such as movies and exclusive content explicitly produced for YouTube. It also offers YouTube Premium, a paid subscription option for watching content without ads. YouTube incorporated the Google AdSense program, generating more revenue for both YouTube and approved content creators. In 2023, YouTube's advertising revenue totaled $31.7 billion, a 2% increase from the $31.1 billion reported in 2022. From Q4 2023 to Q3 2024, YouTube's combined revenue from advertising and subscriptions exceeded $50 billion. + +Since its purchase by Google, YouTube has expanded beyond the core website, creating mobile apps, network television, games, and the ability to link with other platforms. Video categories on YouTube include music videos, video clips, news, short and feature films, songs, documentaries, movie trailers, teasers, TV spots, live streams, vlogs, and more. Most content is generated by individuals, including collaborations between YouTubers and corporate sponsors. Established media, news, and entertainment corporations have also created and expanded their visibility on YouTube channels to reach bigger audiences. + +YouTube has had unprecedented social impact, influencing popular culture, internet trends, and creating multimillionaire celebrities. Despite its growth and success, the platform has been criticized for its facilitation of the spread of misinformation and copyrighted content, routinely violating its users' privacy, excessive censorship, endangering the safety of children and their well-being, and for its inconsistent implementation of platform guidelines. diff --git a/examples/spec-bench/prompts/story.txt b/examples/spec-bench/prompts/story.txt new file mode 100644 index 000000000..f5fabd3e6 --- /dev/null +++ b/examples/spec-bench/prompts/story.txt @@ -0,0 +1 @@ +Give me an extended summary of the history of Bulgaria diff --git a/examples/spec-bench/spec-bench-prompts.h.in b/examples/spec-bench/spec-bench-prompts.h.in new file mode 100644 index 000000000..1c9c65977 --- /dev/null +++ b/examples/spec-bench/spec-bench-prompts.h.in @@ -0,0 +1,5 @@ +#pragma once + +#define SPEC_BENCH_PROMPT_CODE "@SPEC_BENCH_PROMPT_CODE@" +#define SPEC_BENCH_PROMPT_EXTRACT "@SPEC_BENCH_PROMPT_EXTRACT@" +#define SPEC_BENCH_PROMPT_STORY "@SPEC_BENCH_PROMPT_STORY@" diff --git a/examples/spec-bench/spec-bench.cpp b/examples/spec-bench/spec-bench.cpp new file mode 100644 index 000000000..b2c3c9a9f --- /dev/null +++ b/examples/spec-bench/spec-bench.cpp @@ -0,0 +1,1356 @@ +#include "common.h" +#include "chat.h" +#include "speculative.h" +#include "llama.h" +#include "spec-bench-prompts.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::ordered_json; + +struct spec_bench_options { + std::string prompts_path; + std::vector task_names; + std::string output_format = "md"; + bool output_details = false; + bool task_selection_seen = false; + bool inline_prompt_seen = false; + bool file_prompt_seen = false; + int repeat = 1; + int retry = 0; +}; + +struct spec_bench_task { + std::string id; + std::string name; + std::string category; + std::string prompt; + int max_tokens = -1; + bool builtin = false; +}; + +struct spec_bench_stage_delta { + common_speculative_type type = COMMON_SPECULATIVE_TYPE_NONE; + uint64_t num_drafts = 0; + uint64_t accepted_drafts = 0; + uint64_t draft_tokens = 0; + uint64_t accepted_tokens = 0; + std::vector drafted_by_position; + std::vector accepted_by_position; + int64_t t_begin_us = 0; + int64_t t_draft_us = 0; + int64_t t_accept_us = 0; +}; + +struct spec_bench_metrics_delta { + std::vector stages; + uint64_t num_drafts = 0; + uint64_t accepted_drafts = 0; + uint64_t draft_tokens = 0; + uint64_t accepted_tokens = 0; + int64_t t_begin_us = 0; + int64_t t_draft_us = 0; + int64_t t_accept_us = 0; +}; + +struct spec_bench_attempt_result { + bool ok = false; + bool hit_eog = false; + std::string error; + std::string output_text; + llama_tokens output_tokens; + int prompt_tokens = 0; + int generated_tokens = 0; + int retries_used = 0; + double prompt_s = 0.0; + double decode_s = 0.0; + double total_s = 0.0; + std::string effective_prompt; + spec_bench_metrics_delta spec_delta; +}; + +struct spec_bench_record { + spec_bench_task task; + spec_bench_attempt_result result; + int repeat_index = 0; +}; +struct spec_bench_summary { + int attempts = 0; + int successes = 0; + int failures = 0; + int prompt_tokens = 0; + int generated_tokens = 0; + int retries_used = 0; + double prompt_s = 0.0; + double decode_s = 0.0; + double total_s = 0.0; + spec_bench_metrics_delta spec_delta; +}; + +static std::vector spec_bench_builtin_tasks() { + const std::string extract_prompt = SPEC_BENCH_PROMPT_EXTRACT; + + return { + { + /* .id = */ "builtin-code", + /* .name = */ "code", + /* .category = */ "code", + /* .prompt = */ SPEC_BENCH_PROMPT_CODE, + /* .max_tokens = */ -1, + /* .builtin = */ true, + }, + { + /* .id = */ "builtin-extract", + /* .name = */ "extract", + /* .category = */ "extraction", + /* .prompt = */ extract_prompt, + /* .max_tokens = */ -1, + /* .builtin = */ true, + }, + { + /* .id = */ "builtin-story", + /* .name = */ "story", + /* .category = */ "long-form-summary", + /* .prompt = */ SPEC_BENCH_PROMPT_STORY, + /* .max_tokens = */ -1, + /* .builtin = */ true, + }, + }; +} + +static void spec_bench_print_usage(const char * argv0) { + LOG_TEE("usage: %s [benchmark options] [normal llama args]\n", argv0); + LOG_TEE("\n"); + LOG_TEE("benchmark options:\n"); + LOG_TEE(" -p PROMPT / -f FILE one custom plain-text prompt (mutually exclusive)\n"); + LOG_TEE(" --prompts PATH optional strict JSONL prompt-file override\n"); + LOG_TEE(" --task LIST built-in tasks to run, e.g. code,extract,story\n"); + LOG_TEE(" --repeat N repeat each task N times (default: 1)\n"); + LOG_TEE(" --retry N retry each failed task up to N times (default: 0)\n"); + LOG_TEE(" --output-format md|jsonl emit Markdown by default, or JSONL\n"); + LOG_TEE(" --output-details include prompt/output and detailed metrics\n"); + LOG_TEE("\n"); +} + +static bool spec_bench_parse_args( + int argc, + char ** argv, + spec_bench_options & opts, + std::vector & passthrough) { + passthrough.clear(); + passthrough.reserve(argc); + passthrough.push_back(argv[0]); + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + auto require_value = [&](const char * name) -> const char * { + if (i + 1 >= argc) { + LOG_TEE("%s: missing value after %s\n", __func__, name); + return nullptr; + } + return argv[++i]; + }; + + if (arg == "--prompts") { + const char * value = require_value("--prompts"); + if (!value) { + return false; + } + if (!opts.prompts_path.empty()) { + LOG_TEE("--prompts may be specified only once\n"); + return false; + } + opts.prompts_path = value; + continue; + } + if (arg == "-p" || arg == "--prompt") { + if (opts.inline_prompt_seen) { + LOG_TEE("inline prompt may be specified only once\n"); + return false; + } + const char * value = require_value(arg.c_str()); + if (!value) { + return false; + } + opts.inline_prompt_seen = true; + passthrough.push_back(arg); + passthrough.push_back(value); + continue; + } + if (arg == "-f" || arg == "--file") { + if (opts.file_prompt_seen) { + LOG_TEE("prompt file may be specified only once\n"); + return false; + } + const char * value = require_value(arg.c_str()); + if (!value) { + return false; + } + opts.file_prompt_seen = true; + passthrough.push_back(arg); + passthrough.push_back(value); + continue; + } + if (arg == "--dataset") { + LOG_TEE("--dataset is no longer supported; use --prompts PATH\n"); + return false; + } + if (arg == "--task") { + const char * value = require_value("--task"); + if (!value) { + return false; + } + opts.task_selection_seen = true; + for (const auto & selection : string_split(std::string(value), ",")) { + if (string_strip(selection).empty()) { + LOG_TEE("--task must not contain empty selections\n"); + return false; + } + } + for (const auto & name : string_split(std::string(value), ',')) { + const std::string trimmed = string_strip(name); + if (!trimmed.empty()) { + opts.task_names.push_back(trimmed); + } + } + continue; + } + if (arg == "--repeat") { + const char * value = require_value("--repeat"); + if (!value) { + return false; + } + opts.repeat = std::max(1, std::stoi(value)); + continue; + } + if (arg == "--retry") { + const char * value = require_value("--retry"); + if (!value) { + return false; + } + opts.retry = std::max(0, std::stoi(value)); + continue; + } + if (arg == "--output-format") { + const char * value = require_value("--output-format"); + if (!value) { return false; } + opts.output_format = string_strip(value); + if (opts.output_format != "md" && opts.output_format != "jsonl") { + LOG_TEE("--output-format must be md or jsonl\n"); + return false; + } + continue; + } + if (arg == "--output-details") { + opts.output_details = true; + continue; + } + if (arg == "--output") { + LOG_TEE("--output is not a benchmark destination; use --output-format jsonl and redirect stdout\n"); + return false; + } + + passthrough.push_back(arg); + } + + const int single_prompt_modes = (opts.inline_prompt_seen ? 1 : 0) + (opts.file_prompt_seen ? 1 : 0); + if (single_prompt_modes > 1) { + LOG_TEE("choose exactly one of -p/--prompt or -f/--file\n"); + return false; + } + if (!opts.prompts_path.empty() && (single_prompt_modes > 0 || opts.task_selection_seen)) { + LOG_TEE("--prompts cannot be combined with --task, -p/--prompt, or -f/--file\n"); + return false; + } + if (single_prompt_modes > 0 && opts.task_selection_seen) { + LOG_TEE("-p/--prompt and -f/--file cannot be combined with --task\n"); + return false; + } + + return true; +} + +static std::vector spec_bench_make_argv(std::vector & args) { + std::vector out; + out.reserve(args.size()); + for (std::string & arg : args) { + out.push_back(arg.data()); + } + return out; +} + + +static std::vector spec_bench_load_dataset(const std::string & path) { + std::ifstream in(path); + if (!in) { + throw std::runtime_error("failed to open dataset: " + path); + } + + std::vector tasks; + std::set ids; + const std::set allowed_fields = {"id", "name", "category", "prompt", "max_tokens"}; + std::string line; + int line_no = 0; + while (std::getline(in, line)) { + ++line_no; + if (string_strip(line).empty()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " is empty"); + } + + json row; + try { + row = json::parse(line); + } catch (const std::exception & e) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " is invalid JSON: " + e.what()); + } + if (!row.is_object()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " must be a JSON object"); + } + + for (const auto & item : row.items()) { + if (allowed_fields.count(item.key()) == 0) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " has unknown field: " + item.key()); + } + } + if (!row.contains("prompt") || !row.at("prompt").is_string()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " must contain a string prompt"); + } + + spec_bench_task task; + task.id = row.contains("id") ? row.at("id").get() : std::to_string(line_no); + task.name = row.contains("name") ? row.at("name").get() : task.id; + task.category = row.contains("category") ? row.at("category").get() : "dataset"; + task.prompt = row.at("prompt").get(); + if (string_strip(task.id).empty() || string_strip(task.name).empty() || string_strip(task.category).empty()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " has an empty id, name, or category"); + } + if (string_strip(task.prompt).empty()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " has an empty prompt"); + } + if (!ids.insert(task.id).second) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " duplicates id: " + task.id); + } + + task.max_tokens = -1; + if (row.contains("max_tokens")) { + const auto & max_tokens = row.at("max_tokens"); + if (!max_tokens.is_number_integer()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " max_tokens must be a positive integer"); + } + const int64_t value = max_tokens.get(); + if (value <= 0 || value > std::numeric_limits::max()) { + throw std::runtime_error("prompt file line " + std::to_string(line_no) + " max_tokens must be a positive integer"); + } + task.max_tokens = (int) value; + } + task.builtin = false; + tasks.push_back(std::move(task)); + } + + if (tasks.empty()) { + throw std::runtime_error("prompt file contains no rows: " + path); + } + + return tasks; +} + +static std::string spec_bench_prompt_file_basename(const std::string & path) { + const size_t slash = path.find_last_of("/\\"); + const std::string name = slash == std::string::npos ? path : path.substr(slash + 1); + return name.empty() ? "prompt" : name; +} + +static std::vector spec_bench_select_tasks(const spec_bench_options & opts, const gpt_params & params) { + if (opts.inline_prompt_seen || opts.file_prompt_seen) { + if (string_strip(params.prompt).empty()) { + throw std::runtime_error("custom prompt must be non-empty"); + } + const bool from_file = opts.file_prompt_seen; + const std::string name = from_file ? spec_bench_prompt_file_basename(params.prompt_file) : "prompt"; + return {{ + { + "custom-" + name, + name, + "custom", + params.prompt, + -1, + false, + }, + }}; + } + if (!opts.prompts_path.empty()) { + return spec_bench_load_dataset(opts.prompts_path); + } + + std::vector builtin = spec_bench_builtin_tasks(); + if (opts.task_names.empty()) { + return builtin; + } + + std::set wanted; + for (const auto & name : opts.task_names) { + wanted.insert(string_lower(name)); + } + + std::vector selected; + std::set matched; + for (const auto & task : builtin) { + const std::string normalized = string_lower(task.name); + if (wanted.count(normalized) > 0) { + selected.push_back(task); + matched.insert(normalized); + } + } + + if (matched.size() != wanted.size()) { + std::vector unknown; + for (const auto & name : wanted) { + if (matched.count(name) == 0) { + unknown.push_back(name); + } + } + throw std::runtime_error("unknown built-in task name(s): " + string_join(unknown, ", ")); + } + + return selected; +} + +static spec_bench_metrics_delta spec_bench_snapshot_delta( + const common_speculative_metrics_snapshot & before, + const common_speculative_metrics_snapshot & after) { + spec_bench_metrics_delta delta; + const size_t n = std::min(before.stages.size(), after.stages.size()); + delta.stages.reserve(n); + + for (size_t i = 0; i < n; ++i) { + const auto & lhs = before.stages[i]; + const auto & rhs = after.stages[i]; + spec_bench_stage_delta stage; + stage.type = rhs.type; + stage.num_drafts = rhs.n_gen_drafts - lhs.n_gen_drafts; + stage.accepted_drafts = rhs.n_acc_drafts - lhs.n_acc_drafts; + stage.draft_tokens = rhs.n_gen_tokens - lhs.n_gen_tokens; + stage.accepted_tokens = rhs.n_acc_tokens - lhs.n_acc_tokens; + const size_t n_drafted_positions = std::max(lhs.drafted_by_position.size(), rhs.drafted_by_position.size()); + const size_t n_accepted_positions = std::max(lhs.accepted_by_position.size(), rhs.accepted_by_position.size()); + stage.drafted_by_position.resize(n_drafted_positions); + stage.accepted_by_position.resize(n_accepted_positions); + for (size_t position = 0; position < n_drafted_positions; ++position) { + const uint64_t before_value = position < lhs.drafted_by_position.size() ? lhs.drafted_by_position[position] : 0; + const uint64_t after_value = position < rhs.drafted_by_position.size() ? rhs.drafted_by_position[position] : 0; + stage.drafted_by_position[position] = after_value >= before_value ? after_value - before_value : 0; + } + for (size_t position = 0; position < n_accepted_positions; ++position) { + const uint64_t before_value = position < lhs.accepted_by_position.size() ? lhs.accepted_by_position[position] : 0; + const uint64_t after_value = position < rhs.accepted_by_position.size() ? rhs.accepted_by_position[position] : 0; + stage.accepted_by_position[position] = after_value >= before_value ? after_value - before_value : 0; + } + stage.t_begin_us = rhs.t_begin_us - lhs.t_begin_us; + stage.t_draft_us = rhs.t_draft_us - lhs.t_draft_us; + stage.t_accept_us = rhs.t_accept_us - lhs.t_accept_us; + + delta.num_drafts += stage.num_drafts; + delta.accepted_drafts += stage.accepted_drafts; + delta.draft_tokens += stage.draft_tokens; + delta.accepted_tokens += stage.accepted_tokens; + delta.t_begin_us += stage.t_begin_us; + delta.t_draft_us += stage.t_draft_us; + delta.t_accept_us += stage.t_accept_us; + delta.stages.push_back(stage); + } + + return delta; +} + +static void spec_bench_accumulate(spec_bench_summary & summary, const spec_bench_attempt_result & result) { + summary.attempts++; + summary.successes += result.ok ? 1 : 0; + summary.failures += result.ok ? 0 : 1; + summary.prompt_tokens += result.prompt_tokens; + summary.generated_tokens += result.generated_tokens; + summary.retries_used += result.retries_used; + summary.prompt_s += result.prompt_s; + summary.decode_s += result.decode_s; + summary.total_s += result.total_s; + summary.spec_delta.num_drafts += result.spec_delta.num_drafts; + summary.spec_delta.accepted_drafts += result.spec_delta.accepted_drafts; + summary.spec_delta.draft_tokens += result.spec_delta.draft_tokens; + summary.spec_delta.accepted_tokens += result.spec_delta.accepted_tokens; + summary.spec_delta.t_begin_us += result.spec_delta.t_begin_us; + summary.spec_delta.t_draft_us += result.spec_delta.t_draft_us; + summary.spec_delta.t_accept_us += result.spec_delta.t_accept_us; + + if (summary.spec_delta.stages.size() < result.spec_delta.stages.size()) { + summary.spec_delta.stages.resize(result.spec_delta.stages.size()); + } + for (size_t i = 0; i < result.spec_delta.stages.size(); ++i) { + auto & dst = summary.spec_delta.stages[i]; + const auto & src = result.spec_delta.stages[i]; + dst.type = src.type; + dst.num_drafts += src.num_drafts; + dst.accepted_drafts += src.accepted_drafts; + dst.draft_tokens += src.draft_tokens; + dst.accepted_tokens += src.accepted_tokens; + if (dst.drafted_by_position.size() < src.drafted_by_position.size()) { + dst.drafted_by_position.resize(src.drafted_by_position.size()); + } + if (dst.accepted_by_position.size() < src.accepted_by_position.size()) { + dst.accepted_by_position.resize(src.accepted_by_position.size()); + } + for (size_t position = 0; position < src.drafted_by_position.size(); ++position) { + dst.drafted_by_position[position] += src.drafted_by_position[position]; + } + for (size_t position = 0; position < src.accepted_by_position.size(); ++position) { + dst.accepted_by_position[position] += src.accepted_by_position[position]; + } + dst.t_begin_us += src.t_begin_us; + dst.t_draft_us += src.t_draft_us; + dst.t_accept_us += src.t_accept_us; + } +} + +static double spec_bench_acceptance_rate(uint64_t accepted, uint64_t drafted); +static double spec_bench_acceptance_length(uint64_t accepted, uint64_t rounds); + +static json spec_bench_stage_json(const spec_bench_stage_delta & stage) { + const double acceptance_rate = spec_bench_acceptance_rate(stage.accepted_tokens, stage.draft_tokens); + const double acceptance_length = spec_bench_acceptance_length(stage.accepted_tokens, stage.num_drafts); + + json drafted_by_position = json::array(); + json accepted_by_position = json::array(); + json acceptance_rate_by_position = json::array(); + json conditional_acceptance_rate = json::array(); + for (size_t position = 0; position < stage.drafted_by_position.size(); ++position) { + const uint64_t drafted = stage.drafted_by_position[position]; + const uint64_t accepted = position < stage.accepted_by_position.size() + ? stage.accepted_by_position[position] + : 0; + drafted_by_position.push_back(drafted); + accepted_by_position.push_back(accepted); + acceptance_rate_by_position.push_back(drafted > 0 ? (double) accepted / (double) drafted : 0.0); + if (position == 0) { + conditional_acceptance_rate.push_back(nullptr); + } else { + const uint64_t previous_accepted = position - 1 < stage.accepted_by_position.size() + ? stage.accepted_by_position[position - 1] + : 0; + conditional_acceptance_rate.push_back(previous_accepted > 0 + ? json((double) accepted / (double) previous_accepted) + : json(nullptr)); + } + } + + return json{ + {"type", common_speculative_type_to_str(stage.type)}, + {"num_drafts", stage.num_drafts}, + {"accepted_drafts", stage.accepted_drafts}, + {"draft_tokens", stage.draft_tokens}, + {"accepted_tokens", stage.accepted_tokens}, + {"acceptance_rate", acceptance_rate}, + {"acceptance_length", acceptance_length}, + {"drafted_by_position", drafted_by_position}, + {"accepted_by_position", accepted_by_position}, + {"acceptance_rate_by_position", acceptance_rate_by_position}, + {"conditional_acceptance_rate", conditional_acceptance_rate}, + {"t_begin_s", stage.t_begin_us / 1e6}, + {"t_draft_s", stage.t_draft_us / 1e6}, + {"t_accept_s", stage.t_accept_us / 1e6}, + }; +} + +static json spec_bench_metrics_json(const spec_bench_metrics_delta & delta) { + const double acceptance_rate = spec_bench_acceptance_rate(delta.accepted_tokens, delta.draft_tokens); + const double acceptance_length = spec_bench_acceptance_length(delta.accepted_tokens, delta.num_drafts); + + json stages = json::array(); + for (const auto & stage : delta.stages) { + stages.push_back(spec_bench_stage_json(stage)); + } + + return json{ + {"num_drafts", delta.num_drafts}, + {"accepted_drafts", delta.accepted_drafts}, + {"draft_tokens", delta.draft_tokens}, + {"accepted_tokens", delta.accepted_tokens}, + {"acceptance_rate", acceptance_rate}, + {"acceptance_length", acceptance_length}, + {"draft_tokens_per_step", delta.num_drafts > 0 ? (double) delta.draft_tokens / (double) delta.num_drafts : 0.0}, + {"t_begin_s", delta.t_begin_us / 1e6}, + {"t_draft_s", delta.t_draft_us / 1e6}, + {"t_accept_s", delta.t_accept_us / 1e6}, + {"stages", stages}, + }; +} + +static json spec_bench_stage_types_json(const common_params_speculative & params) { + json stages = json::array(); + for (const auto & stage : params.get_resolved_stages()) { + stages.push_back(common_speculative_type_to_str(stage.type)); + } + return stages; +} + +static json spec_bench_task_names_json(const std::vector & tasks) { + json names = json::array(); + for (const auto & task : tasks) { + names.push_back(task.name); + } + return names; +} + +static json spec_bench_sampler_json(const gpt_params & params) { + return json{ + {"seed", params.seed}, + {"temp", params.sparams.temp}, + {"top_k", params.sparams.top_k}, + {"top_p", params.sparams.top_p}, + {"min_p", params.sparams.min_p}, + {"tfs_z", params.sparams.tfs_z}, + {"typical_p", params.sparams.typical_p}, + {"top_n_sigma", params.sparams.top_n_sigma}, + {"penalty_last_n", params.sparams.penalty_last_n}, + {"penalty_repeat", params.sparams.penalty_repeat}, + {"penalty_freq", params.sparams.penalty_freq}, + {"penalty_present", params.sparams.penalty_present}, + {"mirostat", params.sparams.mirostat}, + {"mirostat_tau", params.sparams.mirostat_tau}, + {"mirostat_eta", params.sparams.mirostat_eta}, + {"n_probs", params.sparams.n_probs}, + {"samplers_sequence", json(params.sparams.samplers_sequence)}, + }; +} + +static json spec_bench_runtime_json(const gpt_params & params) { + return json{ + {"model", params.model}, + {"model_alias", params.model_alias}, + {"n_ctx", params.n_ctx}, + {"n_predict", params.n_predict}, + {"n_batch", params.n_batch}, + {"n_ubatch", params.n_ubatch}, + {"n_threads", params.n_threads}, + {"n_threads_batch", params.n_threads_batch}, + {"n_gpu_layers", params.n_gpu_layers}, + {"flash_attn", params.flash_attn}, + {"numa", (int) params.numa}, + }; +} + +static int spec_bench_resolve_max_tokens(const spec_bench_task & task, const gpt_params & params) { + return task.max_tokens > 0 ? task.max_tokens : (params.n_predict > 0 ? params.n_predict : 256); +} + +static std::string spec_bench_decode_tokens( + const llama_context * ctx, + const llama_tokens & tokens, + bool special) { + std::string text; + for (llama_token token : tokens) { + text += common_token_to_piece(ctx, token, special); + } + return text; +} + +static std::string spec_bench_effective_prompt( + llama_model * model, + const gpt_params & params, + const std::string & prompt) { + if (!params.enable_chat_template) { + return prompt; + } + auto chat_templates = common_chat_templates_init(model, params.chat_template); + if (!chat_templates) { + throw std::runtime_error("failed to initialize chat templates"); + } + return common_chat_format_single(chat_templates.get(), {}, common_chat_msg{"user", prompt}, true, params.use_jinja); +} + +static llama_batch spec_bench_make_batch( + const llama_tokens & tokens, + int offset, + int n_tokens, + int n_past) { + llama_batch batch = llama_batch_init(n_tokens, 0, 1); + for (int i = 0; i < n_tokens; ++i) { + // Keep positions and sequence ids available for MTP/DFlash feature capture. + common_batch_add(batch, tokens[offset + i], n_past + i, { 0 }, true); + } + return batch; +} + +static spec_bench_attempt_result spec_bench_run_attempt( + const spec_bench_task & task, + const gpt_params & params, + llama_model * model, + llama_context * ctx, + common_speculative * spec, + common_sampler * sampler) { + spec_bench_attempt_result result; + + if (llama_model_has_encoder(model)) { + result.error = "encoder-decoder models are not supported"; + return result; + } + + const int task_max_tokens = spec_bench_resolve_max_tokens(task, params); + if (task_max_tokens <= 0) { + result.error = "max token budget resolved to zero"; + return result; + } + + result.effective_prompt = spec_bench_effective_prompt(model, params, task.prompt); + llama_tokens prompt_tokens = common_tokenize(ctx, result.effective_prompt, true, true); + result.prompt_tokens = (int) prompt_tokens.size(); + + const int n_ctx = llama_n_ctx(ctx); + if (result.prompt_tokens >= n_ctx - 2) { + result.error = "prompt does not fit into context"; + return result; + } + + common_sampler_reset(sampler); + if (spec != nullptr) { + common_speculative_clear_sequence_kv(spec, ctx, 0); + } else { + llama_kv_cache_clear(ctx); + } + llama_reset_timings(ctx); + if (params.has_mtp) { + llama_set_embeddings(ctx, true); + } + + llama_tokens embd = prompt_tokens; + llama_tokens speculative_tokens = prompt_tokens; + int n_past = 0; + int n_remain = task_max_tokens; + bool embd_is_prompt = true; + int final_prompt_output_index = -1; + llama_pos final_prompt_hidden_pos = -1; + bool have_carry = false; + llama_token carry_token = LLAMA_TOKEN_NULL; + + const auto spec_before = common_speculative_get_metrics_snapshot(spec); + for (llama_token token : prompt_tokens) { + common_sampler_accept(sampler, ctx, token, false); + } + + const int64_t t_prompt_start_us = ggml_time_us(); + + while (!embd.empty()) { + for (int i = 0; i < (int) embd.size(); i += params.n_batch) { + int n_eval = std::min(params.n_batch, (int) embd.size() - i); + llama_batch batch = spec_bench_make_batch(embd, i, n_eval, n_past); + const int decode_result = llama_decode(ctx, batch); + if (decode_result != 0) { + llama_batch_free(batch); + result.error = "prompt decode failed"; + return result; + } + if (spec != nullptr && embd_is_prompt) { + if (common_speculative_on_target_seq_batch(spec, ctx, batch, 0, true) != 0) { + llama_batch_free(batch); + result.error = "speculative prompt warmup failed"; + return result; + } + } + if (embd_is_prompt && i + n_eval == (int) embd.size()) { + final_prompt_output_index = n_eval - 1; + final_prompt_hidden_pos = n_past + n_eval - 1; + } + llama_batch_free(batch); + n_past += n_eval; + } + embd.clear(); + } + + if (spec != nullptr) { + static const llama_tokens empty_speculative_prompt; + const llama_tokens & speculative_prompt = + params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP) && + !params.speculative.has_composite_stage_chain() + ? empty_speculative_prompt + : speculative_tokens; + common_speculative_begin(spec, speculative_prompt); + if (params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP) && + final_prompt_output_index >= 0 && + final_prompt_hidden_pos >= 0 && + !common_speculative_capture_output_hidden(spec, ctx, final_prompt_output_index, 0, final_prompt_hidden_pos)) { + result.error = "failed to capture final prompt hidden state"; + return result; + } + } + if (params.has_mtp) { + llama_set_embeddings(ctx, false); + } + + const int64_t t_prompt_end_us = ggml_time_us(); + const int64_t t_decode_start_us = t_prompt_end_us; + + while (n_remain > 0) { + llama_tokens next_embd; + bool used_speculative = false; + bool have_fallback_sampled = false; + llama_token fallback_sampled = LLAMA_TOKEN_NULL; + + if (spec != nullptr && n_remain >= 3) { + static const llama_tokens empty_speculative_history; + const llama_tokens & draft_history = + params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP) && + !params.speculative.has_composite_stage_chain() + ? empty_speculative_history + : speculative_tokens; + auto round = common_speculative_run_round( + spec, model, ctx, sampler, nullptr, params.speculative, params.sparams, + 0, n_past, n_remain, have_carry, draft_history, carry_token); + if (round.failed) { + result.error = round.error; + return result; + } + if (round.sampled_before_ready && !round.used_speculative) { + have_fallback_sampled = true; + fallback_sampled = round.sampled_before; + } + if (round.used_speculative) { + if (!round.sampled_before_from_carry) { + result.output_tokens.push_back(round.sampled_before); + n_remain -= 1; + } + result.output_tokens.insert(result.output_tokens.end(), round.ids.begin(), round.ids.end()); + n_remain -= (int) round.ids.size(); + n_past += (int) round.ids.size(); + carry_token = round.ids.back(); + have_carry = !llama_token_is_eog(model, carry_token); + if (!have_carry) { + result.hit_eog = true; + n_remain = 0; + } + if (!params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP) || + params.speculative.has_composite_stage_chain()) { + speculative_tokens.push_back(round.sampled_before); + if (round.ids.size() > 1) { + speculative_tokens.insert(speculative_tokens.end(), round.ids.begin(), round.ids.end() - 1); + } + } + used_speculative = true; + } + } + + if (!used_speculative && have_carry) { + next_embd.push_back(carry_token); + have_carry = false; + used_speculative = true; + } + if (!used_speculative) { + const llama_token id = have_fallback_sampled + ? fallback_sampled + : common_sampler_sample_legacy(sampler, ctx, nullptr); + if (!have_fallback_sampled) { + common_sampler_accept(sampler, ctx, id, true); + } + + result.output_tokens.push_back(id); + next_embd.push_back(id); + n_remain -= 1; + } + + result.generated_tokens = (int) result.output_tokens.size(); + if (!result.output_tokens.empty() && llama_token_is_eog(model, result.output_tokens.back())) { + result.hit_eog = true; + break; + } + + embd = std::move(next_embd); + embd_is_prompt = false; + + for (int i = 0; i < (int) embd.size(); i += params.n_batch) { + int n_eval = std::min(params.n_batch, (int) embd.size() - i); + llama_batch batch = spec_bench_make_batch(embd, i, n_eval, n_past); + const int decode_result = llama_decode(ctx, batch); + if (decode_result != 0) { + llama_batch_free(batch); + result.error = "decode failed"; + return result; + } + llama_batch_free(batch); + if (spec != nullptr && (!params.speculative.has_stage_type(COMMON_SPECULATIVE_TYPE_MTP) || params.speculative.has_composite_stage_chain())) { + speculative_tokens.insert(speculative_tokens.end(), embd.begin() + i, embd.begin() + i + n_eval); + } + n_past += n_eval; + } + + embd.clear(); + + if (n_past >= n_ctx - 2) { + break; + } + } + + const int64_t t_decode_end_us = ggml_time_us(); + const auto spec_after = common_speculative_get_metrics_snapshot(spec); + + result.prompt_s = (t_prompt_end_us - t_prompt_start_us) / 1e6; + result.decode_s = (t_decode_end_us - t_decode_start_us) / 1e6; + result.total_s = (t_decode_end_us - t_prompt_start_us) / 1e6; + result.generated_tokens = (int) result.output_tokens.size(); + result.output_text = spec_bench_decode_tokens(ctx, result.output_tokens, params.special); + result.spec_delta = spec_bench_snapshot_delta(spec_before, spec_after); + result.ok = true; + + return result; +} + +static std::string spec_bench_positions(const spec_bench_stage_delta & stage) { + std::ostringstream out; + const size_t count = std::max(stage.drafted_by_position.size(), stage.accepted_by_position.size()); + for (size_t i = 0; i < count; ++i) { + if (i > 0) { out << ", "; } + const uint64_t drafted = i < stage.drafted_by_position.size() ? stage.drafted_by_position[i] : 0; + const uint64_t accepted = i < stage.accepted_by_position.size() ? stage.accepted_by_position[i] : 0; + out << accepted << "/" << drafted; + } + return out.str(); +} + +static json spec_bench_position_array(const std::vector & values) { + json result = json::array(); + for (const uint64_t value : values) { + result.push_back(value); + } + return result; +} + +static double spec_bench_acceptance_rate(uint64_t accepted, uint64_t drafted) { + return drafted > 0 ? (double) accepted / (double) drafted : 0.0; +} + +static double spec_bench_acceptance_length(uint64_t accepted, uint64_t rounds) { + return rounds > 0 ? 1.0 + (double) accepted / (double) rounds : 0.0; +} + +static json spec_bench_attempt_json( + const gpt_params & params, + const spec_bench_options & opts, + const spec_bench_task & task, + const spec_bench_attempt_result & result, + int repeat_index) { + const bool is_baseline = !params.speculative.has_stage_chain(); + const double decode_tps = result.decode_s > 0.0 ? result.generated_tokens / result.decode_s : 0.0; + const double total_tps = result.total_s > 0.0 ? result.generated_tokens / result.total_s : 0.0; + return json{ + {"row_type", "attempt"}, + {"task_id", task.id}, + {"task_name", task.name}, + {"task_category", task.category}, + {"max_tokens", spec_bench_resolve_max_tokens(task, params)}, + {"repeat_index", repeat_index}, + {"builtin", task.builtin}, + {"prompts", opts.prompts_path.empty() ? "builtin-default" : opts.prompts_path}, + {"runtime", spec_bench_runtime_json(params)}, + {"variant", {{"is_baseline", is_baseline}, {"spec_types", spec_bench_stage_types_json(params.speculative)}, {"stage_chain", common_speculative_stage_chain_to_str(params.speculative)}}}, + {"sampler", spec_bench_sampler_json(params)}, + {"timing", {{"prompt_s", result.prompt_s}, {"decode_s", result.decode_s}, {"total_s", result.total_s}, {"decode_tps", decode_tps}, {"overall_tps", total_tps}}}, + {"tokens", {{"prompt", result.prompt_tokens}, {"generated", result.generated_tokens}}}, + {"speculative", spec_bench_metrics_json(result.spec_delta)}, + {"quality", {{"ok", result.ok}, {"error", result.error.empty() ? json(nullptr) : json(result.error)}, {"retries_used", result.retries_used}, {"hit_eog", result.hit_eog}}}, + {"prompt", result.effective_prompt}, + {"output", result.output_text}, + }; +} + +static json spec_bench_summary_json( + const gpt_params & params, + const spec_bench_options & opts, + const std::vector & tasks, + const spec_bench_summary & summary) { + const bool is_baseline = !params.speculative.has_stage_chain(); + const double decode_tps = summary.decode_s > 0.0 ? summary.generated_tokens / summary.decode_s : 0.0; + const double total_tps = summary.total_s > 0.0 ? summary.generated_tokens / summary.total_s : 0.0; + return json{ + {"row_type", "summary"}, + {"prompts", opts.prompts_path.empty() ? "builtin-default" : opts.prompts_path}, + {"requested_tasks", json(opts.task_names)}, + {"selected_tasks", spec_bench_task_names_json(tasks)}, + {"repeat", opts.repeat}, + {"retry", opts.retry}, + {"default_max_tokens", params.n_predict > 0 ? params.n_predict : 256}, + {"runtime", spec_bench_runtime_json(params)}, + {"variant", {{"is_baseline", is_baseline}, {"spec_types", spec_bench_stage_types_json(params.speculative)}, {"stage_chain", common_speculative_stage_chain_to_str(params.speculative)}}}, + {"sampler", spec_bench_sampler_json(params)}, + {"attempts", summary.attempts}, {"successes", summary.successes}, {"failures", summary.failures}, {"retries_used", summary.retries_used}, + {"timing", {{"prompt_s", summary.prompt_s}, {"decode_s", summary.decode_s}, {"total_s", summary.total_s}, {"decode_tps", decode_tps}, {"overall_tps", total_tps}}}, + {"tokens", {{"prompt", summary.prompt_tokens}, {"generated", summary.generated_tokens}}}, + {"speculative", spec_bench_metrics_json(summary.spec_delta)}, + }; +} + +static json spec_bench_compact_attempt_json(const spec_bench_task & task, const spec_bench_attempt_result & result, int repeat_index) { + json stages = json::array(); + for (const auto & stage : result.spec_delta.stages) { + stages.push_back({ + {"type", common_speculative_type_to_str(stage.type)}, + {"drafts", stage.num_drafts}, {"draft_tokens", stage.draft_tokens}, {"accepted", stage.accepted_tokens}, + {"accept_percent", 100.0 * spec_bench_acceptance_rate(stage.accepted_tokens, stage.draft_tokens)}, + {"accept_length", spec_bench_acceptance_length(stage.accepted_tokens, stage.num_drafts)}, + {"drafted_by_position", spec_bench_position_array(stage.drafted_by_position)}, + {"accepted_by_position", spec_bench_position_array(stage.accepted_by_position)}, + }); + } + return json{{"row_type", "attempt"}, {"task", task.name}, {"run", repeat_index + 1}, {"ok", result.ok}, {"stop", !result.ok ? "fail" : result.hit_eog ? "eog" : "limit"}, {"generated", result.generated_tokens}, {"decode_s", result.decode_s}, {"decode_tps", result.decode_s > 0.0 ? result.generated_tokens / result.decode_s : 0.0}, {"stages", stages}, {"error", result.error.empty() ? json(nullptr) : json(result.error)}}; +} + +static json spec_bench_compact_summary_json(const spec_bench_summary & summary) { + return json{{"row_type", "summary"}, {"attempts", summary.attempts}, {"successes", summary.successes}, {"failures", summary.failures}, {"generated", summary.generated_tokens}, {"decode_s", summary.decode_s}, {"decode_tps", summary.decode_s > 0.0 ? summary.generated_tokens / summary.decode_s : 0.0}, {"speculative", spec_bench_metrics_json(summary.spec_delta)}}; +} + +static void spec_bench_print_markdown(const spec_bench_options & opts, const std::vector & records) { + auto number = [](double value, int precision) { + std::ostringstream out; out << std::fixed << std::setprecision(precision) << value; return out.str(); + }; + auto fit = [](const std::string & value, size_t width) { return value.size() <= width ? value : value.substr(0, width - 3) + "..."; }; + constexpr int stage_width = 14; + auto error_text = [&](const std::string & value) { + std::string result = value; + for (char & ch : result) { if (ch == 10 || ch == 13 || ch == 9) { ch = 32; } } + return fit(string_strip(result), 120); + }; + size_t max_backticks = 3; + auto inspect_fence = [&](const std::string & text) { + size_t run = 0; + for (const char ch : text) { + if (ch == char(96)) { + ++run; + max_backticks = std::max(max_backticks, run); + } else { + run = 0; + } + } + }; + for (const auto & record : records) { + inspect_fence(record.result.effective_prompt); + inspect_fence(record.result.output_text); + } + const std::string fence(max_backticks + 1, char(96)); + + if (opts.output_details) { + std::cout << "\n## Prompt and response details\n\n"; + for (const auto & record : records) { + std::cout << "### " << record.task.name << " / run " << (record.repeat_index + 1) << "\n\n"; + std::cout << "Prompt:\n" << fence << "\n" << record.result.effective_prompt << "\n" << fence << "\n\n"; + std::cout << "Response:\n" << fence << "\n" << record.result.output_text << "\n" << fence << "\n\n"; + } + } + + std::cout << "\n| " << std::left << std::setw(8) << "task" << " | " << std::right << std::setw(3) << "run" + << " | " << std::left << std::setw(stage_width) << "stage" << " | " << std::right << std::setw(7) << "tokens" + << " | " << std::left << std::setw(5) << "stop" << " | " << std::right << std::setw(7) << "time(s)" + << " | " << std::setw(7) << "tok/s" << " | " << std::setw(6) << "rounds" + << " | " << std::setw(11) << "accepted" << " | " << std::setw(7) << "rate" + << " | " << std::setw(6) << "a.len" << " | " << std::left << std::setw(28) << "pos accept" << " |\n"; + std::cout << "|----------|" << std::string(stage_width + 2, '-') << "|---------|-------|---------|---------|--------|-------------|---------|--------|------------------------------|\n"; + + auto position_percentages = [&](const spec_bench_stage_delta & stage) { + std::ostringstream out; + const size_t count = std::max(stage.drafted_by_position.size(), stage.accepted_by_position.size()); + for (size_t i = 0; i < count; ++i) { + if (i > 0) { out << " "; } + const uint64_t drafted = i < stage.drafted_by_position.size() ? stage.drafted_by_position[i] : 0; + const uint64_t accepted = i < stage.accepted_by_position.size() ? stage.accepted_by_position[i] : 0; + if (drafted == 0) { out << "-"; } + else { out << std::fixed << std::setprecision(1) << (100.0 * accepted / drafted) << "%"; } + } + return out.str(); + }; + + for (const auto & record : records) { + const auto & result = record.result; + const int run = record.repeat_index + 1; + const std::string stop = !result.ok ? "fail" : result.hit_eog ? "eog" : "limit"; + const std::string tokens = result.ok ? std::to_string(result.generated_tokens) : "-"; + const double tps = result.decode_s > 0.0 ? result.generated_tokens / result.decode_s : 0.0; + auto row = [&](const std::string & stage_name, uint64_t rounds, uint64_t drafted, uint64_t accepted, const std::string & positions) { + const bool has_metrics = drafted > 0 || rounds > 0; + std::cout << "| " << std::left << std::setw(8) << fit(record.task.name, 8) << " | " << std::right << std::setw(3) << run + << " | " << std::left << std::setw(stage_width) << fit(stage_name, stage_width) << " | " << std::right << std::setw(7) << fit(tokens, 7) + << " | " << std::left << std::setw(5) << stop << " | " << std::right << std::setw(7) << (result.ok ? number(result.decode_s, 3) : "-") + << " | " << std::setw(7) << (result.ok ? number(tps, 2) : "-") << " | " << std::setw(6) << (has_metrics ? std::to_string(rounds) : "-") + << " | " << std::setw(11) << (has_metrics ? std::to_string(accepted) + "/" + std::to_string(drafted) : "-") + << " | " << std::setw(7) << (has_metrics ? number(100.0 * spec_bench_acceptance_rate(accepted, drafted), 2) + "%" : "-") + << " | " << std::setw(6) << (has_metrics ? number(spec_bench_acceptance_length(accepted, rounds), 2) : "-") + << " | " << std::left << std::setw(28) << fit(positions.empty() ? "-" : positions, 28) << " |\n"; + }; + if (!result.ok && result.spec_delta.stages.empty()) { row("error", 0, 0, 0, ""); } + else if (result.spec_delta.stages.empty()) { row("base", 0, 0, 0, ""); } + else { + for (const auto & stage : result.spec_delta.stages) { + row(common_speculative_type_to_str(stage.type), stage.num_drafts, stage.draft_tokens, stage.accepted_tokens, position_percentages(stage)); + } + } + } + std::cout << "\n"; + + if (opts.output_details) { + size_t raw_position_width = 30; + for (const auto & record : records) { + for (const auto & stage : record.result.spec_delta.stages) { + raw_position_width = std::max(raw_position_width, spec_bench_positions(stage).size()); + } + } + std::cout << "## Detailed metrics\n\n"; + std::cout << "| " << std::left << std::setw(8) << "task" << " | " << std::right << std::setw(3) << "run" + << " | " << std::setw(10) << "prompt tok" << " | " << std::setw(9) << "prompt s" + << " | " << std::setw(9) << "total s" << " | " << std::left << std::setw(stage_width) << "stage" + << " | " << std::right << std::setw(9) << "draft s" << " | " << std::setw(9) << "accept s" + << " | " << std::left << std::setw(raw_position_width) << "accepted/drafted by position" << " |\n"; + std::cout << "|----------|-----|------------|-----------|-----------|" << std::string(stage_width + 2, '-') << "|-----------|-----------|" + << std::string(raw_position_width + 2, '-') << "|\n"; + for (const auto & record : records) { + const auto & result = record.result; + auto metric_row = [&](const std::string & stage_name, double draft_s, double accept_s, const std::string & positions) { + std::cout << "| " << std::left << std::setw(8) << fit(record.task.name, 8) << " | " << std::right << std::setw(3) << (record.repeat_index + 1) + << " | " << std::setw(10) << result.prompt_tokens << " | " << std::setw(9) << number(result.prompt_s, 3) + << " | " << std::setw(9) << number(result.total_s, 3) << " | " << std::left << std::setw(stage_width) << fit(stage_name, stage_width) + << " | " << std::right << std::setw(9) << number(draft_s, 6) << " | " << std::setw(9) << number(accept_s, 6) + << " | " << std::left << std::setw(raw_position_width) << (positions.empty() ? "-" : positions) << " |\n"; + }; + if (result.spec_delta.stages.empty()) { metric_row("base", 0.0, 0.0, ""); } + else { + for (const auto & stage : result.spec_delta.stages) { + metric_row(common_speculative_type_to_str(stage.type), stage.t_draft_us / 1e6, stage.t_accept_us / 1e6, spec_bench_positions(stage)); + } + } + } + std::cout << "\n"; + } + + if (opts.repeat > 1) { + struct repeat_group { std::string task; std::string stage; std::vector speed; std::vector rate; std::vector length; }; + std::vector groups; + auto get_group = [&](const std::string & task, const std::string & stage) -> repeat_group & { + for (auto & group : groups) { if (group.task == task && group.stage == stage) { return group; } } + groups.push_back({task, stage, {}, {}, {}}); return groups.back(); + }; + for (const auto & record : records) { + if (!record.result.ok) { continue; } + const double speed = record.result.decode_s > 0.0 ? record.result.generated_tokens / record.result.decode_s : 0.0; + if (record.result.spec_delta.stages.empty()) { get_group(record.task.name, "base").speed.push_back(speed); continue; } + for (const auto & stage : record.result.spec_delta.stages) { + auto & group = get_group(record.task.name, common_speculative_type_to_str(stage.type)); + group.speed.push_back(speed); + if (stage.num_drafts > 0 || stage.draft_tokens > 0) { + group.rate.push_back(spec_bench_acceptance_rate(stage.accepted_tokens, stage.draft_tokens)); + group.length.push_back(spec_bench_acceptance_length(stage.accepted_tokens, stage.num_drafts)); + } + } + } + auto mean_std = [](const std::vector & values) { + if (values.empty()) { return std::pair{0.0, 0.0}; } + double mean = 0.0; for (double value : values) { mean += value; } mean /= values.size(); + double variance = 0.0; for (double value : values) { const double delta = value - mean; variance += delta * delta; } + return std::pair{mean, std::sqrt(variance / values.size())}; + }; + std::cout << "Repeat summary (" << opts.repeat << " runs/task)\n\n"; + std::cout << "| " << std::left << std::setw(8) << "task" << " | " << std::setw(stage_width) << "stage" << " | " << std::right << std::setw(4) << "runs" + << " | " << std::setw(8) << "metric n" + << " | " << std::setw(15) << "tok/s mean/std" << " | " << std::setw(15) << "rate mean/std" << " | " << std::setw(15) << "a.len mean/std" << " |\n"; + std::cout << "|----------|" << std::string(stage_width + 2, '-') << "|------|----------|-----------------|-----------------|-----------------|\n"; + for (const auto & group : groups) { + const auto speed = mean_std(group.speed); const auto rate = mean_std(group.rate); const auto length = mean_std(group.length); + std::cout << "| " << std::left << std::setw(8) << fit(group.task, 8) << " | " << std::setw(stage_width) << fit(group.stage, stage_width) + << " | " << std::right << std::setw(4) << group.speed.size() << " | " << std::setw(8) << group.rate.size() + << " | " << std::setw(15) << number(speed.first, 2) + "/" + number(speed.second, 2) + << " | " << std::setw(15) << (group.rate.empty() ? "-" : number(100.0 * rate.first, 2) + "%/" + number(100.0 * rate.second, 2) + "%") + << " | " << std::setw(15) << (group.length.empty() ? "-" : number(length.first, 2) + "/" + number(length.second, 2)) << " |\n"; + } + std::cout << "\n"; + } + + bool printed_errors = false; + for (const auto & record : records) { + if (!record.result.ok) { + if (!printed_errors) { std::cout << "Errors:\n"; printed_errors = true; } + std::cout << "- " << record.task.name << " run " << (record.repeat_index + 1) << ": " << error_text(record.result.error) << "\n"; + } + } +} + +static bool spec_bench_prepare_spec( + gpt_params & params, + llama_model * model, + llama_context * ctx, + common_speculative ** out_spec) { + const bool requested_spec_user = params.speculative.has_stage_chain(); + if (!common_speculative_finalize_startup(params, model)) { + return false; + } + + const bool requested_spec = params.speculative.has_stage_chain(); + if (requested_spec_user && !requested_spec) { + LOG_TEE("%s: speculative decoding was requested but is not runnable with this finalized configuration\n", __func__); + return false; + } + + if (!requested_spec) { + *out_spec = nullptr; + return true; + } + + if (params.sparams.cfg_scale > 1.f || params.grp_attn_n != 1 || llama_model_has_encoder(model)) { + LOG_TEE("%s: this benchmark only supports direct non-CFG decoder-only speculative runs\n", __func__); + return false; + } + + if (!common_speculative_is_compat(ctx)) { + LOG_TEE("%s: speculative decoding is not supported by this context\n", __func__); + return false; + } + + switch (common_speculative_try_init(params.speculative, ctx, out_spec)) { + case COMMON_SPECULATIVE_INIT_READY: + return true; + case COMMON_SPECULATIVE_INIT_SKIPPED: + *out_spec = nullptr; + return true; + case COMMON_SPECULATIVE_INIT_ERR_RECURRENT: + LOG_TEE("%s: recurrent speculative context initialization failure\n", __func__); + return false; + case COMMON_SPECULATIVE_INIT_ERR_MTP: + LOG_TEE("%s: MTP speculative context initialization failure\n", __func__); + return false; + case COMMON_SPECULATIVE_INIT_ERR_GENERIC: + LOG_TEE("%s: speculative context initialization failure\n", __func__); + return false; + } + + return false; +} + +int main(int argc, char ** argv) { + spec_bench_options bench_opts; + std::vector passthrough; + if (!spec_bench_parse_args(argc, argv, bench_opts, passthrough)) { + spec_bench_print_usage(argv[0]); + return 1; + } + + auto argv_storage = spec_bench_make_argv(passthrough); + + + gpt_params params; + if (!gpt_params_parse((int) argv_storage.size(), argv_storage.data(), params)) { + spec_bench_print_usage(argv[0]); + gpt_params_print_usage((int) argv_storage.size(), argv_storage.data(), params); + return 1; + } + + common_speculative_prepare_startup(params); + + if (params.seed == LLAMA_DEFAULT_SEED) { + params.seed = 1234; + } + + std::vector tasks; + try { + tasks = spec_bench_select_tasks(bench_opts, params); + } catch (const std::exception & e) { + LOG_TEE("%s\n", e.what()); + return 1; + } + + std::ostream * out = &std::cout; + + llama_backend_init(); + llama_numa_init(params.numa); + + llama_init_result init = llama_init_from_gpt_params(params); + llama_model * model = init.model; + llama_context * ctx = init.context; + common_speculative * spec = nullptr; + common_sampler * sampler = nullptr; + + if (model == nullptr || ctx == nullptr) { + LOG_TEE("%s: failed to load model/context\n", __func__); + params.speculative.clear_dft(); + llama_backend_free(); + return 1; + } + + if (!spec_bench_prepare_spec(params, model, ctx, &spec)) { + params.speculative.clear_dft(); + llama_free(ctx); + llama_free_model(model); + llama_backend_free(); + return 1; + } + + sampler = common_sampler_init(model, params.sparams); + if (sampler == nullptr) { + LOG_TEE("%s: failed to initialize sampler\n", __func__); + if (spec != nullptr) { + common_speculative_free(spec); + } + params.speculative.clear_dft(); + llama_free(ctx); + llama_free_model(model); + llama_backend_free(); + return 1; + } + + spec_bench_summary summary; + std::vector records; + for (const auto & task : tasks) { + for (int repeat_index = 0; repeat_index < bench_opts.repeat; ++repeat_index) { + spec_bench_attempt_result best_result; + bool success = false; + + for (int attempt = 0; attempt <= bench_opts.retry; ++attempt) { + spec_bench_attempt_result run = spec_bench_run_attempt(task, params, model, ctx, spec, sampler); + run.retries_used = attempt; + best_result = run; + if (run.ok) { + success = true; + break; + } + } + + best_result.retries_used = success ? best_result.retries_used : bench_opts.retry; + spec_bench_accumulate(summary, best_result); + records.push_back({task, best_result, repeat_index}); + if (bench_opts.output_format == "jsonl") { + *out << (bench_opts.output_details + ? spec_bench_attempt_json(params, bench_opts, task, best_result, repeat_index) + : spec_bench_compact_attempt_json(task, best_result, repeat_index)).dump() << '\n'; + } + } + } + + if (bench_opts.output_format == "md") { + spec_bench_print_markdown(bench_opts, records); + } else { + *out << (bench_opts.output_details + ? spec_bench_summary_json(params, bench_opts, tasks, summary) + : spec_bench_compact_summary_json(summary)).dump() << '\n'; + } + out->flush(); + + common_sampler_free(sampler); + if (spec != nullptr) { + common_speculative_free(spec); + } + params.speculative.clear_dft(); + llama_free(ctx); + llama_free_model(model); + llama_backend_free(); + + return summary.failures == 0 ? 0 : 2; +}