Feat speculative benchmark standard (#2156)

* feat: add initial speculative benchmark logic

* feat: enhance speculative benchmark with position tracking and JSONL prompt support

* feat: enhance speculative benchmark with batch processing and parameter limiting

* Refactor spec-bench to support Markdown output and new prompt files

* spec-bench: finalize interface inputs and reports

* spec-bench: finish report cleanup

* spec-bench: remove unused code

* spec-bench: improve docs and output details for metrics clarity

common/speculative.cpp

@@ -159,6 +159,10 @@ struct common_speculative_state {
     size_t n_gen_tokens = 0; // number of tokens generated by this implementation.
     size_t n_acc_tokens = 0; // number of tokens accepted by the target model.
 
+    // Position zero represents speculative position 1.
+    std::vector<uint64_t> drafted_by_position;
+    std::vector<uint64_t> accepted_by_position;
+
     // TODO: track performance of most recent calls
     const bool gen_perf = true; // whether to generate performance stats.
 
@@ -1577,6 +1581,11 @@ llama_tokens common_speculative_draft(
         spec->curr_impl = impl.get();
         impl->n_gen_drafts++;
         impl->n_gen_tokens += result.size();
+        impl->drafted_by_position.resize(std::max(impl->drafted_by_position.size(), result.size()));
+        impl->accepted_by_position.resize(impl->drafted_by_position.size());
+        for (size_t i = 0; i < result.size(); ++i) {
+            impl->drafted_by_position[i]++;
+        }
 
         break; // We have a draft, so break out of the loop and return it.
     }
@@ -1610,6 +1619,12 @@ void common_speculative_accept(common_speculative * spec, uint16_t n_accepted) {
         if (n_accepted > 0) {
             impl->n_acc_drafts++;
             impl->n_acc_tokens += n_accepted;
+
+            const size_t n_accepted_positions = std::min<size_t>(n_accepted, impl->drafted_by_position.size());
+            impl->accepted_by_position.resize(std::max(impl->accepted_by_position.size(), n_accepted_positions));
+            for (size_t i = 0; i < n_accepted_positions; ++i) {
+                impl->accepted_by_position[i]++;
+            }
         }
 
         impl->accept(n_accepted);
@@ -2083,7 +2098,12 @@ int32_t common_speculative_on_target_seq_batch(
         return 0;
     }
 
-    if (!common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_DFLASH)) {
+    // Self-speculative stages do not consume target hidden-state features.
+    if (!common_speculative_has_target_features(spec)) {
+        return 0;
+    }
+
+    if (common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_MTP)) {
         llama_context * ctx_mtp = common_speculative_get_companion_ctx(spec);
         ctx_mtp = ctx_mtp ? ctx_mtp : ctx_tgt;
         if (ctx_mtp == nullptr) {
@@ -2917,6 +2937,34 @@ common_speculative_type common_speculative_current_type(const common_speculative
     return spec->curr_impl->type;
 }
 
+common_speculative_metrics_snapshot common_speculative_get_metrics_snapshot(const common_speculative * spec) {
+    common_speculative_metrics_snapshot snapshot;
+    if (spec == nullptr) {
+        return snapshot;
+    }
+
+    snapshot.stages.reserve(spec->impls.size());
+    for (const auto & impl : spec->impls) {
+        common_speculative_metrics_stage_snapshot stage;
+        stage.type = impl->type;
+        stage.n_call_begin = impl->n_call_begin;
+        stage.n_call_draft = impl->n_call_draft;
+        stage.n_call_accept = impl->n_call_accept;
+        stage.n_gen_drafts = impl->n_gen_drafts;
+        stage.n_acc_drafts = impl->n_acc_drafts;
+        stage.n_gen_tokens = impl->n_gen_tokens;
+        stage.n_acc_tokens = impl->n_acc_tokens;
+        stage.drafted_by_position = impl->drafted_by_position;
+        stage.accepted_by_position = impl->accepted_by_position;
+        stage.t_begin_us = impl->t_begin_us;
+        stage.t_draft_us = impl->t_draft_us;
+        stage.t_accept_us = impl->t_accept_us;
+        snapshot.stages.push_back(std::move(stage));
+    }
+
+    return snapshot;
+}
+
 void common_speculative_context_shift(
         common_speculative * spec,
         llama_seq_id         seq_id,
@@ -3101,3 +3149,153 @@ int32_t mtp_update_kv_cache(struct llama_context * ctx, const llama_batch& batch
     llama_set_mtp_op_type(ctx, MTP_OP_NONE);
     return ret;
 }
+common_speculative_round_result common_speculative_run_round(
+        common_speculative * spec,
+        llama_model * model,
+        llama_context * ctx,
+        common_sampler * sampler,
+        llama_context * ctx_guidance,
+        common_params_speculative params,
+        const common_params_sampling & sparams,
+        llama_seq_id seq_id,
+        llama_pos n_past,
+        int n_predict_budget,
+        bool have_carry,
+        const llama_tokens & draft_history,
+        llama_token carry_token) {
+    common_speculative_round_result result;
+
+    if (spec == nullptr || n_predict_budget == 1) {
+        return result;
+    }
+
+    const int n_ctx = llama_n_ctx(ctx);
+    const int n_batch = llama_n_batch(ctx);
+    int max_usable_draft = params.get_max_stage_n_max();
+    if (max_usable_draft <= 0) {
+        max_usable_draft = params.n_max;
+    }
+    const int configured_n_max = common_speculative_get_configured_n_max(spec);
+    if (configured_n_max > 0) {
+        max_usable_draft = std::min(max_usable_draft, configured_n_max);
+    }
+    if (n_predict_budget >= 0) {
+        max_usable_draft = std::min(max_usable_draft, n_predict_budget - 2);
+    }
+    max_usable_draft = std::min(max_usable_draft, n_ctx - (int) n_past - 2);
+    max_usable_draft = std::min(max_usable_draft, n_batch - 1);
+
+    // A normal speculative round needs room for the sampled token, at least one
+    // draft position, and the verification carry.
+    if (max_usable_draft <= 0) {
+        return result;
+    }
+
+    params.n_max = std::max(0, max_usable_draft);
+    params.n_min = std::min(std::max(0, params.n_min), params.n_max);
+    for (auto & stage : params.stages) {
+        if (stage.has_n_max_override()) {
+            stage.n_max = std::min(stage.n_max, params.n_max);
+        }
+        if (stage.has_n_min_override()) {
+            const int stage_max = stage.has_n_max_override() ? stage.n_max : params.n_max;
+            stage.n_min = std::min(stage.n_min, stage_max);
+        }
+    }
+
+    result.attempted = true;
+    result.sampled_before_from_carry = have_carry;
+    if (have_carry) {
+        result.sampled_before = carry_token;
+    } else {
+        result.sampled_before = common_sampler_sample_legacy(sampler, ctx, ctx_guidance);
+        common_sampler_accept(sampler, ctx, result.sampled_before, true);
+    }
+    result.sampled_before_ready = true;
+
+    auto draft_result = common_speculative_draft_ex(
+        spec,
+        ctx,
+        params,
+        draft_history,
+        result.sampled_before,
+        n_past,
+        seq_id);
+    auto & draft = draft_result.tokens;
+
+    const int min_usable_draft = params.get_min_usable_stage_n_min();
+    if ((int) draft.size() < min_usable_draft || (draft.empty() && !draft_result.target_only)) {
+        return result;
+    }
+
+    if (llama_model_has_recurrent(model) || llama_model_is_openpangu(model)) {
+        if (!common_speculative_before_draft(
+                spec,
+                model,
+                ctx,
+                sampler,
+                sparams,
+                seq_id,
+                n_past,
+                result.sampled_before,
+                (int) draft.size() + 1,
+                params.recurrent_ckpt_mode)) {
+            draft.clear();
+        }
+    }
+
+    if (draft.empty() && !draft_result.target_only) {
+        return result;
+    }
+
+    llama_batch verify_batch = llama_batch_init((int) draft.size() + 1, 0, 1);
+    std::vector<int> verify_indices;
+    verify_indices.reserve(draft.size() + 1);
+
+    common_batch_add(verify_batch, result.sampled_before, n_past, { seq_id }, true);
+    verify_indices.push_back(0);
+    for (size_t i = 0; i < draft.size(); ++i) {
+        common_batch_add(verify_batch, draft[i], n_past + 1 + (llama_pos) i, { seq_id }, true);
+        verify_indices.push_back((int) i + 1);
+    }
+
+    if (llama_decode(ctx, verify_batch) != 0) {
+        llama_batch_free(verify_batch);
+        result.failed = true;
+        result.error = "speculative verify decode failed";
+        return result;
+    }
+
+    std::vector<llama_token> ids;
+    try {
+        ids = common_sampler_sample_and_accept_n(sampler, ctx, verify_indices, draft);
+    } catch (const std::exception & e) {
+        llama_batch_free(verify_batch);
+        result.failed = true;
+        result.error = e.what();
+        return result;
+    }
+
+    std::vector<int32_t> accepted_output_indices;
+    if (!ids.empty()) {
+        accepted_output_indices.assign(verify_indices.begin(), verify_indices.begin() + ids.size());
+    }
+
+    if (!ids.empty()) {
+        common_speculative_commit(
+            spec,
+            ctx,
+            sampler,
+            seq_id,
+            result.sampled_before,
+            ids,
+            (int) draft.size(),
+            n_past + 1,
+            accepted_output_indices);
+        result.ids = std::move(ids);
+        result.used_speculative = true;
+    }
+
+    llama_batch_free(verify_batch);
+    return result;
+}

common/speculative.h

@@ -40,6 +40,31 @@ struct common_speculative_draft_result {
     bool target_only = false;
 };
 
+struct common_speculative_metrics_stage_snapshot {
+    common_speculative_type type = COMMON_SPECULATIVE_TYPE_NONE;
+
+    uint64_t n_call_begin = 0;
+    uint64_t n_call_draft = 0;
+    uint64_t n_call_accept = 0;
+
+    uint64_t n_gen_drafts = 0;
+    uint64_t n_acc_drafts = 0;
+    uint64_t n_gen_tokens = 0;
+    uint64_t n_acc_tokens = 0;
+
+    // Position zero represents speculative position 1.
+    std::vector<uint64_t> drafted_by_position;
+    std::vector<uint64_t> accepted_by_position;
+
+    int64_t t_begin_us = 0;
+    int64_t t_draft_us = 0;
+    int64_t t_accept_us = 0;
+};
+
+struct common_speculative_metrics_snapshot {
+    std::vector<common_speculative_metrics_stage_snapshot> stages;
+};
+
 // comma separated list of all types
 std::string common_speculative_type_name_str();
 
@@ -236,6 +261,8 @@ void common_speculative_print_stats(const common_speculative * spec, double slot
 
 common_speculative_type common_speculative_current_type(const common_speculative * spec);
 
+common_speculative_metrics_snapshot common_speculative_get_metrics_snapshot(const common_speculative * spec);
+
 // Context shift for MTP to match how server handle main model
 void common_speculative_context_shift(
         common_speculative * spec,
@@ -243,3 +270,29 @@ void common_speculative_context_shift(
         llama_pos            kv_keep,
         llama_pos            kv_discard,
         llama_pos            kv_past);
+
+struct common_speculative_round_result {
+    bool attempted = false;
+    bool sampled_before_ready = false;
+    bool sampled_before_from_carry = false;
+    bool used_speculative = false;
+    bool failed = false;
+    std::string error;
+    llama_token sampled_before = LLAMA_TOKEN_NULL;
+    llama_tokens ids;
+};
+
+common_speculative_round_result common_speculative_run_round(
+    common_speculative * spec,
+    llama_model * model,
+    llama_context * ctx,
+    common_sampler * sampler,
+    llama_context * ctx_guidance,
+    common_params_speculative params,
+    const common_params_sampling & sparams,
+    llama_seq_id seq_id,
+    llama_pos n_past,
+    int n_predict_budget,
+    bool have_carry,
+    const llama_tokens & draft_history,
+    llama_token carry_token);

examples/CMakeLists.txt

@@ -50,6 +50,7 @@ else()
     endif()
     add_subdirectory(save-load-state)
     add_subdirectory(simple)
+    add_subdirectory(spec-bench)
     add_subdirectory(speculative)
     add_subdirectory(sweep-bench)
     add_subdirectory(tokenize)

examples/spec-bench/CMakeLists.txt

@@ -0,0 +1,41 @@
+set(TARGET llama-spec-bench)
+set(SPEC_BENCH_PROMPT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/prompts")
+set(SPEC_BENCH_PROMPT_FILES
+    "${SPEC_BENCH_PROMPT_DIR}/code.txt"
+    "${SPEC_BENCH_PROMPT_DIR}/extract.txt"
+    "${SPEC_BENCH_PROMPT_DIR}/story.txt"
+)
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SPEC_BENCH_PROMPT_FILES})
+
+function(spec_bench_escape_cpp_string output_var input_value)
+    string(REPLACE "\\" "\\\\" value "${input_value}")
+    string(REPLACE [=["]=] [=[\"]=] value "${value}")
+    string(REPLACE "\r" "\\r" value "${value}")
+    string(REPLACE "\n" "\\n" value "${value}")
+    set(${output_var} "${value}" PARENT_SCOPE)
+endfunction()
+
+file(READ "${SPEC_BENCH_PROMPT_DIR}/code.txt" SPEC_BENCH_PROMPT_CODE_RAW)
+file(READ "${SPEC_BENCH_PROMPT_DIR}/extract.txt" SPEC_BENCH_PROMPT_EXTRACT_RAW)
+file(READ "${SPEC_BENCH_PROMPT_DIR}/story.txt" SPEC_BENCH_PROMPT_STORY_RAW)
+string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_CODE_RAW "${SPEC_BENCH_PROMPT_CODE_RAW}")
+string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_EXTRACT_RAW "${SPEC_BENCH_PROMPT_EXTRACT_RAW}")
+string(REGEX REPLACE "\n$" "" SPEC_BENCH_PROMPT_STORY_RAW "${SPEC_BENCH_PROMPT_STORY_RAW}")
+string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_CODE_RAW "${SPEC_BENCH_PROMPT_CODE_RAW}")
+string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_EXTRACT_RAW "${SPEC_BENCH_PROMPT_EXTRACT_RAW}")
+string(REGEX REPLACE "\r$" "" SPEC_BENCH_PROMPT_STORY_RAW "${SPEC_BENCH_PROMPT_STORY_RAW}")
+spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_CODE "${SPEC_BENCH_PROMPT_CODE_RAW}")
+spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_EXTRACT "${SPEC_BENCH_PROMPT_EXTRACT_RAW}")
+spec_bench_escape_cpp_string(SPEC_BENCH_PROMPT_STORY "${SPEC_BENCH_PROMPT_STORY_RAW}")
+configure_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/spec-bench-prompts.h.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/spec-bench-prompts.h"
+    @ONLY
+)
+
+
+add_executable(${TARGET} spec-bench.cpp)
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_17)
+target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")

examples/spec-bench/README.md

@@ -0,0 +1,81 @@
+# ik_llama.cpp/examples/spec-bench
+
+`llama-spec-bench` is a direct C++ speculative benchmark for prompt-driven tasks.
+It reuses the normal `llama-common` startup and speculative lifecycle instead of
+benchmarking through `llama-server`.
+
+## Scope
+
+- built-in canonical tasks: `code`, `extract`, `story`
+- all three canonical built-in workloads by default, or one plain custom prompt via `-p` / `-f`
+- optional strict JSONL prompt-file override for structured multi-prompt workloads
+- baseline and speculative runs use the same binary and normal model/sampler args
+- Markdown report by default; compact JSONL is available with `--output-format jsonl`
+- per-stage drafted and accepted counts by speculative position
+
+## Benchmark-specific flags
+
+- `--prompts <path>`: replace the built-in tasks with a strict JSONL prompt file
+- `-p, --prompt <text>`: run one inline custom prompt
+- `-f, --file <path>`: run one plain-text custom prompt file; the file is one prompt, not one task per line
+- `--task <name[,name...]>`: select built-in tasks
+- `--repeat <n>`: repeat each task `n` times
+- `--retry <n>`: retry transient task failures up to `n` times
+- `--output-format jsonl`: select the common JSONL output convention; output is written to `stdout`
+- `--output-details`: print prompts and responses first, followed by normal and detailed Markdown metrics; JSONL includes complete details
+- `--predict <n>` / `-n <n>`: command-level generation budget for every task without a row override
+
+## Input modes
+
+Choose exactly one mode: built-ins (optionally narrowed with `--task`), one `-p` prompt, one `-f` file, or one `--prompts` JSONL dataset.
+Examples:
+
+```bash
+./build/bin/llama-spec-bench -m model.gguf -n 4 -p "Write a merge sort in C++."
+./build/bin/llama-spec-bench -m model.gguf -n 4 -f examples/spec-bench/prompts/code.txt
+```
+
+## Dataset format
+
+Each JSONL line must be an object containing a non-empty `prompt` string. Optional fields are
+`id`, `name`, `category`, and positive integer `max_tokens`:
+
+```json
+{"id":"task-1","name":"math","category":"reasoning","prompt":"Solve 12*17.","max_tokens":64}
+```
+
+IDs default to the one-based input line number and must be unique. Unknown fields,
+duplicate IDs, empty prompts, malformed JSON, and invalid `max_tokens` values are rejected.
+The file replaces the built-in task set for that invocation.
+
+The canonical prompts are embedded into the executable at configure time from `prompts/code.txt`,
+`prompts/extract.txt`, and `prompts/story.txt`; no source-tree or network access is needed at runtime.
+
+Compact JSONL includes raw `drafted_by_position` and `accepted_by_position` arrays for
+every stage. Detailed JSONL additionally includes the derived
+`acceptance_rate_by_position` and `conditional_acceptance_rate` arrays. Array element
+zero is speculative position one; the first conditional rate is `null` because it has
+no preceding position.
+
+Acceptance length is defined consistently as `1 + accepted_tokens / num_drafts` in detailed JSON, compact JSON, Markdown, and repeat summaries.
+
+Repeated attempts are executed in one process. Stateful drafting stages, including
+adaptive n-gram stages and lookup caches, may therefore carry learned state from an
+earlier task or repeat; use `--repeat 1` and separate invocations when independent
+samples are required. Pin the chat-template mode (`--jinja` or `--no-jinja`) when
+comparing runs because it changes the effective prompt. Speculative verification can
+also diverge from a baseline after a near-tie because batched evaluation changes
+floating-point reduction order, so this tool is a performance and acceptance benchmark,
+not a bit-identical output checker.
+
+## Example
+
+```bash
+./build/bin/llama-spec-bench \
+  -m model.gguf \
+  --seed 123 \
+  --temp 0 \
+  --predict 256 \
+  --output-format jsonl \
+  --task code,extract,story > results.jsonl
+```

examples/spec-bench/prompts/code.txt

@@ -0,0 +1 @@
+Write a quick sort python algorithm, answer only the code.

examples/spec-bench/prompts/extract.txt

@@ -0,0 +1,9 @@
+Extract all core events with their exact dates into a bulleted list
+
+YouTube is an American online video sharing platform owned by Google. YouTube was founded on February 14, 2005, by Chad Hurley, Jawed Karim, and Steve Chen, who were former employees of PayPal. Headquartered in San Bruno, California, it is the second-most-visited website in the world, after Google itself. In January 2024, YouTube had more than 2.7 billion monthly active users, who collectively consumed more than one billion hours of video content every day. As of May 2019, videos were being uploaded to the platform at a rate of more than 500 hours of content per minute, and as of mid-2024, there were approximately 14.8 billion videos in total.
+
+On November 13, 2006, YouTube was purchased by Google for US$1.65 billion (equivalent to $2.44 billion in 2025). Google expanded YouTube's business model from generating revenue through advertisements alone to offering paid content such as movies and exclusive content explicitly produced for YouTube. It also offers YouTube Premium, a paid subscription option for watching content without ads. YouTube incorporated the Google AdSense program, generating more revenue for both YouTube and approved content creators. In 2023, YouTube's advertising revenue totaled $31.7 billion, a 2% increase from the $31.1 billion reported in 2022. From Q4 2023 to Q3 2024, YouTube's combined revenue from advertising and subscriptions exceeded $50 billion.
+
+Since its purchase by Google, YouTube has expanded beyond the core website, creating mobile apps, network television, games, and the ability to link with other platforms. Video categories on YouTube include music videos, video clips, news, short and feature films, songs, documentaries, movie trailers, teasers, TV spots, live streams, vlogs, and more. Most content is generated by individuals, including collaborations between YouTubers and corporate sponsors. Established media, news, and entertainment corporations have also created and expanded their visibility on YouTube channels to reach bigger audiences.
+
+YouTube has had unprecedented social impact, influencing popular culture, internet trends, and creating multimillionaire celebrities. Despite its growth and success, the platform has been criticized for its facilitation of the spread of misinformation and copyrighted content, routinely violating its users' privacy, excessive censorship, endangering the safety of children and their well-being, and for its inconsistent implementation of platform guidelines.

examples/spec-bench/prompts/story.txt

@@ -0,0 +1 @@
+Give me an extended summary of the history of Bulgaria

examples/spec-bench/spec-bench-prompts.h.in

@@ -0,0 +1,5 @@
+#pragma once
+
+#define SPEC_BENCH_PROMPT_CODE "@SPEC_BENCH_PROMPT_CODE@"
+#define SPEC_BENCH_PROMPT_EXTRACT "@SPEC_BENCH_PROMPT_EXTRACT@"
+#define SPEC_BENCH_PROMPT_STORY "@SPEC_BENCH_PROMPT_STORY@"