mirror of
https://github.com/ikawrakow/ik_llama.cpp.git
synced 2026-08-12 22:29:39 +04:00
Feat speculative benchmark standard (#2156)
* feat: add initial speculative benchmark logic * feat: enhance speculative benchmark with position tracking and JSONL prompt support * feat: enhance speculative benchmark with batch processing and parameter limiting * Refactor spec-bench to support Markdown output and new prompt files * spec-bench: finalize interface inputs and reports * spec-bench: finish report cleanup * spec-bench: remove unused code * spec-bench: improve docs and output details for metrics clarity
This commit is contained in:
Samuel Oliveira Alves
+199
-1
@@ -159,6 +159,10 @@ struct common_speculative_state {
|
||||
size_t n_gen_tokens = 0; // number of tokens generated by this implementation.
|
||||
size_t n_acc_tokens = 0; // number of tokens accepted by the target model.
|
||||
|
||||
// Position zero represents speculative position 1.
|
||||
std::vector<uint64_t> drafted_by_position;
|
||||
std::vector<uint64_t> accepted_by_position;
|
||||
|
||||
// TODO: track performance of most recent calls
|
||||
const bool gen_perf = true; // whether to generate performance stats.
|
||||
|
||||
@@ -1577,6 +1581,11 @@ llama_tokens common_speculative_draft(
|
||||
spec->curr_impl = impl.get();
|
||||
impl->n_gen_drafts++;
|
||||
impl->n_gen_tokens += result.size();
|
||||
impl->drafted_by_position.resize(std::max(impl->drafted_by_position.size(), result.size()));
|
||||
impl->accepted_by_position.resize(impl->drafted_by_position.size());
|
||||
for (size_t i = 0; i < result.size(); ++i) {
|
||||
impl->drafted_by_position[i]++;
|
||||
}
|
||||
|
||||
break; // We have a draft, so break out of the loop and return it.
|
||||
}
|
||||
@@ -1610,6 +1619,12 @@ void common_speculative_accept(common_speculative * spec, uint16_t n_accepted) {
|
||||
if (n_accepted > 0) {
|
||||
impl->n_acc_drafts++;
|
||||
impl->n_acc_tokens += n_accepted;
|
||||
|
||||
const size_t n_accepted_positions = std::min<size_t>(n_accepted, impl->drafted_by_position.size());
|
||||
impl->accepted_by_position.resize(std::max(impl->accepted_by_position.size(), n_accepted_positions));
|
||||
for (size_t i = 0; i < n_accepted_positions; ++i) {
|
||||
impl->accepted_by_position[i]++;
|
||||
}
|
||||
}
|
||||
|
||||
impl->accept(n_accepted);
|
||||
@@ -2083,7 +2098,12 @@ int32_t common_speculative_on_target_seq_batch(
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_DFLASH)) {
|
||||
// Self-speculative stages do not consume target hidden-state features.
|
||||
if (!common_speculative_has_target_features(spec)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (common_speculative_has_type(spec, COMMON_SPECULATIVE_TYPE_MTP)) {
|
||||
llama_context * ctx_mtp = common_speculative_get_companion_ctx(spec);
|
||||
ctx_mtp = ctx_mtp ? ctx_mtp : ctx_tgt;
|
||||
if (ctx_mtp == nullptr) {
|
||||
@@ -2917,6 +2937,34 @@ common_speculative_type common_speculative_current_type(const common_speculative
|
||||
return spec->curr_impl->type;
|
||||
}
|
||||
|
||||
common_speculative_metrics_snapshot common_speculative_get_metrics_snapshot(const common_speculative * spec) {
|
||||
common_speculative_metrics_snapshot snapshot;
|
||||
if (spec == nullptr) {
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
snapshot.stages.reserve(spec->impls.size());
|
||||
for (const auto & impl : spec->impls) {
|
||||
common_speculative_metrics_stage_snapshot stage;
|
||||
stage.type = impl->type;
|
||||
stage.n_call_begin = impl->n_call_begin;
|
||||
stage.n_call_draft = impl->n_call_draft;
|
||||
stage.n_call_accept = impl->n_call_accept;
|
||||
stage.n_gen_drafts = impl->n_gen_drafts;
|
||||
stage.n_acc_drafts = impl->n_acc_drafts;
|
||||
stage.n_gen_tokens = impl->n_gen_tokens;
|
||||
stage.n_acc_tokens = impl->n_acc_tokens;
|
||||
stage.drafted_by_position = impl->drafted_by_position;
|
||||
stage.accepted_by_position = impl->accepted_by_position;
|
||||
stage.t_begin_us = impl->t_begin_us;
|
||||
stage.t_draft_us = impl->t_draft_us;
|
||||
stage.t_accept_us = impl->t_accept_us;
|
||||
snapshot.stages.push_back(std::move(stage));
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
void common_speculative_context_shift(
|
||||
common_speculative * spec,
|
||||
llama_seq_id seq_id,
|
||||
@@ -3101,3 +3149,153 @@ int32_t mtp_update_kv_cache(struct llama_context * ctx, const llama_batch& batch
|
||||
llama_set_mtp_op_type(ctx, MTP_OP_NONE);
|
||||
return ret;
|
||||
}
|
||||
common_speculative_round_result common_speculative_run_round(
|
||||
common_speculative * spec,
|
||||
llama_model * model,
|
||||
llama_context * ctx,
|
||||
common_sampler * sampler,
|
||||
llama_context * ctx_guidance,
|
||||
common_params_speculative params,
|
||||
const common_params_sampling & sparams,
|
||||
llama_seq_id seq_id,
|
||||
llama_pos n_past,
|
||||
int n_predict_budget,
|
||||
bool have_carry,
|
||||
const llama_tokens & draft_history,
|
||||
llama_token carry_token) {
|
||||
common_speculative_round_result result;
|
||||
|
||||
if (spec == nullptr || n_predict_budget == 1) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const int n_ctx = llama_n_ctx(ctx);
|
||||
const int n_batch = llama_n_batch(ctx);
|
||||
int max_usable_draft = params.get_max_stage_n_max();
|
||||
if (max_usable_draft <= 0) {
|
||||
max_usable_draft = params.n_max;
|
||||
}
|
||||
const int configured_n_max = common_speculative_get_configured_n_max(spec);
|
||||
if (configured_n_max > 0) {
|
||||
max_usable_draft = std::min(max_usable_draft, configured_n_max);
|
||||
}
|
||||
if (n_predict_budget >= 0) {
|
||||
max_usable_draft = std::min(max_usable_draft, n_predict_budget - 2);
|
||||
}
|
||||
max_usable_draft = std::min(max_usable_draft, n_ctx - (int) n_past - 2);
|
||||
max_usable_draft = std::min(max_usable_draft, n_batch - 1);
|
||||
|
||||
// A normal speculative round needs room for the sampled token, at least one
|
||||
// draft position, and the verification carry.
|
||||
if (max_usable_draft <= 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
params.n_max = std::max(0, max_usable_draft);
|
||||
params.n_min = std::min(std::max(0, params.n_min), params.n_max);
|
||||
for (auto & stage : params.stages) {
|
||||
if (stage.has_n_max_override()) {
|
||||
stage.n_max = std::min(stage.n_max, params.n_max);
|
||||
}
|
||||
if (stage.has_n_min_override()) {
|
||||
const int stage_max = stage.has_n_max_override() ? stage.n_max : params.n_max;
|
||||
stage.n_min = std::min(stage.n_min, stage_max);
|
||||
}
|
||||
}
|
||||
|
||||
result.attempted = true;
|
||||
result.sampled_before_from_carry = have_carry;
|
||||
if (have_carry) {
|
||||
result.sampled_before = carry_token;
|
||||
} else {
|
||||
result.sampled_before = common_sampler_sample_legacy(sampler, ctx, ctx_guidance);
|
||||
common_sampler_accept(sampler, ctx, result.sampled_before, true);
|
||||
}
|
||||
result.sampled_before_ready = true;
|
||||
|
||||
auto draft_result = common_speculative_draft_ex(
|
||||
spec,
|
||||
ctx,
|
||||
params,
|
||||
draft_history,
|
||||
result.sampled_before,
|
||||
n_past,
|
||||
seq_id);
|
||||
auto & draft = draft_result.tokens;
|
||||
|
||||
const int min_usable_draft = params.get_min_usable_stage_n_min();
|
||||
if ((int) draft.size() < min_usable_draft || (draft.empty() && !draft_result.target_only)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (llama_model_has_recurrent(model) || llama_model_is_openpangu(model)) {
|
||||
if (!common_speculative_before_draft(
|
||||
spec,
|
||||
model,
|
||||
ctx,
|
||||
sampler,
|
||||
sparams,
|
||||
seq_id,
|
||||
n_past,
|
||||
result.sampled_before,
|
||||
(int) draft.size() + 1,
|
||||
params.recurrent_ckpt_mode)) {
|
||||
draft.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (draft.empty() && !draft_result.target_only) {
|
||||
return result;
|
||||
}
|
||||
|
||||
llama_batch verify_batch = llama_batch_init((int) draft.size() + 1, 0, 1);
|
||||
std::vector<int> verify_indices;
|
||||
verify_indices.reserve(draft.size() + 1);
|
||||
|
||||
common_batch_add(verify_batch, result.sampled_before, n_past, { seq_id }, true);
|
||||
verify_indices.push_back(0);
|
||||
for (size_t i = 0; i < draft.size(); ++i) {
|
||||
common_batch_add(verify_batch, draft[i], n_past + 1 + (llama_pos) i, { seq_id }, true);
|
||||
verify_indices.push_back((int) i + 1);
|
||||
}
|
||||
|
||||
if (llama_decode(ctx, verify_batch) != 0) {
|
||||
llama_batch_free(verify_batch);
|
||||
result.failed = true;
|
||||
result.error = "speculative verify decode failed";
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<llama_token> ids;
|
||||
try {
|
||||
ids = common_sampler_sample_and_accept_n(sampler, ctx, verify_indices, draft);
|
||||
} catch (const std::exception & e) {
|
||||
llama_batch_free(verify_batch);
|
||||
result.failed = true;
|
||||
result.error = e.what();
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<int32_t> accepted_output_indices;
|
||||
if (!ids.empty()) {
|
||||
accepted_output_indices.assign(verify_indices.begin(), verify_indices.begin() + ids.size());
|
||||
}
|
||||
|
||||
if (!ids.empty()) {
|
||||
common_speculative_commit(
|
||||
spec,
|
||||
ctx,
|
||||
sampler,
|
||||
seq_id,
|
||||
result.sampled_before,
|
||||
ids,
|
||||
(int) draft.size(),
|
||||
n_past + 1,
|
||||
accepted_output_indices);
|
||||
result.ids = std::move(ids);
|
||||
result.used_speculative = true;
|
||||
}
|
||||
|
||||
llama_batch_free(verify_batch);
|
||||
return result;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user