diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 72e6f26ed1..1c3de6739f 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -432,7 +432,8 @@ struct server_metrics { struct bucket { uint64_t count = 0; // number of tokens - uint64_t steps = 0; // for generation, this excludes first generated token (logits from prompt batch) + uint64_t steps = 0; // number of decode steps, + // this excludes first generated token (logits from prompt batch) uint64_t time = 0; // in microseconds // the rate uses the decode steps, so that "free" tokens do not inflate it @@ -447,16 +448,15 @@ struct server_metrics { } }; - // prompt tokens reused from the cache need no decode, so they only have a count - - // these are reset by reset_bucket() - bucket prompt_bucket; - bucket predict_bucket; - uint64_t n_prompt_cached_bucket = 0; + // these are reset by reset_bucket(), only the rate is read from them + bucket prompt_bucket; + bucket predict_bucket; // metrics below are cumulative since the server started - bucket prompt; // only processed tokens, not cached tokens - bucket predict; + bucket prompt; // only processed tokens, cached ones are counted separately below + bucket predict; + + // tokens reused from the cache need no decode, so they only have a count uint64_t n_prompt_cached = 0; uint64_t n_tokens_max = 0; @@ -474,9 +474,8 @@ struct server_metrics { } void reset_bucket() { - prompt_bucket = {}; - predict_bucket = {}; - n_prompt_cached_bucket = 0; + prompt_bucket = {}; + predict_bucket = {}; } void add_prompt(uint64_t n_tokens, uint64_t t_us) { @@ -485,10 +484,8 @@ struct server_metrics { } void add_prompt_cached(uint64_t n_tokens) { - n_prompt_cached += n_tokens; - n_prompt_cached_bucket += n_tokens; + n_prompt_cached += n_tokens; } - }; // diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 2facd71bd1..8d6e8f7087 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -320,7 +320,7 @@ struct server_slot { // this is for printing timings with slot progress, not part of metrics int64_t t_print_last = 0; - int32_t n_decoded_last = 0; + int32_t n_gen_last = 0; void reset() { SLT_DBG(*this, "%s", "\n"); @@ -347,6 +347,7 @@ struct server_slot { task_prev = std::move(task); task.reset(); + // note: callback_on_reset() must have run before this, see release() stats = {}; n_accepted_per_pos.clear(); @@ -567,18 +568,18 @@ struct server_slot { } const double n_gen_second = stats.n_gen_tps(); - const double n_gen_second_win = 1e6 / (t_now - t_print_last) * (stats.n_gen - n_decoded_last); + const double n_gen_second_win = 1e6 / (t_now - t_print_last) * (stats.n_gen - n_gen_last); t_print_last = t_now; - n_decoded_last = stats.n_gen; + n_gen_last = stats.n_gen; - SLT_INF(*this, "n_decoded = %6d, tg = %6.2f t/s, tg_3s = %6.2f t/s\n", (int) stats.n_gen, n_gen_second, n_gen_second_win); + SLT_INF(*this, "n_gen = %6d, tg = %6.2f t/s, tg_3s = %6.2f t/s\n", (int) stats.n_gen, n_gen_second, n_gen_second_win); } void print_timings_pp() const { - const double t_prompt_processing = stats.t_prompt_ms(); + const double t_prompt_total = stats.t_prompt_ms(); - if (t_prompt_processing < 3000.0) { + if (t_prompt_total < 3000.0) { return; } @@ -586,12 +587,12 @@ struct server_slot { const double f_progress = task->n_tokens() > 0 ? (double) prompt.n_tokens() / task->n_tokens() : 0.0; SLT_INF(*this, "prompt processing, n_tokens = %6d, progress = %.2f, t = %6.2f s / %.2f tokens per second\n", - (int) stats.n_prompt_processed, f_progress, t_prompt_processing / 1e3, n_prompt_second); + (int) stats.n_prompt_processed, f_progress, t_prompt_total / 1e3, n_prompt_second); } void print_timings() const { - const double t_prompt_processing = stats.t_prompt_ms(); - const double t_token_generation = stats.t_gen_ms(); + const double t_prompt_total = stats.t_prompt_ms(); + const double t_gen_total = stats.t_gen_ms(); const double t_prompt = stats.t_prompt_per_token_ms(); const double n_prompt_second = stats.n_prompt_tps(); @@ -601,15 +602,15 @@ struct server_slot { SLT_INF(*this, "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", - t_prompt_processing, (int) stats.n_prompt_processed, t_prompt, n_prompt_second); + t_prompt_total, (int) stats.n_prompt_processed, t_prompt, n_prompt_second); SLT_INF(*this, " eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", - t_token_generation, (int) stats.n_gen, t_gen, n_gen_second); + t_gen_total, (int) stats.n_gen, t_gen, n_gen_second); SLT_INF(*this, " total time = %10.2f ms / %5d tokens\n", - t_prompt_processing + t_token_generation, (int) (stats.n_prompt_processed + stats.n_gen)); + t_prompt_total + t_gen_total, (int) (stats.n_prompt_processed + stats.n_gen)); SLT_INF(*this, " graphs reused = %10d\n", @@ -1823,7 +1824,7 @@ private: slot.stop = STOP_TYPE_LIMIT; slot.has_next_token = false; - SLT_DBG(slot, "stopped due to running out of context capacity, prompt.n_tokens() = %d, task.n_tokens = %d, n_decoded = %d, n_ctx = %d\n", + SLT_DBG(slot, "stopped due to running out of context capacity, prompt.n_tokens() = %d, task.n_tokens = %d, n_gen = %d, n_ctx = %d\n", slot.prompt.n_tokens(), slot.task->n_tokens(), (int) slot.stats.n_gen, slot.n_ctx); } @@ -1832,7 +1833,7 @@ private: slot.stop = STOP_TYPE_LIMIT; slot.has_next_token = false; - SLT_DBG(slot, "stopped by limit, n_decoded = %d, n_predict = %d\n", (int) slot.stats.n_gen, slot.task->params.n_predict); + SLT_DBG(slot, "stopped by limit, n_gen = %d, n_predict = %d\n", (int) slot.stats.n_gen, slot.task->params.n_predict); } if (slot.has_new_line) { @@ -1856,7 +1857,7 @@ private: // cut the last line slot.generated_text.erase(pos, std::string::npos); - SLT_DBG(slot, "stopped by indentation limit, n_decoded = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent); + SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent); } } @@ -1880,7 +1881,7 @@ private: slot.stop = STOP_TYPE_LIMIT; slot.has_next_token = false; - SLT_DBG(slot, "stopped by time limit, n_decoded = %d, t_max_predict_ms = %d ms\n", (int) slot.stats.n_gen, (int) slot.task->params.t_max_predict_ms); + SLT_DBG(slot, "stopped by time limit, n_gen = %d, t_max_predict_ms = %d ms\n", (int) slot.stats.n_gen, (int) slot.task->params.t_max_predict_ms); } } @@ -1891,7 +1892,7 @@ private: SLT_DBG(slot, "%s", "stopped by EOS\n"); } - SLT_DBG(slot, "n_decoded = %d, n_remaining = %d, next token: %5d '%s'\n", (int) slot.stats.n_gen, slot.n_remaining(), result.tok, token_str.c_str()); + SLT_DBG(slot, "n_gen = %d, n_remaining = %d, next token: %5d '%s'\n", (int) slot.stats.n_gen, slot.n_remaining(), result.tok, token_str.c_str()); return slot.has_next_token; // continue } @@ -3476,7 +3477,7 @@ private: batch.set_output(batch.size() - 1, true); slot.stats.n_gen = 0; - slot.i_batch = batch.size() - 1; + slot.i_batch = batch.size() - 1; slot.init_sampler(); } else { @@ -3726,7 +3727,7 @@ private: if (slot.stats.n_gen == 1) { slot.stats.update_prompt_last(); slot.t_print_last = t_now; - slot.n_decoded_last = 0; + slot.n_gen_last = 0; } slot.stats.update_gen_last(); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 817cb778b8..64afbc5edf 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1522,11 +1522,11 @@ std::string server_task_result_metrics::to_metrics() { { "prompt_tokens_total", "Number of prompt tokens processed, excluding cached tokens", - metrics.prompt.count + (double) metrics.prompt.count }, { "prompt_tokens_cached_total", "Number of prompt tokens reused from the cache", - metrics.n_prompt_cached + (double) metrics.n_prompt_cached }, { "prompt_seconds_total", "Total time spent processing prompts", @@ -1534,7 +1534,7 @@ std::string server_task_result_metrics::to_metrics() { }, { "tokens_predicted_total", "Number of generation tokens processed", - metrics.predict.count + (double) metrics.predict.count }, { "tokens_predicted_seconds_total", "Total time spent generating tokens", @@ -1542,23 +1542,23 @@ std::string server_task_result_metrics::to_metrics() { }, { "n_decode_total", "Total number of llama_decode() calls, excluding speculative decoding and multimodal decoding", - metrics.n_decode + (double) metrics.n_decode }, { "n_tokens_max", "Largest observed sequence length (prompt + generation)", - metrics.n_tokens_max + (double) metrics.n_tokens_max }, { "spec_decode_num_draft_tokens_total", "Speculative: Total draft tokens generated", - metrics.n_draft_tokens + (double) metrics.n_draft_tokens }, { "spec_decode_num_accepted_tokens_total", "Speculative: Total draft tokens accepted by the target model", - metrics.n_draft_accepted + (double) metrics.n_draft_accepted }, { "spec_decode_num_drafts_total", "Speculative: Total speculative decoding verification steps", - metrics.n_draft_verif_steps + (double) metrics.n_draft_verif_steps }, }; @@ -1574,15 +1574,15 @@ std::string server_task_result_metrics::to_metrics() { }, { "requests_processing", "Number of requests processing", - (uint64_t) n_processing_slots + (double) n_processing_slots }, { "requests_deferred", "Number of requests deferred", - (uint64_t) n_tasks_deferred + (double) n_tasks_deferred }, { "n_busy_slots_per_decode", "Average number of busy slots per llama_decode() call", - (float) metrics.n_busy_slots / std::max((float) metrics.n_decode, 1.f) + (double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0) }, }; @@ -1590,9 +1590,9 @@ std::string server_task_result_metrics::to_metrics() { auto add_items = [&prometheus](const char * type, const std::vector & items) { for (const auto & item : items) { - prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n" - << "# TYPE llamacpp:" << item.name << " " << type << "\n" - << "llamacpp:" << item.name << " " << item.value.get() << "\n"; + prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n" + << "# TYPE llamacpp:" << item.name << " " << type << "\n" + << "llamacpp:" << item.name << " " << item.value << "\n"; } }; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 9200dbf49d..b6da4d4bd6 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -508,7 +508,7 @@ struct server_task_result_metrics : server_task_result { struct metric_item { std::string name; std::string description; - json value; // can be int or double + double value; // prometheus values are always float64 }; std::string to_metrics(); };