LLM-Inference/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-12 22:31:11 +04:00
Code Issues Packages Projects Releases Wiki Activity

rm result_timings

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-12 00:33:11 +02:00
parent b16f898b2a
commit 75d2de4bac
5 changed files with 110 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.cpp
+27
View File
@@ -58,6 +58,33 @@ json format_error_response(const std::string & message, const enum error_type ty
};
}
//
// server_slot_stats
//
json server_slot_stats::to_json() const {
json base = {
{"cache_n", n_prompt_cached},
{"prompt_n", n_prompt_processed},
{"prompt_ms", t_prompt_ms()},
{"prompt_per_token_ms", t_prompt_ms() / n_prompt_processed},
{"prompt_per_second", n_prompt_tps()},
{"predicted_n", n_predict},
{"predicted_ms", t_gen_ms()},
{"predicted_per_token_ms", t_gen_ms() / n_predict},
{"predicted_per_second", n_gen_tps()},
};
if (n_draft_tokens > 0) {
base["draft_n"] = n_draft_tokens;
base["draft_n_accepted"] = n_draft_accepted;
}
return base;
}
//
// random string / id
//
tools/server/server-common.h
+62
View File
@@ -449,6 +449,68 @@ struct server_pipe {
struct server_slot;
// shared between server_slot and server_task_result_*
struct server_slot_stats {
uint64_t n_prompt_cached = 0;
uint64_t n_prompt_processed = 0;
uint64_t n_predict = 0;
// Speculative decoding stats (mirror server_metrics)
uint64_t n_draft_tokens = 0;
uint64_t n_draft_accepted = 0;
uint64_t n_draft_verif_steps = 0;
std::vector<uint64_t> n_accepted_per_pos;
// these are absolute timestamps (in us)
uint64_t t_start = 0;
uint64_t t_prompt_last = 0;
uint64_t t_gen_last = 0;
// can only move one direction: start -> prompt -> gen
void update_prompt_start() {
GGML_ASSERT(t_start == 0);
t_start = ggml_time_us();
}
void update_prompt_last() {
GGML_ASSERT(t_start > 0);
t_prompt_last = ggml_time_us();
}
void update_gen_last() {
GGML_ASSERT(t_prompt_last > 0);
t_gen_last = ggml_time_us();
}
// these are time durations
int64_t t_ellapsed_us() const {
return ggml_time_us() - t_start;
}
double t_prompt_ms() const {
return (t_prompt_last - t_start) / 1000.0;
}
double t_gen_ms() const {
// clamp to 1 us to avoid division by zero on the caller side
return std::max<int64_t>(1, t_gen_last - t_prompt_last) / 1000.0;
}
// other derived metrics
double n_prompt_tps() const {
const double t_ms = t_prompt_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0;
}
double n_gen_tps() const {
const double t_ms = t_gen_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_predict : 0.0;
}
// false if the slot never started, i.e. the task result carries no stats
bool is_set() const {
return t_start > 0;
}
json to_json() const;
};
// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot
struct server_metrics {
int64_t t_start = 0;
tools/server/server-context.cpp
+2 -81
View File
@@ -192,59 +192,6 @@ struct server_batch {
}
};
struct server_slot_stats {
uint64_t n_prompt_cached = 0;
uint64_t n_prompt_processed = 0;
uint64_t n_predict = 0;
// Speculative decoding stats (mirror server_metrics)
uint64_t n_draft_tokens = 0;
uint64_t n_draft_accepted = 0;
uint64_t n_draft_verif_steps = 0;
std::vector<uint64_t> n_accepted_per_pos;
// these are absolute timestamps (in us)
uint64_t t_start = 0;
uint64_t t_prompt_last = 0;
uint64_t t_gen_last = 0;
// can only move one direction: start -> prompt -> gen
void update_prompt_start() {
GGML_ASSERT(t_start == 0);
t_start = ggml_time_us();
}
void update_prompt_last() {
GGML_ASSERT(t_start > 0);
t_prompt_last = ggml_time_us();
}
void update_gen_last() {
GGML_ASSERT(t_prompt_last > 0);
t_gen_last = ggml_time_us();
}
// these are time durations
int64_t t_ellapsed_us() const {
return ggml_time_us() - t_start;
}
double t_prompt_ms() const {
return (t_prompt_last - t_start) / 1000.0;
}
double t_gen_ms() const {
// clamp to 1 us to avoid division by zero on the caller side
return std::max<int64_t>(1, t_gen_last - t_prompt_last) / 1000.0;
}
// other derived metrics
double n_prompt_tps() const {
const double t_ms = t_prompt_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0;
}
double n_gen_tps() const {
const double t_ms = t_gen_ms();
return t_ms > 0.0 ? 1e3 / t_ms * n_predict : 0.0;
}
};
struct server_slot {
int id;
@@ -566,32 +513,6 @@ struct server_slot {
}
}
result_timings get_timings() const {
const double t_prompt_processing = stats.t_prompt_ms();
const double t_token_generation = stats.t_gen_ms();
result_timings timings;
timings.cache_n = stats.n_prompt_cached;
timings.prompt_n = stats.n_prompt_processed;
timings.prompt_ms = t_prompt_processing;
timings.prompt_per_token_ms = t_prompt_processing / stats.n_prompt_processed;
timings.prompt_per_second = stats.n_prompt_tps();
timings.predicted_n = stats.n_predict;
timings.predicted_ms = t_token_generation;
timings.predicted_per_token_ms = t_token_generation / stats.n_predict;
timings.predicted_per_second = stats.n_gen_tps();
// Add speculative metrics
if (stats.n_draft_tokens > 0) {
timings.draft_n = stats.n_draft_tokens;
timings.draft_n_accepted = stats.n_draft_accepted;
}
return timings;
}
size_t find_stopping_strings(const std::string & text, const size_t last_token_size, bool is_full_stop) {
GGML_ASSERT(task);
@@ -2123,7 +2044,7 @@ private:
// populate timings if this is final response or timings_per_token is enabled
if (slot.stop != STOP_TYPE_NONE || slot.task->params.timings_per_token) {
res->timings = slot.get_timings();
res->stats = slot.stats;
}
queue_results.send(std::move(res));
@@ -2150,7 +2071,7 @@ private:
res->content = std::move(slot.generated_text);
res->tokens = std::move(slot.generated_tokens);
}
res->timings = slot.get_timings();
res->stats = slot.stats;
res->prompt = slot.task->tokens.detokenize(ctx_tgt, true);
res->response_fields = std::move(slot.task->params.response_fields);
tools/server/server-task.cpp
+17 -45
View File
@@ -236,34 +236,6 @@ common_chat_msg task_result_state::update_chat_msg(
return chat_msg;
}
//
// result_timings
//
json result_timings::to_json() const {
json base = {
{"cache_n", cache_n},
{"prompt_n", prompt_n},
{"prompt_ms", prompt_ms},
{"prompt_per_token_ms", prompt_per_token_ms},
{"prompt_per_second", prompt_per_second},
{"predicted_n", predicted_n},
{"predicted_ms", predicted_ms},
{"predicted_per_token_ms", predicted_per_token_ms},
{"predicted_per_second", predicted_per_second},
};
if (draft_n > 0) {
base["draft_n"] = draft_n;
base["draft_n_accepted"] = draft_n_accepted;
}
return base;
}
//
// result_prompt_progress
//
@@ -382,7 +354,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() {
{"stop_type", stop_type_to_str(stop)},
{"stopping_word", stopping_word},
{"tokens_cached", n_tokens_cached},
{"timings", timings.to_json()},
{"timings", stats.to_json()},
};
if (!stream && !probs_output.empty()) {
res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);
@@ -432,8 +404,8 @@ json server_task_result_cmpl_final::to_json_oaicompat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
return res;
@@ -480,8 +452,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
return res;
@@ -541,8 +513,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {
});
}
if (timings.prompt_n >= 0) {
deltas.back().push_back({"timings", timings.to_json()});
if (stats.is_set()) {
deltas.back().push_back({"timings", stats.to_json()});
}
// extra fields for debugging purposes
@@ -734,8 +706,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
}}
});
if (timings.prompt_n >= 0) {
server_sent_events.back().at("data").push_back({"timings", timings.to_json()});
if (stats.is_set()) {
server_sent_events.back().at("data").push_back({"timings", stats.to_json()});
}
return server_sent_events;
@@ -1086,8 +1058,8 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {
{"tokens_evaluated", n_prompt_tokens},
};
// populate the timings object when needed (usually for the last response or with timings_per_token enabled)
if (timings.prompt_n > 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
@@ -1126,8 +1098,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {
if (verbose) {
res["__verbose"] = to_json_non_oaicompat();
}
if (timings.prompt_n >= 0) {
res.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
res.push_back({"timings", stats.to_json()});
}
if (is_progress) {
res.push_back({"prompt_progress", progress.to_json()});
@@ -1180,8 +1152,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_chat() {
};
}
if (timings.prompt_n >= 0) {
last_json.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
last_json.push_back({"timings", stats.to_json()});
}
if (is_progress) {
last_json.push_back({"prompt_progress", progress.to_json()});
@@ -1330,8 +1302,8 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
if (!events.empty()) {
json & data = events.back().at("data");
if (timings.prompt_n >= 0) {
data.push_back({"timings", timings.to_json()});
if (stats.is_set()) {
data.push_back({"timings", stats.to_json()});
}
if (is_progress) {
data.push_back({"prompt_progress", progress.to_json()});
tools/server/server-task.h
+2 -22
View File
@@ -259,26 +259,6 @@ struct server_task {
}
};
struct result_timings {
int32_t cache_n = -1;
int32_t prompt_n = -1;
double prompt_ms = 0.0;
double prompt_per_token_ms = 0.0;
double prompt_per_second = 0.0;
int32_t predicted_n = -1;
double predicted_ms = 0.0;
double predicted_per_token_ms = 0.0;
double predicted_per_second = 0.0;
// Optional speculative metrics - only included when > 0
int32_t draft_n = 0;
int32_t draft_n_accepted = 0;
json to_json() const;
};
struct result_prompt_progress {
int32_t total = 0;
int32_t cache = 0;
@@ -343,7 +323,7 @@ struct server_task_result_cmpl_final : server_task_result {
bool stream;
bool include_usage;
result_timings timings;
server_slot_stats stats;
std::string prompt;
bool truncated;
@@ -425,7 +405,7 @@ struct server_task_result_cmpl_partial : server_task_result {
bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
// ref: https://github.com/ggml-org/llama.cpp/pull/23884
completion_token_output prob_output;
result_timings timings;
server_slot_stats stats;
result_prompt_progress progress;
// response formatting