LLM-Inference/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-08-12 22:29:39 +04:00
Code Issues Packages Projects Releases Wiki Activity

server: reset cache tokens after pp stops (#1787)

Browse Source 
Co-authored-by: firecoperana <firecoperana>
This commit is contained in:
firecoperana
2026-05-13 09:05:32 +03:00
committed by GitHub
co-authored by firecoperana <firecoperana>
parent f9a93c37e2
commit cdc288bc97
2 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/server/server-context.cpp
+9 -4
View File
@@ -721,9 +721,8 @@ void server_slot::release() {
command = SLOT_COMMAND_RELEASE;
state = SLOT_STATE_IDLE;
task.reset();
llama_decode_reset();
}
llama_decode_reset();
}
@@ -4545,8 +4544,14 @@ void server_context::process_batch_tokens(int32_t & n_batch) {
for (auto& slot : slots) {
slot.state = SLOT_STATE_PROCESSING;
slot.command = SLOT_COMMAND_NONE;
slot.release();
if (ret != user_cancel) {
if (ret == user_cancel) {
llama_pos cur_pos = llama_kv_cache_seq_pos_max(slot.ctx, slot.id);
slot.n_past = slot.cache_tokens.size_up_to_pos(cur_pos + 1);
slot.cache_tokens.keep_first(slot.n_past);
slot.release();
}
else {
slot.release();
LLAMA_LOG_INFO("n_past = %d\n", (int)slot.cache_tokens.size());
send_error(slot, "Input prompt is too big compared to KV size. Please try increasing KV size.");
}