LLM-Inference/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-08-12 22:29:39 +04:00
Code Issues Packages Projects Releases Wiki Activity

Fix DSV4 tool calls and reasoning (#2242)

Browse Source 
* Fix DSV4 tool calls and reasoning

There are multiple changes. The most important one is the wiring, to
avoid falling back to the autoparser. With autoparser, all arguments
will be forced by the generated grammar to have the `string="true"`
attribute, which then breaks prompt caching, as it would diverge from
what is rendered by the template. Parallel tool calls also doesn't work
when falling back to autoparser.

Other changes:
* Order tool results by tool call order.
* Consume `</think>` instead of `<think></think>` when thinking is
  disabled.
* Use "preserved thinking" mode when any tool is defined, otherwise use
  "interleaved thinking" mode, e.g. for multi-turns chat. Set template
  arg `drop_thinking` to false to force "preserved thinking" mode even
  when no tool is defined.
* Add a message to system prompt when reasoning effort is set to max.

The changes were made by following:
1. The Technical Report: https://arxiv.org/abs/2606.19348
2. Reference implementatin: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731/blob/main/encoding/encoding_dsv4.py
3. VLLM implementation: https://github.com/vllm-project/vllm/blob/main/vllm/tokenizers/deepseek_v4_encoding.py

For the last bullet point about max reasoning effort, the reference
implementation diverges from the other 2, so we follow the techinical
report and the VLLM implementation, for now. This needs more testing.

* Add back trailing newline

* Update the reasoning effort instruction to follow the reference impl

Using the prompt counting test from @coder543, 0731 does have a special
instruction for "high" and another special instruction for "max".

This will break preview, but assuming most people will use the 0731
release, it should be fine.

[1] https://www.reddit.com/r/DeepSeek/comments/1vdqjwr/openrouter_reasoning_effort_levels_are_broken_for/
This commit is contained in:
Yap Sok Ann
2026-08-04 19:28:06 +03:00
committed by GitHub
parent af4e0cbdb6
commit 6b55d2c750
2 changed files with 116 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/chat.cpp
+90 -10
View File
@@ -19,7 +19,9 @@
#include <cstdlib>
#include <ctime>
#include <exception>
#include <algorithm>
#include <functional>
#include <map>
#include <optional>
#include <sstream>
@@ -1794,11 +1796,83 @@ static common_chat_params common_chat_params_init_gigachat_v3(
return data;
}
// The DeepSeek V4 reference implementation renders consecutive tool results into a single
// user block, ordered by the tool call order of the preceding assistant message (matched
// by tool call id) rather than by the order they appear in the conversation.
static json deepseek_v4_sort_tool_results(const json & messages) {
json adjusted = messages;
std::map<std::string, size_t> call_order;
for (size_t i = 0; i < adjusted.size();) {
const auto & msg = adjusted[i];
const auto role = msg.value("role", "");
if (role == "assistant" && msg.contains("tool_calls") &&
msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
call_order.clear();
const auto & tool_calls = msg.at("tool_calls");
for (size_t idx = 0; idx < tool_calls.size(); idx++) {
auto id = tool_calls[idx].value("id", "");
if (!id.empty()) {
call_order[id] = idx;
}
}
i++;
continue;
}
if (role != "user" && role != "tool") {
i++;
continue;
}
std::vector<size_t> tool_positions;
size_t run_end = i;
for (; run_end < adjusted.size(); run_end++) {
const auto r = adjusted[run_end].value("role", "");
if (r == "tool") {
tool_positions.push_back(run_end);
} else if (r != "user") {
break;
}
}
if (tool_positions.size() > 1 && !call_order.empty()) {
std::vector<json> results;
results.reserve(tool_positions.size());
for (auto pos : tool_positions) {
results.push_back(adjusted[pos]);
}
std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
const auto order = [&](const json & m) {
auto it = call_order.find(m.value("tool_call_id", ""));
return it == call_order.end() ? (size_t) 0 : it->second;
};
return order(a) < order(b);
});
for (size_t k = 0; k < tool_positions.size(); k++) {
adjusted[tool_positions[k]] = std::move(results[k]);
}
}
i = run_end;
}
return adjusted;
}
static common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
std::optional<json> adjusted_messages;
if (is_v4) {
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
}
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = "<think>";
@@ -1817,8 +1891,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
const std::string DSML = "｜DSML｜";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
const std::string FC_START = "<" + DSML + "function_calls>";
const std::string FC_END = "</" + DSML + "function_calls>";
const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";
const std::string FC_START = "<" + DSML + TC_BLOCK + ">";
const std::string FC_END = "</" + DSML + TC_BLOCK + ">";
const std::string INVOKE_START = "<" + DSML + "invoke";
const std::string INVOKE_END = "</" + DSML + "invoke>";
const std::string PARAM_START = "<" + DSML + "parameter";
@@ -1832,9 +1907,12 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
if (extract_reasoning && inputs.enable_thinking) {
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
} else if (extract_reasoning) {
// Thinking disabled but reasoning extraction requested: the generation prompt
// contains an empty <think></think> pair that must still be consumed.
reasoning = p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
// Thinking disabled but reasoning extraction requested: consume the thinking
// delimiters the generation prompt placed. V3.2 emits an empty <think></think>
// pair, while V4 emits a bare </think>.
reasoning = is_v4
? p.optional(p.literal(THINK_END))
: p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
}
if (has_response_format) {
@@ -2575,12 +2653,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_minimax_m3(tmpl, params);
}
// DeepSeek V3.2 format detection: template defines dsml_token and uses it for tool calls.
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
if (src.find("dsml_token") != std::string::npos &&
src.find("function_calls") != std::string::npos &&
src.find("DSML") != std::string::npos) {
LOG_DBG("Using specialized template: DeepSeek V3.2\n");
src.find("DSML") != std::string::npos &&
(src.find("function_calls") != std::string::npos ||
src.find("tool_calls") != std::string::npos)) {
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
return common_chat_params_init_deepseek_v3_2(tmpl, params);
}
models/templates/deepseek-ai-DeepSeek-V4.jinja
+26 -5
View File
@@ -8,12 +8,17 @@
{%- set thinking = false -%}
{%- endif -%}
{%- endif -%}
{%- if not drop_thinking is defined -%}
{%- set drop_thinking = true -%}
{%- endif -%}
{%- set dsml_token = '｜DSML｜' -%}
{%- set thinking_start_token = '<think>' -%}
{%- set thinking_end_token = '</think>' -%}
{%- set reasoning_effort_high = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%}
{%- set reasoning_effort_max = 'Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\nYou MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\nDo not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n' -%}
{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}
{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}
{%- set ns = namespace(system_prompt = '', is_first_sp = true) -%}
{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}
{%- for message in messages -%}
{%- if message['role'] == 'system' -%}
{%- if ns.is_first_sp -%}
@@ -25,7 +30,7 @@
{%- endif -%}
{%- endfor -%}
{%- if tools is defined and tools -%}
{%- set ts = namespace(schemas = '') -%}
{%- set ts = namespace(schemas='') -%}
{%- for tool in tools -%}
{%- if tool['type'] == 'function' -%}
{%- set ts.schemas = ts.schemas + (tool['function'] | tojson) + '\n' -%}
@@ -38,14 +43,29 @@
{%- endif -%}
{%- endif -%}
{{- bos_token -}}
{%- if thinking and reasoning_effort is defined -%}
{%- if reasoning_effort == 'high' -%}
{{- reasoning_effort_high -}}
{%- elif reasoning_effort == 'max' -%}
{{- reasoning_effort_max -}}
{%- endif -%}
{%- endif -%}
{{- ns.system_prompt -}}
{%- set last_user_idx = namespace(value = -1) -%}
{%- set last_user_idx = namespace(value=-1) -%}
{%- for message in messages -%}
{%- if message['role'] == 'user' or message['role'] == 'developer' or message['role'] == 'tool' -%}
{%- set last_user_idx.value = loop.index0 -%}
{%- endif -%}
{%- endfor -%}
{%- set state = namespace(in_user = false) -%}
{%- set state = namespace(in_user=false) -%}
{%- if tools is defined and tools -%}
{%- set ns.has_tool_calls = true -%}
{%- endif -%}
{%- for message in messages -%}
{%- if message['role'] == 'tool' -%}
{%- set ns.has_tool_calls = true -%}
{%- endif -%}
{%- endfor -%}
{%- for message in messages -%}
{%- if message['role'] == 'user' or message['role'] == 'developer' -%}
{%- if state.in_user -%}
@@ -67,7 +87,8 @@
{%- set state.in_user = false -%}
{{- '<｜Assistant｜>' -}}
{%- set is_after_last_user = loop.index0 > last_user_idx.value -%}
{%- if is_after_last_user and thinking -%}
{%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%}
{%- if retain_reasoning and thinking -%}
{{- thinking_start_token -}}
{%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}
{{- message['reasoning_content'] -}}