From b0ed986aecb0187e8bc09357808c69fadcc78a9c Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 13 Feb 2026 00:55:35 +0100 Subject: [PATCH] -> Refactor autoparser analyzer structure -> Fix content truncation -> Fix errors in capability detection due to non-empty assistant message -> Add missing debug prints for Jinja --- common/chat-auto-parser-generator.cpp | 159 ++- common/chat-diff-analyzer.cpp | 1124 ++++++++--------- common/chat-diff-analyzer.h | 269 ++-- common/chat-peg-parser.cpp | 4 +- common/chat.cpp | 2 +- common/jinja/caps.cpp | 11 +- common/jinja/runtime.cpp | 4 +- .../templates/deepseek-ai-DeepSeek-V3.1.jinja | 6 +- tests/test-chat-auto-parser.cpp | 98 +- tests/test-chat-peg-parser.cpp | 2 +- tests/test-chat-template.cpp | 58 +- tools/parser/debug-template-parser.cpp | 62 +- 12 files changed, 903 insertions(+), 896 deletions(-) diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index e9fe71c1d6..13ec14fb64 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -30,8 +30,8 @@ common_chat_params universal_peg_generator::generate_parser(const common_chat_te const struct templates_params & inputs, const diff_analysis_result & analysis) { // Check for thinking forced open - bool thinking_forced_open = (analysis.reasoning == reasoning_mode::FORCED_OPEN); - bool thinking_forced_closed = (analysis.reasoning == reasoning_mode::FORCED_CLOSED); + bool thinking_forced_open = (analysis.reasoning.mode == reasoning_mode::FORCED_OPEN); + bool thinking_forced_closed = (analysis.reasoning.mode == reasoning_mode::FORCED_CLOSED); // Build the parser using the analysis results auto parser = build_parser(analysis, inputs, thinking_forced_open, thinking_forced_closed); @@ -44,7 +44,7 @@ common_chat_params universal_peg_generator::generate_parser(const common_chat_te data.parser = parser.save(); // Build grammar if tools are present - bool has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + bool has_tools = analysis.tools.format.mode != tool_format::NONE && inputs.tools.is_array() && !inputs.tools.empty(); bool include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; if (include_grammar) { @@ -60,9 +60,9 @@ common_chat_params universal_peg_generator::generate_parser(const common_chat_te }); // Set grammar triggers based on tool section markers (fall back to per-call markers) - std::string trigger_marker = !analysis.markers.tool_section_start.empty() - ? analysis.markers.tool_section_start - : analysis.markers.per_call_start; + std::string trigger_marker = !analysis.tools.format.section_start.empty() + ? analysis.tools.format.section_start + : analysis.tools.format.per_call_start; if (!trigger_marker.empty()) { data.grammar_triggers = { { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker } @@ -79,26 +79,24 @@ common_peg_arena universal_peg_generator::build_parser(const diff_analysis_resul bool thinking_forced_closed) { return build_chat_peg_unified_parser([&](common_chat_peg_unified_builder & p) { p.set_allow_python_dict_format(true); - const auto & m = analysis.markers; - common_peg_parser reasoning = p.eps(); bool extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; bool enable_thinking = inputs.enable_thinking; - if (extract_reasoning && enable_thinking && analysis.reasoning != reasoning_mode::NONE) { + if (extract_reasoning && enable_thinking && analysis.reasoning.mode != reasoning_mode::NONE) { if (thinking_forced_open || thinking_forced_closed) { // Thinking is forced open OR forced closed with enable_thinking=true // In both cases, expect only the closing tag (opening was in template) - reasoning = p.reasoning(p.until(m.reasoning_end)) + m.reasoning_end; - } else if (analysis.reasoning == reasoning_mode::TAG_BASED || - analysis.reasoning == reasoning_mode::TOOLS_ONLY) { + reasoning = p.reasoning(p.until(analysis.reasoning.end)) + analysis.reasoning.end; + } else if (analysis.reasoning.mode == reasoning_mode::TAG_BASED || + analysis.reasoning.mode == reasoning_mode::TOOLS_ONLY) { // Standard tag-based reasoning OR tools-only mode (reasoning appears with tools) // Both use the same tag-based pattern if markers are available - if (!m.reasoning_start.empty() && !m.reasoning_end.empty()) { - reasoning = p.optional(m.reasoning_start + p.reasoning(p.until(m.reasoning_end)) + m.reasoning_end); + if (!analysis.reasoning.start.empty() && !analysis.reasoning.end.empty()) { + reasoning = p.optional(analysis.reasoning.start + p.reasoning(p.until(analysis.reasoning.end)) + analysis.reasoning.end); } - } else if (analysis.reasoning == reasoning_mode::DELIMITER) { - reasoning = p.optional(p.reasoning(p.until(m.reasoning_end)) + m.reasoning_end); + } else if (analysis.reasoning.mode == reasoning_mode::DELIMITER) { + reasoning = p.optional(p.reasoning(p.until(analysis.reasoning.end)) + analysis.reasoning.end); } } @@ -109,19 +107,19 @@ common_peg_arena universal_peg_generator::build_parser(const diff_analysis_resul return reasoning + p.space() + p.content(p.schema(p.json(), "response-format", inputs.json_schema)) + p.end(); } - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && analysis.supports_tools) { + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && analysis.jinja_caps.supports_tool_calls) { return build_tool_parser(p, analysis, inputs, reasoning); } - if (analysis.content == content_mode::ALWAYS_WRAPPED && - !m.content_start.empty() && !m.content_end.empty()) { + if (analysis.content.mode == content_mode::ALWAYS_WRAPPED && + !analysis.content.start.empty() && !analysis.content.end.empty()) { - bool extracting_reasoning = extract_reasoning && enable_thinking && analysis.reasoning != reasoning_mode::NONE; + bool extracting_reasoning = extract_reasoning && enable_thinking && analysis.reasoning.mode != reasoning_mode::NONE; if (extracting_reasoning) { - return reasoning + m.content_start + p.content(p.until(m.content_end)) + m.content_end + p.end(); + return reasoning + analysis.content.start + p.content(p.until(analysis.content.end)) + analysis.content.end + p.end(); } - return p.content(p.until(m.content_start)) + m.content_start + p.content(p.until(m.content_end)) + m.content_end + p.end(); + return p.content(p.until(analysis.content.start)) + analysis.content.start + p.content(p.until(analysis.content.end)) + analysis.content.end + p.end(); } return reasoning + p.content(p.rest()) + p.end(); }); @@ -133,7 +131,7 @@ common_peg_parser universal_peg_generator::build_tool_parser( const templates_params & inputs, const common_peg_parser & reasoning) { - switch (analysis.tools) { + switch (analysis.tools.format.mode) { case tool_format::JSON_NATIVE: return build_tool_parser_json_native(p, analysis, inputs, reasoning); case tool_format::TAG_WITH_JSON: @@ -151,42 +149,40 @@ common_peg_parser universal_peg_generator::build_tool_parser_json_native( const templates_params & inputs, const common_peg_parser & reasoning) { - const auto & m = analysis.markers; - // Build effective field names with dot notation if function_field is set - std::string name_field = analysis.name_field; - std::string args_field = analysis.args_field; + std::string name_field = analysis.tools.format.name_field; + std::string args_field = analysis.tools.format.args_field; - if (!analysis.function_field.empty() && - analysis.function_field != "function" && + if (!analysis.tools.format.function_field.empty() && + analysis.tools.format.function_field != "function" && name_field.find('.') == std::string::npos) { - name_field = analysis.function_field + "." + name_field; - args_field = analysis.function_field + "." + args_field; + name_field = analysis.tools.format.function_field + "." + name_field; + args_field = analysis.tools.format.function_field + "." + args_field; } auto tools_parser = p.standard_json_tools( - m.tool_section_start, - m.tool_section_end, + analysis.tools.format.section_start, + analysis.tools.format.section_end, inputs.tools, inputs.parallel_tool_calls, inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, - analysis.tools_array_wrapped, - analysis.fun_name_is_key, - analysis.id_field, - analysis.gen_id_field, - analysis.parameter_order + analysis.tools.format.tools_array_wrapped, + analysis.tools.format.fun_name_is_key, + analysis.tools.format.id_field, + analysis.tools.format.gen_id_field, + analysis.tools.format.parameter_order ); // Handle content wrappers if present - if (analysis.content == content_mode::ALWAYS_WRAPPED && - !m.content_start.empty() && !m.content_end.empty()) { - auto wrapped_content = p.optional(m.content_start + p.content(p.until(m.content_end)) + m.content_end); + if (analysis.content.mode == content_mode::ALWAYS_WRAPPED && + !analysis.content.start.empty() && !analysis.content.end.empty()) { + auto wrapped_content = p.optional(analysis.content.start + p.content(p.until(analysis.content.end)) + analysis.content.end); return reasoning + wrapped_content + tools_parser + p.end(); } - auto content_before_tools = m.tool_section_start.empty() ? p.eps() : p.until(m.tool_section_start); + auto content_before_tools = analysis.tools.format.section_start.empty() ? p.eps() : p.until(analysis.tools.format.section_start); return reasoning + p.optional(p.content(content_before_tools)) + tools_parser + p.end(); } @@ -196,7 +192,6 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( const templates_params & inputs, const common_peg_parser & reasoning) { - const auto & m = analysis.markers; common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { @@ -206,17 +201,17 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( // Build call_id parser based on position (if supported) common_peg_parser call_id_section = p.eps(); - if (analysis.call_id_pos == call_id_position::BETWEEN_FUNC_AND_ARGS && - !m.call_id_prefix.empty() && !m.call_id_suffix.empty()) { - call_id_section = p.optional(m.call_id_prefix + p.tool_id(p.until(m.call_id_suffix))) + m.call_id_suffix; + if (analysis.tools.call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && + !analysis.tools.call_id.prefix.empty() && !analysis.tools.call_id.suffix.empty()) { + call_id_section = p.optional(analysis.tools.call_id.prefix + p.tool_id(p.until(analysis.tools.call_id.suffix))) + analysis.tools.call_id.suffix; } - auto func_parser = p.tool_open(m.func_name_prefix + p.tool_name(p.literal(name)) + m.func_name_suffix) + + auto func_parser = p.tool_open(analysis.tools.function.name_prefix + p.tool_name(p.literal(name)) + analysis.tools.function.name_suffix) + call_id_section + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)); - if (!m.func_close.empty()) { - func_parser = func_parser + m.func_close; + if (!analysis.tools.function.close.empty()) { + func_parser = func_parser + analysis.tools.function.close; } tool_choice |= p.rule("tool-" + name, func_parser); @@ -226,30 +221,26 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( common_peg_parser tool_calls = p.eps(); - if (!m.per_call_start.empty()) { - auto wrapped_call = m.per_call_start + tool_choice + m.per_call_end; + if (!analysis.tools.format.per_call_start.empty()) { + auto wrapped_call = analysis.tools.format.per_call_start + tool_choice + analysis.tools.format.per_call_end; if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call)); } else { tool_calls = p.trigger_rule("tool-call", wrapped_call); } - if (!m.tool_section_start.empty()) { - tool_calls = p.trigger_rule("tool-calls", p.literal(m.tool_section_start) + p.space() + - tool_calls + p.space() + (m.tool_section_end.empty() ? p.end() : p.literal(m.tool_section_end))); + if (!analysis.tools.format.section_start.empty()) { + tool_calls = p.trigger_rule("tool-calls", p.literal(analysis.tools.format.section_start) + p.space() + + tool_calls + p.space() + (analysis.tools.format.section_end.empty() ? p.end() : p.literal(analysis.tools.format.section_end))); } } else { - std::string separator = m.call_separator; - if (separator.empty()) { - separator = ", "; // Default - } - + std::string separator = ", "; // Default if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", - m.tool_section_start + tool_choice + p.zero_or_more(separator + tool_choice) + m.tool_section_end); + analysis.tools.format.section_start + tool_choice + p.zero_or_more(separator + tool_choice) + analysis.tools.format.section_end); } else { tool_calls = p.trigger_rule("tool-call", - m.tool_section_start + tool_choice + m.tool_section_end); + analysis.tools.format.section_start + tool_choice + analysis.tools.format.section_end); } } @@ -257,7 +248,7 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !m.tool_section_start.empty() ? m.tool_section_start : m.per_call_start; + std::string trigger_marker = !analysis.tools.format.section_start.empty() ? analysis.tools.format.section_start : analysis.tools.format.per_call_start; auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); return reasoning + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } @@ -268,7 +259,6 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( const templates_params & inputs, const common_peg_parser & reasoning) { - const auto & m = analysis.markers; common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { @@ -293,13 +283,13 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( auto type = param_schema.value("type", "object"); auto arg = p.tool_arg( - p.tool_arg_open(m.arg_name_prefix + p.tool_arg_name(p.literal(param_name)) + m.arg_name_suffix) + m.arg_value_prefix + + p.tool_arg_open(analysis.tools.arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) + analysis.tools.arguments.name_suffix) + analysis.tools.arguments.value_prefix + (type == "string" ? - p.tool_arg_string_value(p.schema(p.until(m.arg_value_suffix), + p.tool_arg_string_value(p.schema(p.until(analysis.tools.arguments.value_suffix), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, true)) : p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema)) + p.space()) + - p.tool_arg_close(p.literal(m.arg_value_suffix)) + p.tool_arg_close(p.literal(analysis.tools.arguments.value_suffix)) ); if (is_required) { @@ -320,23 +310,23 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( // Build call_id parser based on position (if supported) common_peg_parser call_id_section = p.eps(); - if (analysis.call_id_pos == call_id_position::BETWEEN_FUNC_AND_ARGS && - !m.call_id_prefix.empty() && !m.call_id_suffix.empty()) { - call_id_section = p.optional(m.call_id_prefix + p.tool_id(p.until(m.call_id_suffix))) + m.call_id_suffix; + if (analysis.tools.call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && + !analysis.tools.call_id.prefix.empty() && !analysis.tools.call_id.suffix.empty()) { + call_id_section = p.optional(analysis.tools.call_id.prefix + p.tool_id(p.until(analysis.tools.call_id.suffix))) + analysis.tools.call_id.suffix; } - auto func_parser = p.tool_open(m.func_name_prefix + p.tool_name(p.literal(name)) + m.func_name_suffix) + + auto func_parser = p.tool_open(analysis.tools.function.name_prefix + p.tool_name(p.literal(name)) + analysis.tools.function.name_suffix) + call_id_section + p.space() + args_seq; - if (!m.func_close.empty()) { - func_parser = func_parser + p.space() + p.tool_close(p.literal(m.func_close)); - } else if (!m.per_call_end.empty()) { + if (!analysis.tools.function.close.empty()) { + func_parser = func_parser + p.space() + p.tool_close(p.literal(analysis.tools.function.close)); + } else if (!analysis.tools.format.per_call_end.empty()) { // When there's no func_close but there is a per_call_end marker, use peek() to ensure // we only emit tool_close when we can actually see the closing marker. This prevents // premature closing during partial parsing when we've seen e.g. "" (end) or "" prefix that failed to match. - func_parser = func_parser + p.tool_close(p.peek(p.literal(m.per_call_end))); + func_parser = func_parser + p.tool_close(p.peek(p.literal(analysis.tools.format.per_call_end))); } else { func_parser = func_parser + p.tool_close(p.space()); // force this to process tool closing callbacks in mapper } @@ -348,29 +338,26 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( common_peg_parser tool_calls = p.eps(); - if (!m.per_call_start.empty()) { - auto wrapped_call = m.per_call_start + p.space() + tool_choice + p.space() + m.per_call_end; + if (!analysis.tools.format.per_call_start.empty()) { + auto wrapped_call = analysis.tools.format.per_call_start + p.space() + tool_choice + p.space() + analysis.tools.format.per_call_end; if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call)); } else { tool_calls = p.trigger_rule("tool-call", wrapped_call); } - if (!m.tool_section_start.empty()) { - tool_calls = p.trigger_rule("tool-calls", p.literal(m.tool_section_start) + p.space() + - tool_calls + p.space() + (m.tool_section_end.empty() ? p.end() : p.literal(m.tool_section_end))); + if (!analysis.tools.format.section_start.empty()) { + tool_calls = p.trigger_rule("tool-calls", p.literal(analysis.tools.format.section_start) + p.space() + + tool_calls + p.space() + (analysis.tools.format.section_end.empty() ? p.end() : p.literal(analysis.tools.format.section_end))); } } else { - std::string separator = m.call_separator; - if (separator.empty()) { - separator = ", "; // Default - } + std::string separator = ", "; // Default if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", - m.tool_section_start + p.space() + tool_choice + p.zero_or_more(separator + tool_choice) + p.space() + m.tool_section_end); + analysis.tools.format.section_start + p.space() + tool_choice + p.zero_or_more(separator + tool_choice) + p.space() + analysis.tools.format.section_end); } else { tool_calls = p.trigger_rule("tool-call", - m.tool_section_start + p.space() + tool_choice + p.space() + m.tool_section_end); + analysis.tools.format.section_start + p.space() + tool_choice + p.space() + analysis.tools.format.section_end); } } @@ -378,7 +365,7 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !m.tool_section_start.empty() ? m.tool_section_start : m.per_call_start; + std::string trigger_marker = !analysis.tools.format.section_start.empty() ? analysis.tools.format.section_start : analysis.tools.format.per_call_start; auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); return reasoning + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index 080fecfce2..03978f6e57 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -20,76 +20,77 @@ static std::vector void { - if (tmpl.src.find("content.split('')") != std::string::npos && analysis.reasoning == reasoning_mode::NONE) { - analysis.reasoning = reasoning_mode::FORCED_OPEN; - analysis.markers.reasoning_start = ""; - analysis.markers.reasoning_end = ""; - analysis.preserved_tokens.push_back(""); - analysis.preserved_tokens.push_back(""); - LOG_DBG(ANSI_ORANGE "[Patch: old Qwen/Deepseek thinking template]\n" ANSI_RESET); + if (tmpl.src.find("content.split('')") != std::string::npos && + analysis.reasoning.mode == reasoning_mode::NONE) { + analysis.reasoning.mode = reasoning_mode::FORCED_OPEN; + analysis.reasoning.start = ""; + analysis.reasoning.end = ""; + analysis.preserved_tokens.push_back(""); + analysis.preserved_tokens.push_back(""); + LOG_DBG(ANSI_ORANGE "[Patch: old Qwen/Deepseek thinking template]\n" ANSI_RESET); } }, // Granite 3.3, with separate reasoning and content markers [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { if (tmpl.src.find("Write your thoughts between and write your response between " "") != std::string::npos) { - analysis.reasoning = reasoning_mode::TAG_BASED; - analysis.markers.reasoning_start = ""; - analysis.markers.reasoning_end = ""; - analysis.preserved_tokens.push_back(""); - analysis.preserved_tokens.push_back(""); - analysis.content = content_mode::WRAPPED_WITH_REASONING; - analysis.markers.content_start = ""; - analysis.markers.content_end = ""; - analysis.preserved_tokens.push_back(""); - analysis.preserved_tokens.push_back(""); - LOG_DBG(ANSI_ORANGE "[Patch: Granite 3.3]\n" ANSI_RESET); + analysis.reasoning.mode = reasoning_mode::TAG_BASED; + analysis.reasoning.start = ""; + analysis.reasoning.end = ""; + analysis.preserved_tokens.push_back(""); + analysis.preserved_tokens.push_back(""); + analysis.content.mode = content_mode::WRAPPED_WITH_REASONING; + analysis.content.start = ""; + analysis.content.end = ""; + analysis.preserved_tokens.push_back(""); + analysis.preserved_tokens.push_back(""); + LOG_DBG(ANSI_ORANGE "[Patch: Granite 3.3]\n" ANSI_RESET); } }, // Cohere Command R+ - content wrapped in <|CHATBOT_TOKEN|>...<|END_OF_TURN_TOKEN|> [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { if (tmpl.src.find("<|CHATBOT_TOKEN|>") != std::string::npos && - tmpl.src.find("<|END_OF_TURN_TOKEN|>") != std::string::npos && - analysis.markers.content_start.empty()) { - analysis.content = content_mode::ALWAYS_WRAPPED; - analysis.markers.content_start = "<|CHATBOT_TOKEN|>"; - analysis.markers.content_end = "<|END_OF_TURN_TOKEN|>"; - analysis.preserved_tokens.push_back("<|CHATBOT_TOKEN|>"); - analysis.preserved_tokens.push_back("<|END_OF_TURN_TOKEN|>"); - LOG_DBG(ANSI_ORANGE "[Patch: Cohere Command R+]\n" ANSI_RESET); + tmpl.src.find("<|END_OF_TURN_TOKEN|>") != std::string::npos && analysis.content.start.empty()) { + analysis.content.mode = content_mode::ALWAYS_WRAPPED; + analysis.content.start = "<|CHATBOT_TOKEN|>"; + analysis.content.end = "<|END_OF_TURN_TOKEN|>"; + analysis.preserved_tokens.push_back("<|CHATBOT_TOKEN|>"); + analysis.preserved_tokens.push_back("<|END_OF_TURN_TOKEN|>"); + LOG_DBG(ANSI_ORANGE "[Patch: Cohere Command R+]\n" ANSI_RESET); } }, // Functionary - no tool call section delimiter [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { if (tmpl.src.find("set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", " "\"code_interpreter\") | list | length > 0") != std::string::npos) { - analysis.content = content_mode::PLAIN; - analysis.markers.content_end = ""; - analysis.markers.func_name_prefix = ""; - analysis.markers.tool_section_start = ""; - analysis.markers.tool_section_end = ""; - analysis.markers.per_call_start = ""); - analysis.preserved_tokens.push_back("<|eom_id|>"); - analysis.preserved_tokens.push_back(""); - analysis.preserved_tokens.push_back(""); - LOG_DBG(ANSI_ORANGE "[Patch: Functionary 3.1]\n" ANSI_RESET); - } + analysis.content.mode = content_mode::PLAIN; + analysis.content.end = ""; + analysis.tools.function.name_prefix = ""; + analysis.tools.format.section_start = ""; + analysis.tools.format.section_end = ""; + analysis.tools.format.per_call_start = ""); + analysis.preserved_tokens.push_back("<|eom_id|>"); + analysis.preserved_tokens.push_back(""); + analysis.preserved_tokens.push_back(""); + LOG_DBG(ANSI_ORANGE "[Patch: Functionary 3.1]\n" ANSI_RESET); + } }, // DeepSeek-R1-Distill-Qwen [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { - if (tmpl.src.find("{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'") != + if (tmpl.src.find( + "{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'") != std::string::npos) { - analysis.markers.tool_section_start = "<｜tool▁calls▁begin｜>"; - analysis.markers.tool_section_end = "<｜tool▁calls▁end｜>"; - analysis.markers.per_call_start = "<｜tool▁call▁begin｜>function"; - analysis.markers.func_name_prefix = "<｜tool▁sep｜>"; - analysis.markers.per_call_end = "<｜tool▁call▁end｜>"; - analysis.markers.func_close = "```"; + analysis.tools.format.section_start = "<｜tool▁calls▁begin｜>"; + analysis.tools.format.section_end = "<｜tool▁calls▁end｜>"; + analysis.tools.format.per_call_start = "<｜tool▁call▁begin｜>function"; + analysis.tools.function.name_prefix = "<｜tool▁sep｜>"; + analysis.tools.format.per_call_end = "<｜tool▁call▁end｜>"; + analysis.tools.function.close = "```"; } } }); @@ -191,14 +192,12 @@ diff_analysis_result differential_analyzer::analyze(const common_chat_template & LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET); - auto caps = tmpl.original_caps(); - result.supports_tools = caps.supports_tools || caps.supports_tool_calls; - result.supports_parallel_calls = caps.supports_parallel_tool_calls; + result.jinja_caps = tmpl.original_caps(); - analyze_reasoning(tmpl, result); - analyze_content(tmpl, result); - if (result.supports_tools) { - analyze_tools(tmpl, result); + result.reasoning = analyze_reasoning(tmpl, result.jinja_caps.supports_tool_calls); + result.content = analyze_content(tmpl, result.reasoning); + if (result.jinja_caps.supports_tool_calls) { + result.tools = analyze_tools(tmpl, result.jinja_caps, result.reasoning); } collect_preserved_tokens(result); @@ -211,18 +210,21 @@ diff_analysis_result differential_analyzer::analyze(const common_chat_template & return result; } -void differential_analyzer::analyze_reasoning(const common_chat_template & tmpl, diff_analysis_result & result) { +reasoning_analysis differential_analyzer::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools) { LOG_DBG(ANSI_ORANGE "Phase 1: Reasoning analysis\n" ANSI_RESET); + reasoning_analysis result; + compare_reasoning_presence(tmpl, result); compare_thinking_enabled(tmpl, result); - if (result.supports_tools) { + if (supports_tools) { compare_reasoning_scope(tmpl, result); } + + return result; } -void differential_analyzer::compare_reasoning_presence(const common_chat_template & tmpl, - diff_analysis_result & result) { +void differential_analyzer::compare_reasoning_presence(const common_chat_template & tmpl, reasoning_analysis & reasoning) { json user_msg = json{ { "role", "user" }, { "content", "Hello" } @@ -248,38 +250,34 @@ void differential_analyzer::compare_reasoning_presence(const common_chat_templat tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_reasoning }); }); if (!comparison) { - LOG_DBG(ANSI_ORANGE "R1: Template application failed, skipping reasoning detection\n" ANSI_RESET); + LOG_DBG(ANSI_ORANGE "%s: Template application failed, skipping reasoning detection\n" ANSI_RESET, __func__); return; } const auto & diff = comparison->diff; - LOG_DBG(ANSI_ORANGE "R1 diff - suffix: " ANSI_RESET "'%s', " ANSI_ORANGE "left: " ANSI_RESET "'%s', " ANSI_ORANGE - "right: " ANSI_ORANGE "'%s'\n" ANSI_RESET, - diff.suffix.c_str(), diff.left.c_str(), diff.right.c_str()); - const std::string reasoning_content = "Let me think about this."; if (!diff.right.empty() && diff.right.find(reasoning_content) != std::string::npos) { auto seg = prune_whitespace_segments(segmentize_markers(diff.right)); if (seg.size() >= 3 && trim_whitespace(seg[1].value) == reasoning_content) { // easy one: opening marker - reasoning - closing marker (possibly with trailing whitespace) - result.reasoning = reasoning_mode::TAG_BASED; - result.markers.reasoning_start = trim_whitespace(seg[0].value); - result.markers.reasoning_end = trim_leading_whitespace(seg[2].value); + reasoning.mode = reasoning_mode::TAG_BASED; + reasoning.start = trim_whitespace(seg[0].value); + reasoning.end = trim_leading_whitespace(seg[2].value); for (size_t i = 3; i < seg.size(); i++) { - result.markers.reasoning_end += seg[i].value; + reasoning.end += seg[i].value; } // we always truncate because this doesn't really influence correctness but model might not always generate newline - result.markers.reasoning_end = trim_whitespace(result.markers.reasoning_end); + reasoning.end = trim_whitespace(reasoning.end); } else if (seg.size() >= 2 && trim_whitespace(seg[0].value) == reasoning_content) { // delimited - result.reasoning = reasoning_mode::DELIMITER; - result.markers.reasoning_end = trim_leading_whitespace(seg[1].value); + reasoning.mode = reasoning_mode::DELIMITER; + reasoning.end = trim_leading_whitespace(seg[1].value); for (size_t i = 2; i < seg.size(); i++) { - result.markers.reasoning_end += seg[i].value; + reasoning.end += seg[i].value; } - result.markers.reasoning_end = trim_whitespace(result.markers.reasoning_end); + reasoning.end = trim_whitespace(reasoning.end); } else if (seg.size() == 1 && trim_whitespace(seg[0].value) == reasoning_content) { // the marker might be in the prefix actually, let's check for case of // left: empty @@ -297,16 +295,16 @@ void differential_analyzer::compare_reasoning_presence(const common_chat_templat if (marker_seg.type == segment_type::TEXT) { marker_seg = pre_seg[pre_seg.size() - 2]; } - result.reasoning = reasoning_mode::FORCED_CLOSED; - result.markers.reasoning_start = trim_whitespace(marker_seg.value); - result.markers.reasoning_end = trim_whitespace(suf_seg[0].value); + reasoning.mode = reasoning_mode::FORCED_CLOSED; + reasoning.start = trim_whitespace(marker_seg.value); + reasoning.end = trim_whitespace(suf_seg[0].value); } } } } } -void differential_analyzer::compare_thinking_enabled(const common_chat_template & tmpl, diff_analysis_result & result) { +void differential_analyzer::compare_thinking_enabled(const common_chat_template & tmpl, reasoning_analysis & reasoning) { json user_msg = json{ { "role", "user" }, { "content", "Hello" } @@ -320,15 +318,12 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template auto comparison = compare_variants(tmpl, params, [&](template_params & p) { p.enable_thinking = true; }); if (!comparison) { - LOG_DBG("R2: Template application failed\n"); + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET , __func__); return; } const auto & diff = comparison->diff; - LOG_DBG("R2 diff - suffix: '%s', left: '%s', right: '%s'\n", diff.suffix.c_str(), diff.left.c_str(), - diff.right.c_str()); - std::string left_trimmed = diff.left; trim_whitespace(left_trimmed); @@ -337,18 +332,15 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template trim_whitespace(right_trimmed); if (!right_trimmed.empty() && string_ends_with(comparison->output_B, right_trimmed)) { - if (result.markers.reasoning_start.empty()) { - result.markers.reasoning_start = right_trimmed; - result.reasoning = reasoning_mode::FORCED_OPEN; - LOG_DBG("R2: Detected forced-open reasoning with start marker: '%s'\n", right_trimmed.c_str()); + if (reasoning.start.empty()) { + reasoning.start = right_trimmed; + reasoning.mode = reasoning_mode::FORCED_OPEN; } } } - if (result.markers.reasoning_start.empty() && !result.markers.reasoning_end.empty()) { - result.reasoning = reasoning_mode::DELIMITER; - LOG_DBG("R2: Delimiter-based reasoning detected (empty start, end: '%s')\n", - result.markers.reasoning_end.c_str()); + if (reasoning.start.empty() && !reasoning.end.empty()) { + reasoning.mode = reasoning_mode::DELIMITER; } // Check for FORCED_CLOSED: when enable_thinking=false produces both start and end markers, @@ -360,33 +352,30 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template // Both should end with the assistant role marker // Check if output_A has both reasoning_start and reasoning_end markers // while output_B has only reasoning_start - if (!result.markers.reasoning_start.empty()) { + if (!reasoning.start.empty()) { // Check if output_A contains both start and end markers - bool A_has_start = output_A.find(result.markers.reasoning_start) != std::string::npos; - bool A_has_end = !result.markers.reasoning_end.empty() && - output_A.find(result.markers.reasoning_end) != std::string::npos; + bool A_has_start = output_A.find(reasoning.start) != std::string::npos; + bool A_has_end = !reasoning.end.empty() && output_A.find(reasoning.end) != std::string::npos; // Check if output_B contains only the start marker (and not the end marker) - bool B_has_start = output_B.find(result.markers.reasoning_start) != std::string::npos; - bool B_has_end = !result.markers.reasoning_end.empty() && - output_B.find(result.markers.reasoning_end) != std::string::npos; + bool B_has_start = output_B.find(reasoning.start) != std::string::npos; + bool B_has_end = !reasoning.end.empty() && output_B.find(reasoning.end) != std::string::npos; // For FORCED_CLOSED: A should have both, B should have only start if (A_has_start && A_has_end && B_has_start && !B_has_end) { - result.reasoning = reasoning_mode::FORCED_CLOSED; - LOG_DBG("R2: Detected forced-closed reasoning\n"); + reasoning.mode = reasoning_mode::FORCED_CLOSED; } - } else if (!result.markers.reasoning_end.empty()) { + } else if (!reasoning.end.empty()) { // We might not have detected the reasoning open marker until now, // but this is another chance to do so auto diff = comparison->diff; auto diff_rt = trim_whitespace(diff.right); auto diff_lt = trim_whitespace(diff.left); - if (diff_rt.empty() && diff_lt == result.markers.reasoning_end) { + if (diff_rt.empty() && diff_lt == reasoning.end) { auto seg = segmentize_markers(trim_whitespace(diff.prefix)); if (!seg.empty() && seg[seg.size() - 1].type == MARKER) { // this is FORCED_CLOSED - result.markers.reasoning_start = seg[seg.size() - 1].value; - result.reasoning = reasoning_mode::FORCED_CLOSED; + reasoning.start = seg[seg.size() - 1].value; + reasoning.mode = reasoning_mode::FORCED_CLOSED; } } } @@ -394,21 +383,21 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template // Check for slash-in-tag pattern: vs // diff shows: suffix="think>", left="/", right="" (or vice versa) - if (result.markers.reasoning_start.empty() && result.markers.reasoning_end.empty()) { + if (reasoning.start.empty() && reasoning.end.empty()) { if (diff.right.empty() && trim_whitespace(diff.left) == "/") { auto seg_A = segmentize_markers(trim_trailing_whitespace(comparison->output_A)); auto seg_B = segmentize_markers(trim_trailing_whitespace(comparison->output_B)); if (!seg_A.empty() && !seg_B.empty() && seg_A[seg_A.size() - 1].type == segment_type::MARKER && seg_B[seg_B.size() - 1].type == segment_type::MARKER) { - result.reasoning = reasoning_mode::FORCED_CLOSED; - result.markers.reasoning_start = seg_B[seg_B.size() - 1].value; - result.markers.reasoning_end = seg_A[seg_A.size() - 1].value; + reasoning.mode = reasoning_mode::FORCED_CLOSED; + reasoning.start = seg_B[seg_B.size() - 1].value; + reasoning.end = seg_A[seg_A.size() - 1].value; } } } } -void differential_analyzer::compare_reasoning_scope(const common_chat_template & tmpl, diff_analysis_result & result) { +void differential_analyzer::compare_reasoning_scope(const common_chat_template & tmpl, reasoning_analysis & reasoning) { json assistant_reasoning_content = json{ { "role", "assistant" }, { "content", "Here is my response." }, @@ -433,23 +422,18 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_reasoning_tools }); }); if (!comparison) { - LOG_DBG("R3: Template application failed\n"); + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); return; } - const auto & diff = comparison->diff; - std::string reasoning_content = "Let me think."; - LOG_DBG("R3 diff - prefix: '%s', suffix: '%s', left: '%s', right: '%s'\n", diff.prefix.c_str(), diff.suffix.c_str(), - diff.left.c_str(), diff.right.c_str()); - // Check if reasoning only appears in variant B (with tools) bool reasoning_in_A = comparison->output_A.find(reasoning_content) != std::string::npos; bool reasoning_in_B = comparison->output_B.find(reasoning_content) != std::string::npos; if (!reasoning_in_A && reasoning_in_B) { - result.reasoning = reasoning_mode::TOOLS_ONLY; + reasoning.mode = reasoning_mode::TOOLS_ONLY; LOG_DBG("R3: Detected TOOLS_ONLY reasoning mode\n"); // Extract reasoning markers from output_B @@ -464,8 +448,7 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & for (auto & segment : segments_before) { if (segment.type == segment_type::MARKER) { - result.markers.reasoning_start = segment.value; - LOG_DBG("R3: Found reasoning_start: '%s'\n", result.markers.reasoning_start.c_str()); + reasoning.start = segment.value; break; } } @@ -477,30 +460,25 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & if (!after_reasoning.empty()) { // Try to find matching end marker - if (!result.markers.reasoning_start.empty()) { + if (!reasoning.start.empty()) { auto segments = segmentize_markers(after_reasoning); for (auto & segment : segments) { if (segment.type == segment_type::MARKER) { - result.markers.reasoning_end = segment.value; + reasoning.end = segment.value; break; } } - if (!result.markers.reasoning_end.empty()) { - LOG_DBG("R3: Found reasoning_end (matched): '%s'\n", result.markers.reasoning_end.c_str()); - } } } } } } -void differential_analyzer::analyze_content(const common_chat_template & tmpl, diff_analysis_result & result) { +content_analysis differential_analyzer::analyze_content(const common_chat_template & tmpl, const reasoning_analysis & reasoning) { LOG_DBG(ANSI_ORANGE "Phase 2: Content analysis\n" ANSI_RESET); - compare_content_values(tmpl, result); -} - -void differential_analyzer::compare_content_values(const common_chat_template & tmpl, diff_analysis_result & result) { + content_analysis result; + json assistant_content_only = json{ { "role", "assistant" }, { "content", "Response text" } @@ -533,8 +511,7 @@ void differential_analyzer::compare_content_values(const common_chat_template & }); if (!comparison_with_tools || !comparison_with_reasoning) { - LOG_DBG("C1: Template application failed\n"); - return; + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); } const auto & diff_tools = comparison_with_tools->diff; @@ -548,52 +525,115 @@ void differential_analyzer::compare_content_values(const common_chat_template & if (trim_whitespace(diff_reasoning.left) == response || (segments.size() == 2 && trim_whitespace(segments[0].value) == response)) { // We only have the content text in the diff (possibly with a stray EOG marker), so no markers - LOG_DBG("C1: No content markers\n"); - result.content = content_mode::PLAIN; + result.mode = content_mode::PLAIN; found_plain_content = true; - } else if (result.reasoning != reasoning_mode::NONE && !result.markers.reasoning_end.empty() && - diff_reasoning.left.find(result.markers.reasoning_end) != std::string::npos) { + } else if (reasoning.mode != reasoning_mode::NONE && !reasoning.end.empty() && + diff_reasoning.left.find(reasoning.end) != std::string::npos) { std::string post_closed_reasoning = diff_reasoning.left.substr( - diff_reasoning.left.find(result.markers.reasoning_end) + result.markers.reasoning_end.length()); + diff_reasoning.left.find(reasoning.end) + reasoning.end.length()); if (trim_whitespace(post_closed_reasoning) == "Response text") { LOG_DBG("C1: No content markers after stripping reasoning close marker\n"); - result.content = content_mode::PLAIN; + result.mode = content_mode::PLAIN; found_plain_content = true; } } } if (!found_plain_content) { std::string rdiff = diff_reasoning.left; - if (!result.markers.reasoning_end.empty() && rdiff.find(result.markers.reasoning_end) != std::string::npos) { - rdiff = rdiff.substr(rdiff.find(result.markers.reasoning_end) + result.markers.reasoning_end.length()); + if (!reasoning.end.empty() && rdiff.find(reasoning.end) != std::string::npos) { + rdiff = rdiff.substr(rdiff.find(reasoning.end) + reasoning.end.length()); } // Take the more promising diff std::string pure_content = rdiff.length() > diff_tools.left.length() ? rdiff : diff_tools.left; size_t pos = pure_content.find("Response text"); if (pos == std::string::npos) { - LOG_DBG("C1: Error: response text not found - improper template application?"); - return; + LOG_DBG(ANSI_ORANGE "%s: Error: response text not found - improper template application?\n" ANSI_RESET, __func__); + return result; } - result.markers.content_start = trim_leading_whitespace(pure_content.substr(0, pos)); - result.markers.content_end = - trim_leading_whitespace(pure_content.substr(pos + 13)); // 13 - len of "Response text" + result.start = trim_leading_whitespace(pure_content.substr(0, pos)); + result.end = trim_leading_whitespace(pure_content.substr(pos + 13)); // 13 - len of "Response text" // TODO: WRAPPED_WITH_REASONING } // Determine content mode - if (!result.markers.content_start.empty() || !result.markers.content_end.empty()) { - result.content = content_mode::ALWAYS_WRAPPED; - LOG_DBG("C1: Content is ALWAYS_WRAPPED\n"); + if (!result.start.empty() || !result.end.empty()) { + result.mode = content_mode::ALWAYS_WRAPPED; // TODO: END_DELIMITED content mode - delimited at end but not at start? } + + return result; } -void differential_analyzer::analyze_tool_call_format(const std::string & haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - diff_analysis_result & result) { +tool_analysis differential_analyzer::analyze_tools(const common_chat_template & tmpl, + const jinja::caps & caps, + const reasoning_analysis & reasoning) { + tool_analysis result; + LOG_DBG(ANSI_ORANGE "Phase 3: Tool call analysis\n" ANSI_RESET); + + result.format = analyze_tool_calls(tmpl, reasoning); + + if (result.format.mode != tool_format::NONE && result.format.mode != tool_format::JSON_NATIVE) { + if (caps.supports_parallel_tool_calls) { + check_per_call_markers(tmpl, result.format); + } + result.function = extract_function_markers(tmpl, result.format); + if (result.format.mode == tool_format::TAG_WITH_TAGGED) { + result.arguments = analyze_arguments(tmpl, result); + } + extract_argument_separator(tmpl, result.arguments); + extract_args_markers(tmpl, result, result.arguments); + result.call_id = extract_call_id_markers(tmpl, result.format); + } + + return result; +} + +tool_format_analysis differential_analyzer::analyze_tool_calls(const common_chat_template & tmpl, + const reasoning_analysis & reasoning) { + json assistant_no_tools = json{ + { "role", "assistant" }, + { "content", "Response." } + }; + + json assistant_with_tools = json{ + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", json::array({ first_tool_call }) } + }; + + template_params params; + params.messages = json::array({ user_msg, assistant_no_tools }); + params.tools = tools; + params.add_generation_prompt = false; + params.enable_thinking = true; + + auto comparison = compare_variants( + tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_tools }); }); + + if (!comparison) { + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); + return tool_format_analysis(); + } + + const auto & diff = comparison->diff; + + std::string tool_section = diff.right; + + if (tool_section.empty()) { + return tool_format_analysis(); + } + + return analyze_tool_call_format(tool_section, "foofoo", "first", reasoning); +} + +tool_format_analysis differential_analyzer::analyze_tool_call_format(const std::string & haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle, + const reasoning_analysis & reasoning) { + tool_format_analysis result; + if (fun_name_needle.empty() || arg_name_needle.empty() || haystack.empty()) { - return; + return result; } auto in_json_haystack = [&haystack](const std::string & needle) -> bool { @@ -618,43 +658,44 @@ void differential_analyzer::analyze_tool_call_format(const std::string & hays if (in_json_haystack(fun_name_needle)) { // no need to check further, we're in JSON land - result.tools = tool_format::JSON_NATIVE; + result.mode = tool_format::JSON_NATIVE; } else if (in_json_haystack(arg_name_needle)) { - result.tools = tool_format::TAG_WITH_JSON; + result.mode = tool_format::TAG_WITH_JSON; } else { - result.tools = tool_format::TAG_WITH_TAGGED; + result.mode = tool_format::TAG_WITH_TAGGED; } // first, remove any reasoning markers std::string clean_haystack = haystack; - if (!result.markers.reasoning_start.empty()) { - auto pos = haystack.find(result.markers.reasoning_start); + if (!reasoning.start.empty()) { + auto pos = haystack.find(reasoning.start); if (pos != std::string::npos) { - clean_haystack = haystack.substr(0, pos) + haystack.substr(pos + result.markers.reasoning_start.length()); + clean_haystack = haystack.substr(0, pos) + haystack.substr(pos + reasoning.start.length()); } } - if (!result.markers.reasoning_end.empty()) { - auto pos = clean_haystack.find(result.markers.reasoning_end); + if (!reasoning.end.empty()) { + auto pos = clean_haystack.find(reasoning.end); if (pos != std::string::npos) { - clean_haystack = - clean_haystack.substr(0, pos) + clean_haystack.substr(pos + result.markers.reasoning_end.length()); + clean_haystack = clean_haystack.substr(0, pos) + clean_haystack.substr(pos + reasoning.end.length()); } } - if (result.tools == tool_format::JSON_NATIVE) { + if (result.mode == tool_format::JSON_NATIVE) { analyze_tool_call_format_json_native(clean_haystack, fun_name_needle, arg_name_needle, result); } else { analyze_tool_call_format_non_json(clean_haystack, fun_name_needle, result); } // always relax whitespace requirements on ending markers since they don't influence content - result.markers.tool_section_end = trim_whitespace(result.markers.tool_section_end); - result.markers.per_call_end = trim_whitespace(result.markers.per_call_end); + result.section_end = trim_whitespace(result.section_end); + result.per_call_end = trim_whitespace(result.per_call_end); + + return result; } void differential_analyzer::analyze_tool_call_format_json_native(const std::string & clean_haystack, const std::string & fun_name_needle, const std::string & arg_name_needle, - diff_analysis_result & result) { + tool_format_analysis & format) { // we might not have the typical OpenAI tool calling structure int json_start = clean_haystack.find_first_of('{'); int json_end = clean_haystack.find_last_of('}'); @@ -662,28 +703,28 @@ void differential_analyzer::analyze_tool_call_format_json_native(const std::stri json call_struct = json::parse(cut); auto register_field = [&](const std::string & prefix, const nlohmann::detail::iteration_proxy_value & subel) { if (subel.value().is_string() && std::string(subel.value()).find("call0000") != std::string::npos) { - result.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); + format.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); } else if (subel.value().is_string() && std::string(subel.value()) == fun_name_needle) { - result.name_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); + format.name_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); } else if (subel.value().dump().find(arg_name_needle) != std::string::npos) { // handle both string and JSON obj variants - result.args_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); + format.args_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); } else if (subel.key().find("id") != std::string::npos) { // heuristics for generated id field - result.gen_id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); + format.gen_id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key(); } }; for (const auto & el : call_struct.items()) { if (el.key() == fun_name_needle) { - result.fun_name_is_key = true; + format.fun_name_is_key = true; // When function name is the key, there's no name field and args are direct - result.name_field.clear(); - result.args_field.clear(); + format.name_field.clear(); + format.args_field.clear(); // Don't register this element - the function name IS the key, not a field } else { if (el.value().is_object() && el.value().dump().find(arg_name_needle) == std::string::npos) { // not the args object - result.function_field = el.key(); + format.function_field = el.key(); for (const auto & subel : el.value().items()) { register_field(el.key(), subel); } @@ -700,11 +741,11 @@ void differential_analyzer::analyze_tool_call_format_json_native(const std::stri if (json_start > 0 && space_or_bracket(true, clean_haystack[json_start - 1])) { for (--json_start; space_or_bracket(true, clean_haystack[json_start]) && json_start >= 0; json_start--) { if (clean_haystack[json_start] == '[') { - result.tools_array_wrapped = true; + format.tools_array_wrapped = true; break; } } - if (!result.tools_array_wrapped) { + if (!format.tools_array_wrapped) { json_start++; // we ate into the last pre-json character } } @@ -716,35 +757,35 @@ void differential_analyzer::analyze_tool_call_format_json_native(const std::stri } std::vector> located_params; - if (!result.name_field.empty()) { - located_params.push_back({ clean_haystack.find(result.name_field), result.name_field }); + if (!format.name_field.empty()) { + located_params.push_back({ clean_haystack.find(format.name_field), format.name_field }); } - if (!result.args_field.empty()) { - located_params.push_back({ clean_haystack.find(result.args_field), result.args_field }); + if (!format.args_field.empty()) { + located_params.push_back({ clean_haystack.find(format.args_field), format.args_field }); } - if (!result.id_field.empty()) { - located_params.push_back({ clean_haystack.find(result.id_field), result.id_field }); + if (!format.id_field.empty()) { + located_params.push_back({ clean_haystack.find(format.id_field), format.id_field }); } - if (!result.gen_id_field.empty()) { - located_params.push_back({ clean_haystack.find(result.gen_id_field), result.gen_id_field }); + if (!format.gen_id_field.empty()) { + located_params.push_back({ clean_haystack.find(format.gen_id_field), format.gen_id_field }); } std::sort(located_params.begin(), located_params.end()); for (auto & pair : located_params) { - result.parameter_order.push_back(pair.second); + format.parameter_order.push_back(pair.second); } // we can immediately extract tool calling markers too - result.markers.tool_section_start = trim_leading_whitespace(clean_haystack.substr(0, json_start)); - result.markers.tool_section_end = trim_whitespace(clean_haystack.substr(json_end)); + format.section_start = trim_leading_whitespace(clean_haystack.substr(0, json_start)); + format.section_end = trim_whitespace(clean_haystack.substr(json_end)); // When tools_array_wrapped is true, the closing bracket is part of the array structure, // not a separate section end marker. Clear tool_section_end to avoid duplicate brackets. - if (result.tools_array_wrapped && result.markers.tool_section_end == "]") { - result.markers.tool_section_end.clear(); + if (format.tools_array_wrapped && format.section_end == "]") { + format.section_end.clear(); } } void differential_analyzer::analyze_tool_call_format_non_json(const std::string & clean_haystack, const std::string & fun_name_needle, - diff_analysis_result & result) { + tool_format_analysis & format) { // we need to split by markers... auto haystack_split = segmentize_markers(trim_leading_whitespace(clean_haystack)); int where_is_nemo = 0; @@ -776,17 +817,15 @@ void differential_analyzer::analyze_tool_call_format_non_json(const std::string if (haystack_split[seg].type == MARKER) { if (!had_marker) { had_marker = true; - result.markers.per_call_start = haystack_split[seg].value + result.markers.per_call_start; + format.per_call_start = haystack_split[seg].value + format.per_call_start; } else { - result.markers.tool_section_start = - haystack_split[seg].value + result.markers.tool_section_start; + format.section_start = haystack_split[seg].value + format.section_start; } } else { if (had_marker) { - result.markers.tool_section_start = - haystack_split[seg].value + result.markers.tool_section_start; + format.section_start = haystack_split[seg].value + format.section_start; } else { - result.markers.per_call_start = haystack_split[seg].value + result.markers.per_call_start; + format.per_call_start = haystack_split[seg].value + format.per_call_start; } } } @@ -797,15 +836,15 @@ void differential_analyzer::analyze_tool_call_format_non_json(const std::string backtracked_so_far++; if (!had_marker) { had_marker = true; - result.markers.tool_section_end = haystack_split[seg].value + result.markers.tool_section_end; + format.section_end = haystack_split[seg].value + format.section_end; } else { - result.markers.per_call_end = haystack_split[seg].value + result.markers.per_call_end; + format.per_call_end = haystack_split[seg].value + format.per_call_end; } } else { if (had_marker) { - result.markers.per_call_end = haystack_split[seg].value + result.markers.per_call_end; + format.per_call_end = haystack_split[seg].value + format.per_call_end; } else { - result.markers.tool_section_end = haystack_split[seg].value + result.markers.tool_section_end; + format.section_end = haystack_split[seg].value + format.section_end; } } if (backtracked_so_far >= how_many_markers) { @@ -814,19 +853,19 @@ void differential_analyzer::analyze_tool_call_format_non_json(const std::string } } else { for (int seg = 0; seg < where_is_nemo; seg++) { - result.markers.tool_section_start += haystack_split[seg].value; + format.section_start += haystack_split[seg].value; } for (size_t seg = haystack_split.size() - 1; seg > (size_t) where_is_nemo; seg--) { - result.markers.tool_section_end = haystack_split[seg].value + result.markers.tool_section_end; + format.section_end = haystack_split[seg].value + format.section_end; if (haystack_split[seg].type == segment_type::MARKER) { break; } } } } else { - result.markers.tool_section_start += haystack_split[0].value; + format.section_start += haystack_split[0].value; for (size_t seg = haystack_split.size() - 1; seg > (size_t) where_is_nemo; seg--) { - result.markers.tool_section_end = haystack_split[seg].value + result.markers.tool_section_end; + format.section_end = haystack_split[seg].value + format.section_end; if (haystack_split[seg].type == segment_type::MARKER) { break; } @@ -834,28 +873,7 @@ void differential_analyzer::analyze_tool_call_format_non_json(const std::string } } -void differential_analyzer::analyze_tools(const common_chat_template & tmpl, diff_analysis_result & result) { - LOG_DBG(ANSI_ORANGE "Phase 3: Tool call analysis\n" ANSI_RESET); - analyze_tool_calls(tmpl, result); - - if (result.tools == tool_format::NONE) { - LOG_DBG("T1: No tool support found\n"); - // Continue anyway - we may still have useful markers - } else if (result.tools != tool_format::JSON_NATIVE) { - if (result.supports_parallel_calls) { - check_per_call_markers(tmpl, result); - } - extract_function_markers(tmpl, result); - extract_argument_separator(tmpl, result); - extract_args_markers(tmpl, result); - extract_call_id_markers(tmpl, result); - if (result.tools == tool_format::TAG_WITH_TAGGED) { - analyze_arguments(tmpl, result); - } - } -} - -void differential_analyzer::check_per_call_markers(const common_chat_template & tmpl, diff_analysis_result & result) { +void differential_analyzer::check_per_call_markers(const common_chat_template & tmpl, tool_format_analysis & result) { json assistant_one_tool = json{ { "role", "assistant" }, { "content", "" }, @@ -878,112 +896,23 @@ void differential_analyzer::check_per_call_markers(const common_chat_template & tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_tools }); }); if (!one_vs_two) { - LOG_DBG("T2: Generating double tool call comparison failed\n"); + LOG_DBG(ANSI_ORANGE "%s: Generating double tool call comparison failed\n" ANSI_RESET, __func__); return; } std::string second_tool_content = trim_leading_whitespace(one_vs_two->diff.right); - if (!result.markers.tool_section_start.empty() && - second_tool_content.find(result.markers.tool_section_start) == 0) { - result.markers.per_call_start = result.markers.tool_section_start; - result.markers.per_call_end = result.markers.tool_section_end; - result.markers.tool_section_start.clear(); - result.markers.tool_section_end.clear(); + if (!result.section_start.empty() && + second_tool_content.find(result.section_start) == 0) { + result.per_call_start = result.section_start; + result.per_call_end = result.section_end; + result.section_start.clear(); + result.section_end.clear(); } } -void differential_analyzer::analyze_tool_calls(const common_chat_template & tmpl, diff_analysis_result & result) { - json assistant_no_tools = json{ - { "role", "assistant" }, - { "content", "Response." } - }; +tool_function_analysis differential_analyzer::extract_function_markers(const common_chat_template & tmpl, const tool_format_analysis & analysis) { + tool_function_analysis result; - json assistant_with_tools = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call }) } - }; - - template_params params; - params.messages = json::array({ user_msg, assistant_no_tools }); - params.tools = tools; - params.add_generation_prompt = false; - params.enable_thinking = true; - - auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_tools }); }); - - if (!comparison) { - LOG_DBG("T1: Template application failed\n"); - return; - } - - const auto & diff = comparison->diff; - LOG_DBG("T1 diff - prefix: '%s', suffix: '%s'\n", diff.prefix.c_str(), diff.suffix.c_str()); - LOG_DBG("T1 diff - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); - - std::string tool_section = diff.right; - - if (tool_section.empty()) { - return; - } - - analyze_tool_call_format(tool_section, "foofoo", "first", result); - - LOG_DBG("T1: tool_section_start='%s', tool_section_end='%s'\n", result.markers.tool_section_start.c_str(), - result.markers.tool_section_end.c_str()); -} - -void differential_analyzer::extract_call_separator(const common_chat_template & tmpl, - diff_analysis_result & result, - std::string & second_call_content) { - json assistant_one_call = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call }) } - }; - - json assistant_two_calls = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call, second_tool_call }) } - }; - - template_params params; - params.messages = json::array({ user_msg, assistant_one_call }); - params.tools = tools; - params.add_generation_prompt = false; - params.enable_thinking = true; - - auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_calls }); }); - - if (!comparison) { - LOG_DBG("T2: Template application failed\n"); - return; - } - - const auto & diff = comparison->diff; - LOG_DBG("T2 diff - prefix: '%s', suffix: '%s'\n", diff.prefix.c_str(), diff.suffix.c_str()); - LOG_DBG("T2 diff - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); - - if (!diff.right.empty()) { - std::string first_func_name = "foofoo"; - std::string second_func_name = "barbar"; - - std::string separator = until_common_prefix(diff.right, first_func_name, second_func_name); - result.markers.call_separator = trim_whitespace(separator); - - LOG_DBG("T2: call_separator='%s'\n", result.markers.call_separator.c_str()); - - result.supports_parallel_calls = true; - second_call_content = diff.right; - - LOG_DBG("T2: second_call_content='%s', supports_parallel_calls=true\n", second_call_content.c_str()); - } -} - -void differential_analyzer::extract_function_markers(const common_chat_template & tmpl, diff_analysis_result & result) { json assistant_nocall = json{ { "role", "assistant" }, { "content", "BBBB" }, @@ -1011,8 +940,8 @@ void differential_analyzer::extract_function_markers(const common_chat_template tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_barbar }); }); if (!comparison) { - LOG_DBG("T3: Template application failed\n"); - return; + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); + return result; } const auto & diff = comparison->diff; @@ -1021,26 +950,26 @@ void differential_analyzer::extract_function_markers(const common_chat_template if (diff.left.find("foofoo") != std::string::npos && diff.right.find("barbar") != std::string::npos) { std::string prefix_marker; - if (!result.markers.per_call_start.empty()) { - prefix_marker = result.markers.per_call_start; + if (!analysis.per_call_start.empty()) { + prefix_marker = analysis.per_call_start; } else { - prefix_marker = result.markers.tool_section_start; + prefix_marker = analysis.section_start; } if (!prefix_marker.empty() && diff.prefix.rfind(prefix_marker) != std::string::npos) { - result.markers.func_name_prefix = + result.name_prefix = diff.prefix.substr(diff.prefix.rfind(prefix_marker) + prefix_marker.size()); } auto seg = segmentize_markers(diff.left); for (const auto & s : seg) { if (s.value.find("foofoo") == std::string::npos) { - result.markers.func_name_prefix += s.value; + result.name_prefix += s.value; } else { size_t pos = s.value.find("foofoo"); std::string pre = s.value.substr(0, pos); std::string post = s.value.substr(pos + 6); // 6 = len("foofoo") - result.markers.func_name_prefix += pre; - result.markers.func_name_suffix += post; + result.name_prefix += pre; + result.name_suffix += post; break; } } @@ -1050,7 +979,7 @@ void differential_analyzer::extract_function_markers(const common_chat_template size_t stop_internal_pos = 0; for (const auto & ss : seg_suf) { bool has_needle = false; - if (result.tools == tool_format::TAG_WITH_JSON) { + if (analysis.mode == tool_format::TAG_WITH_JSON) { has_needle = (ss.type == segment_type::TEXT && ss.value.find_first_of("{[") != std::string::npos); if (has_needle) { stop_internal_pos = ss.value.find_first_of("{["); @@ -1066,7 +995,7 @@ void differential_analyzer::extract_function_markers(const common_chat_template stop++; } if (stop < seg_suf.size() - 1) { - if (result.tools == tool_format::TAG_WITH_TAGGED) { + if (analysis.mode == tool_format::TAG_WITH_TAGGED) { size_t how_far = 0; if (stop > 0) { if (seg_suf[stop].type == segment_type::MARKER) { @@ -1075,24 +1004,24 @@ void differential_analyzer::extract_function_markers(const common_chat_template how_far = stop - 1; } for (size_t i = 0; i < how_far; i++) { - result.markers.func_name_suffix += seg_suf[i].value; + result.name_suffix += seg_suf[i].value; } } } else { for (size_t i = 0; i < stop; i++) { - result.markers.func_name_suffix += seg_suf[i].value; + result.name_suffix += seg_suf[i].value; } const std::string & stopper = seg_suf[stop].value; - result.markers.func_name_suffix += stopper.substr(0, stop_internal_pos); + result.name_suffix += stopper.substr(0, stop_internal_pos); } } // now just to find the closer std::string suffix_marker; - if (!result.markers.per_call_end.empty()) { - suffix_marker = result.markers.per_call_end; + if (!analysis.per_call_end.empty()) { + suffix_marker = analysis.per_call_end; } else { - suffix_marker = result.markers.tool_section_end; + suffix_marker = analysis.section_end; } std::string closer_suffix; if (suffix_marker.empty()) { @@ -1106,18 +1035,18 @@ void differential_analyzer::extract_function_markers(const common_chat_template } if (!closer_suffix.empty()) { auto closer_seg = segmentize_markers(closer_suffix); - bool need_to_eat_arg_marker = (result.tools == tool_format::TAG_WITH_TAGGED); + bool need_to_eat_arg_marker = (analysis.mode == tool_format::TAG_WITH_TAGGED); size_t last_arg_seg = closer_seg.size() - 1; for (int i = (int) closer_seg.size() - 1; i >= 0; i--) { if (closer_seg[i].value.find("YYYY") != std::string::npos) { last_arg_seg = i; } } - if (result.tools == tool_format::TAG_WITH_JSON) { + if (analysis.mode == tool_format::TAG_WITH_JSON) { const auto & entire_seg = closer_seg[last_arg_seg].value; size_t pos = entire_seg.find_last_of("}]"); if (pos != std::string::npos && pos < entire_seg.size() - 1) { - result.markers.func_close = trim_leading_whitespace(entire_seg.substr(pos + 1)); + result.close = trim_leading_whitespace(entire_seg.substr(pos + 1)); } } for (size_t i = last_arg_seg + 1; i < closer_seg.size(); i++) { @@ -1125,23 +1054,204 @@ void differential_analyzer::extract_function_markers(const common_chat_template if (need_to_eat_arg_marker) { need_to_eat_arg_marker = false; } else { - result.markers.func_close += closer_seg[i].value; + result.close += closer_seg[i].value; } } else if (!need_to_eat_arg_marker) { - result.markers.func_close += closer_seg[i].value; + result.close += closer_seg[i].value; } } } - result.markers.func_close = trim_leading_whitespace(result.markers.func_close); + result.close = trim_leading_whitespace(result.close); + } + return result; +} - LOG_DBG("T3: func_name_prefix='%s', func_name_suffix='%s', func_close='%s'\n", - result.markers.func_name_prefix.c_str(), result.markers.func_name_suffix.c_str(), - result.markers.func_close.c_str()); +tool_arguments_analysis differential_analyzer::analyze_arguments(const common_chat_template & tmpl, const tool_analysis & tool_analysis) { + LOG_DBG(ANSI_ORANGE "Phase 4: Argument analysis\n" ANSI_RESET); + + tool_arguments_analysis result; + + extract_argument_name_markers(tmpl, result); + extract_argument_value_markers(tmpl, tool_analysis, result); + + return result; +} + +void differential_analyzer::extract_argument_name_markers(const common_chat_template & tmpl, + tool_arguments_analysis & args_analysis) { + json assistant_first_arg = json{ + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", json::array({ first_tool_call_one_arg }) } + }; + + json assistant_second_arg = json{ + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", json::array({ first_tool_call_other_arg }) } + }; + + template_params params; + params.messages = json::array({ user_msg, assistant_first_arg }); + params.tools = tools; + params.add_generation_prompt = false; + params.enable_thinking = true; + + auto comparison = compare_variants( + tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_second_arg }); }); + + if (!comparison) { + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); + return; + } + + const auto & diff = comparison->diff; + + if (!diff.left.empty() && !diff.right.empty()) { + size_t common_len = 0; + size_t min_len = std::min(diff.left.length(), diff.right.length()); + while (common_len < min_len && diff.left[common_len] == diff.right[common_len]) { + common_len++; + } + + if (common_len > 0) { // we have a marker structure with the name *inside* the marker + std::string common_prefix = diff.left.substr(0, common_len); + std::string left_remainder = diff.left.substr(common_len); + std::string right_remainder = diff.right.substr(common_len); + size_t left_close = + left_remainder.find_first_of("\"X"); // because arg-val is XXXX, can be quoted or unquoted + size_t right_close = right_remainder.find_first_of("\"Y"); // here arg-val is YYYY + + if (left_close != std::string::npos && right_close != std::string::npos) { + std::string left_name = left_remainder.substr(0, 5); // 5 = len("first") + std::string right_name = right_remainder.substr(0, 6); // 6 = len("second") + + if (left_name == "first" && right_name == "second") { + args_analysis.name_prefix = trim_whitespace(common_prefix); + std::string suffix_left = left_remainder.substr(5, left_close - 5); + std::string suffix_right = right_remainder.substr(6, right_close - 6); + if (suffix_left == suffix_right) { + args_analysis.name_suffix = trim_leading_whitespace(suffix_left); + } + } + } + } else if (diff.left.substr(0, 5) == "first" && diff.right.substr(0, 6) == "second") { + // we most likely have actual markers for argument names + auto pre_seg = segmentize_markers(diff.prefix); + for (int i = pre_seg.size() - 1; i >= 0; i--) { + args_analysis.name_prefix = args_analysis.name_prefix + pre_seg[i].value; + if (pre_seg[i].type == segment_type::MARKER) { + break; + } + } + auto left_seg = segmentize_markers(diff.left); + if (left_seg.size() == 1) { // only the name + maybe extra whitespace / normal chars in differing part + args_analysis.name_suffix = diff.left.substr(5); + auto suf_seg= segmentize_markers(diff.suffix); + for (size_t i = 0; i < suf_seg.size(); i++) { + args_analysis.name_suffix += suf_seg[i].value; + if (suf_seg[i].type == segment_type::MARKER) { + if (i < suf_seg.size() - 2 && suf_seg[i + 1].type == segment_type::TEXT && + trim_whitespace(suf_seg[i + 1].value).empty()) { + // we need to include post-marker whitespace/newlines as well + args_analysis.name_suffix += suf_seg[i + 1].value; + } + break; + } + } + } else { + for (size_t i = 0; i < left_seg.size(); i++) { + std::string to_add; + if (i == 0) { + to_add = left_seg[i].value.substr(5); + } else { + to_add = left_seg[i].value; + } + args_analysis.name_suffix += to_add; + if (left_seg[i].type == segment_type::MARKER) { + if (i < left_seg.size() - 2 && left_seg[i + 1].type == segment_type::TEXT && + trim_whitespace(left_seg[i + 1].value).empty()) { + // we need to include post-marker whitespace/newlines as well + args_analysis.name_suffix += left_seg[i + 1].value; + } + break; + } + } + } + } + } +} + +void differential_analyzer::extract_argument_value_markers(const common_chat_template & tmpl, + const tool_analysis & analysis, + tool_arguments_analysis & args_analysis) { + json assistant_val_X = json{ + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", json::array({ first_tool_call_one_arg }) } + }; + + json assistant_val_Y = json{ + { "role", "assistant" }, + { "content", "" }, + { "tool_calls", json::array({ first_tool_call_one_arg_other_val }) } + }; + + template_params params; + params.messages = json::array({ user_msg, assistant_val_X }); + params.tools = tools; + params.add_generation_prompt = false; + params.enable_thinking = true; + + auto comparison = compare_variants( + tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_val_Y }); }); + + if (!comparison) { + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); + return; + } + + const auto & diff = comparison->diff; + + if (diff.left == "XXXX" && diff.right == "YYYY") { + std::string arg_name_ending = "first" + args_analysis.name_suffix; + std::string prefix = diff.prefix; + if (prefix.rfind(arg_name_ending) != std::string::npos) { + prefix = prefix.substr(prefix.rfind(arg_name_ending) + arg_name_ending.size()); + } + if (!prefix.empty()) { + auto seg_pre = segmentize_markers(prefix); + for (int i = seg_pre.size() - 1; i >= 0; i--) { + args_analysis.value_prefix = seg_pre[i].value + args_analysis.value_prefix; + if (seg_pre[i].type == segment_type::MARKER) { + break; + } + } + } + + std::string value_suffix = diff.suffix; + if (!analysis.function.close.empty()) { + size_t func_close_pos = value_suffix.find(analysis.function.close); + if (func_close_pos != std::string::npos) { + value_suffix = value_suffix.substr(0, func_close_pos); + } + } else if (!analysis.format.per_call_end.empty() || !analysis.format.section_end.empty()) { + std::string end_marker = + !analysis.format.per_call_end.empty() ? analysis.format.per_call_end : analysis.format.section_end; + size_t end_marker_pos = value_suffix.find(end_marker); + if (end_marker_pos != std::string::npos) { + value_suffix = value_suffix.substr(0, end_marker_pos); + } + } + value_suffix = trim_leading_whitespace(value_suffix); + if (!value_suffix.empty()) { + args_analysis.value_suffix = value_suffix; + } } } void differential_analyzer::extract_argument_separator(const common_chat_template & tmpl, - diff_analysis_result & result) { + tool_arguments_analysis & args_analysis) { json assistant_one_arg = json{ { "role", "assistant" }, { "content", "" }, @@ -1164,31 +1274,30 @@ void differential_analyzer::extract_argument_separator(const common_chat_templat tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_args }); }); if (!comparison) { - LOG_DBG("T4: Template application failed\n"); + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); return; } const auto & diff = comparison->diff; - LOG_DBG("T4 diff - suffix: '%s'\n", diff.suffix.c_str()); - LOG_DBG("T4 diff - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); if (!diff.right.empty()) { std::string separator = until_common_prefix(diff.right, "first", "second"); - result.markers.arg_separator = separator; - LOG_DBG("T4: arg_separator='%s'\n", result.markers.arg_separator.c_str()); + args_analysis.separator = separator; } } -void differential_analyzer::extract_args_markers(const common_chat_template & tmpl, diff_analysis_result & result) { +void differential_analyzer::extract_args_markers(const common_chat_template & tmpl, + const tool_analysis & analysis, + tool_arguments_analysis & args_analysis) { json assistant_no_args = json{ - { "role", "assistant" }, - { "content", "" }, + { "role", "assistant"}, + { "content", "" }, { "tool_calls", json::array({ first_tool_call_zero_args }) } }; json assistant_with_args = json{ - { "role", "assistant" }, - { "content", "" }, + { "role", "assistant"}, + { "content", "" }, { "tool_calls", json::array({ first_tool_call_one_arg }) } }; @@ -1202,19 +1311,15 @@ void differential_analyzer::extract_args_markers(const common_chat_template & tm tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_args }); }); if (!comparison) { - LOG_DBG("T5: Template application failed\n"); + LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); return; } const auto & diff = comparison->diff; - LOG_DBG("T5 diff - suffix: '%s'\n", diff.suffix.c_str()); - LOG_DBG("T5 diff - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); - if (result.markers.args_start.empty() && result.tools != tool_format::JSON_NATIVE) { - std::string prefix_marker = !result.markers.tool_section_start.empty() ? result.markers.tool_section_start : - result.markers.per_call_start; - std::string suffix_marker = - !result.markers.tool_section_end.empty() ? result.markers.tool_section_end : result.markers.per_call_end; + if (analysis.format.mode != tool_format::JSON_NATIVE) { + std::string prefix_marker = !analysis.format.section_start.empty() ? analysis.format.section_start : analysis.format.per_call_start; + std::string suffix_marker = !analysis.format.section_end.empty() ? analysis.format.section_end : analysis.format.per_call_end; // these might happen earlier in the tools section as an example or somewhere else, so we need to find the closest ones size_t prefix_pos = prefix_marker.empty() ? 0 : diff.prefix.rfind(prefix_marker); size_t suffix_pos = suffix_marker.empty() ? diff.suffix.size() : diff.suffix.find(suffix_marker); @@ -1230,15 +1335,15 @@ void differential_analyzer::extract_args_markers(const common_chat_template & tm std::string args_end = after_common_suffix(suffix_cut, "{}", "\"XXXX\"}"); if (!args_start.empty() || !args_end.empty()) { - result.markers.args_start = args_start; - result.markers.args_end = args_end; - LOG_DBG("T5: Custom argument container detected: start='%s', end='%s'\n", args_start.c_str(), - args_end.c_str()); + args_analysis.start = args_start; + args_analysis.end = args_end; } } } -void differential_analyzer::extract_call_id_markers(const common_chat_template & tmpl, diff_analysis_result & result) { +tool_id_analysis differential_analyzer::extract_call_id_markers(const common_chat_template & tmpl, tool_format_analysis & analysis) { + tool_id_analysis result; + json assistant_id1 = json{ { "role", "assistant" }, { "content", "" }, @@ -1261,8 +1366,8 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_id2 }); }); if (!comparison) { - LOG_DBG("T6: Template application failed for call_id detection\n"); - return; + LOG_DBG(ANSI_ORANGE "%s: Template application failed for call_id detection\n" ANSI_RESET, __func__); + return result; } const auto & diff = comparison->diff; @@ -1270,8 +1375,7 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & LOG_DBG("T6 diff (call_id) - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); if (diff.left.empty() && diff.right.empty()) { - LOG_DBG("T6: No call_id difference detected\n"); - return; + return result; } std::string id_value_1 = "call00001"; @@ -1302,8 +1406,7 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & if (args_in_suffix != std::string::npos && (args_in_prefix == std::string::npos || args_in_prefix > diff.prefix.length())) { // Args are in suffix, so call_id is BETWEEN_FUNC_AND_ARGS - result.call_id_pos = call_id_position::BETWEEN_FUNC_AND_ARGS; - LOG_DBG("T6: Detected BETWEEN_FUNC_AND_ARGS position\n"); + result.pos = call_id_position::BETWEEN_FUNC_AND_ARGS; // The prefix ends with: ... // Segmentize to find the call_id_prefix marker @@ -1328,14 +1431,12 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & } if (!marker_before_id.empty()) { - result.markers.call_id_prefix = marker_before_id; - LOG_DBG("T6: call_id_prefix='%s'\n", result.markers.call_id_prefix.c_str()); + result.prefix = marker_before_id; } else { // Fallback: look for the last marker in after_func for (int i = (int) segments.size() - 1; i >= 0; i--) { if (segments[i].type == segment_type::MARKER) { - result.markers.call_id_prefix = segments[i].value; - LOG_DBG("T6: call_id_prefix (fallback)='%s'\n", result.markers.call_id_prefix.c_str()); + result.prefix = segments[i].value; break; } } @@ -1345,8 +1446,8 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & auto suffix_segments = segmentize_markers(diff.suffix); for (size_t i = 0; i < suffix_segments.size(); i++) { if (suffix_segments[i].type == segment_type::MARKER) { - result.markers.call_id_suffix = suffix_segments[i].value; - LOG_DBG("T6: call_id_suffix='%s'\n", result.markers.call_id_suffix.c_str()); + result.suffix = suffix_segments[i].value; + LOG_DBG("T6: call_id_suffix='%s'\n", result.suffix.c_str()); break; } // Stop if we hit the args @@ -1356,8 +1457,7 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & } } else if (args_in_prefix != std::string::npos) { // Args are in prefix, so call_id is POST_ARGS - result.call_id_pos = call_id_position::POST_ARGS; - LOG_DBG("T6: POST_ARGS call_id position detected\n"); + result.pos = call_id_position::POST_ARGS; // Extract markers from between args and the ID std::string after_args = diff.prefix.substr(args_in_prefix); @@ -1367,8 +1467,8 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & auto segments = segmentize_markers(between_args_and_id); for (int i = (int) segments.size() - 1; i >= 0; i--) { if (segments[i].type == segment_type::MARKER) { - result.markers.call_id_prefix = segments[i].value; - LOG_DBG("T6: call_id_prefix='%s'\n", result.markers.call_id_prefix.c_str()); + result.prefix = segments[i].value; + LOG_DBG("T6: call_id_prefix='%s'\n", result.prefix.c_str()); break; } } @@ -1378,23 +1478,20 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & auto suffix_segments = segmentize_markers(diff.suffix); for (const auto & seg : suffix_segments) { if (seg.type == segment_type::MARKER) { - result.markers.call_id_suffix = seg.value; - LOG_DBG("T6: call_id_suffix='%s'\n", result.markers.call_id_suffix.c_str()); + result.suffix = seg.value; break; } } } } else if (func_name_in_suffix != std::string::npos && func_name_in_prefix == std::string::npos) { // Function name is only in suffix - call_id is PRE_FUNC_NAME - result.call_id_pos = call_id_position::PRE_FUNC_NAME; - LOG_DBG("T6: PRE_FUNC_NAME call_id position detected\n"); + result.pos = call_id_position::PRE_FUNC_NAME; // Extract call_id_prefix from prefix (last marker before the common_id_part) auto prefix_segments = segmentize_markers(diff.prefix); for (int i = (int) prefix_segments.size() - 1; i >= 0; i--) { if (prefix_segments[i].type == segment_type::MARKER) { - result.markers.call_id_prefix = prefix_segments[i].value; - LOG_DBG("T6: call_id_prefix='%s'\n", result.markers.call_id_prefix.c_str()); + result.prefix = prefix_segments[i].value; break; } } @@ -1404,210 +1501,20 @@ void differential_analyzer::extract_call_id_markers(const common_chat_template & auto suffix_segments = segmentize_markers(before_func); for (const auto & seg : suffix_segments) { if (seg.type == segment_type::MARKER) { - result.markers.call_id_suffix = seg.value; - LOG_DBG("T6: call_id_suffix='%s'\n", result.markers.call_id_suffix.c_str()); + result.suffix = seg.value; break; } } - } else { - LOG_DBG("T6: Unable to determine call_id position\n"); } // When call_id is detected, per_call_end may have been incorrectly set to include // the call_id_suffix and sample args. Clear it if it starts with call_id_suffix. - if (result.call_id_pos != call_id_position::NONE && !result.markers.call_id_suffix.empty() && - result.markers.per_call_end.find(result.markers.call_id_suffix) == 0) { - result.markers.per_call_end.clear(); - LOG_DBG("T6: Cleared per_call_end (was incorrectly including call_id_suffix)\n"); - } -} - -void differential_analyzer::analyze_arguments(const common_chat_template & tmpl, diff_analysis_result & result) { - LOG_DBG(ANSI_ORANGE "Phase 4: Argument analysis\n" ANSI_RESET); - - extract_argument_name_markers(tmpl, result); - extract_argument_value_markers(tmpl, result); -} - -void differential_analyzer::extract_argument_name_markers(const common_chat_template & tmpl, - diff_analysis_result & result) { - json assistant_first_arg = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call_one_arg }) } - }; - - json assistant_second_arg = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call_other_arg }) } - }; - - template_params params; - params.messages = json::array({ user_msg, assistant_first_arg }); - params.tools = tools; - params.add_generation_prompt = false; - params.enable_thinking = true; - - auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_second_arg }); }); - - if (!comparison) { - LOG_DBG("A1: Template application failed\n"); - return; + if (result.pos != call_id_position::NONE && !result.suffix.empty() && + analysis.per_call_end.find(result.suffix) == 0) { + analysis.per_call_end.clear(); } - const auto & diff = comparison->diff; - LOG_DBG("A1 diff - suffix: '%s', left: '%s', right: '%s'\n", diff.suffix.c_str(), diff.left.c_str(), - diff.right.c_str()); - - if (!diff.left.empty() && !diff.right.empty()) { - size_t common_len = 0; - size_t min_len = std::min(diff.left.length(), diff.right.length()); - while (common_len < min_len && diff.left[common_len] == diff.right[common_len]) { - common_len++; - } - - if (common_len > 0) { // we have a marker structure with the name *inside* the marker - std::string common_prefix = diff.left.substr(0, common_len); - std::string left_remainder = diff.left.substr(common_len); - std::string right_remainder = diff.right.substr(common_len); - size_t left_close = - left_remainder.find_first_of("\"X"); // because arg-val is XXXX, can be quoted or unquoted - size_t right_close = right_remainder.find_first_of("\"Y"); // here arg-val is YYYY - - if (left_close != std::string::npos && right_close != std::string::npos) { - std::string left_name = left_remainder.substr(0, 5); // 5 = len("first") - std::string right_name = right_remainder.substr(0, 6); // 6 = len("second") - - if (left_name == "first" && right_name == "second") { - result.markers.arg_name_prefix = trim_whitespace(common_prefix); - std::string suffix_left = left_remainder.substr(5, left_close - 5); - std::string suffix_right = right_remainder.substr(6, right_close - 6); - if (suffix_left == suffix_right) { - result.markers.arg_name_suffix = trim_leading_whitespace(suffix_left); - } - LOG_DBG("A1: arg_name_prefix='%s', arg_name_suffix='%s'\n", result.markers.arg_name_prefix.c_str(), - result.markers.arg_name_suffix.c_str()); - } - } - } else if (diff.left.substr(0, 5) == "first" && diff.right.substr(0, 6) == "second") { - // we most likely have actual markers for argument names - auto pre_seg = segmentize_markers(diff.prefix); - for (int i = pre_seg.size() - 1; i >= 0; i--) { - result.markers.arg_name_prefix = result.markers.arg_name_prefix + pre_seg[i].value; - if (pre_seg[i].type == segment_type::MARKER) { - break; - } - } - auto left_seg = segmentize_markers(diff.left); - if (left_seg.size() == 1) { // only the name + maybe extra whitespace / normal chars in differing part - result.markers.arg_name_suffix = diff.left.substr(5); - auto suf_seg= segmentize_markers(diff.suffix); - for (size_t i = 0; i < suf_seg.size(); i++) { - result.markers.arg_name_suffix += suf_seg[i].value; - if (suf_seg[i].type == segment_type::MARKER) { - if (i < suf_seg.size() - 2 && suf_seg[i + 1].type == segment_type::TEXT && - trim_whitespace(suf_seg[i + 1].value).empty()) { - // we need to include post-marker whitespace/newlines as well - result.markers.arg_name_suffix += suf_seg[i + 1].value; - } - break; - } - } - } else { - for (size_t i = 0; i < left_seg.size(); i++) { - std::string to_add; - if (i == 0) { - to_add = left_seg[i].value.substr(5); - } else { - to_add = left_seg[i].value; - } - result.markers.arg_name_suffix += to_add; - if (left_seg[i].type == segment_type::MARKER) { - if (i < left_seg.size() - 2 && left_seg[i + 1].type == segment_type::TEXT && - trim_whitespace(left_seg[i + 1].value).empty()) { - // we need to include post-marker whitespace/newlines as well - result.markers.arg_name_suffix += left_seg[i + 1].value; - } - break; - } - } - } - } - } -} - -void differential_analyzer::extract_argument_value_markers(const common_chat_template & tmpl, - diff_analysis_result & result) { - json assistant_val_X = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call_one_arg }) } - }; - - json assistant_val_Y = json{ - { "role", "assistant" }, - { "content", "" }, - { "tool_calls", json::array({ first_tool_call_one_arg_other_val }) } - }; - - template_params params; - params.messages = json::array({ user_msg, assistant_val_X }); - params.tools = tools; - params.add_generation_prompt = false; - params.enable_thinking = true; - - auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_val_Y }); }); - - if (!comparison) { - LOG_DBG("A2: Template application failed\n"); - return; - } - - const auto & diff = comparison->diff; - LOG_DBG("A2 diff - suffix: '%s'\n", diff.suffix.c_str()); - LOG_DBG("A2 diff - left: '%s', right: '%s'\n", diff.left.c_str(), diff.right.c_str()); - - if (diff.left == "XXXX" && diff.right == "YYYY") { - std::string arg_name_ending = "first" + result.markers.arg_name_suffix; - std::string prefix = diff.prefix; - if (prefix.rfind(arg_name_ending) != std::string::npos) { - prefix = prefix.substr(prefix.rfind(arg_name_ending) + arg_name_ending.size()); - } - if (!prefix.empty()) { - auto seg_pre = segmentize_markers(prefix); - for (int i = seg_pre.size() - 1; i >= 0; i--) { - result.markers.arg_value_prefix = seg_pre[i].value + result.markers.arg_value_prefix; - if (seg_pre[i].type == segment_type::MARKER) { - break; - } - } - } - - std::string value_suffix = diff.suffix; - if (!result.markers.func_close.empty()) { - size_t func_close_pos = value_suffix.find(result.markers.func_close); - if (func_close_pos != std::string::npos) { - value_suffix = value_suffix.substr(0, func_close_pos); - } - } else if (!result.markers.per_call_end.empty() || !result.markers.tool_section_end.empty()) { - std::string end_marker = - !result.markers.per_call_end.empty() ? result.markers.per_call_end : result.markers.tool_section_end; - size_t end_marker_pos = value_suffix.find(end_marker); - if (end_marker_pos != std::string::npos) { - value_suffix = value_suffix.substr(0, end_marker_pos); - } - } - value_suffix = trim_leading_whitespace(value_suffix); - if (!value_suffix.empty()) { - result.markers.arg_value_suffix = value_suffix; - } - - LOG_DBG("A2: arg_value_prefix='%s', arg_value_suffix='%s'\n", result.markers.arg_value_prefix.c_str(), - result.markers.arg_value_suffix.c_str()); - } + return result; } void differential_analyzer::collect_preserved_tokens(diff_analysis_result & result) { @@ -1623,23 +1530,24 @@ void differential_analyzer::collect_preserved_tokens(diff_analysis_result & resu } }; - add_token(result.markers.reasoning_start); - add_token(result.markers.reasoning_end); - add_token(result.markers.content_start); - add_token(result.markers.content_end); - add_token(result.markers.tool_section_start); - add_token(result.markers.tool_section_end); - add_token(result.markers.per_call_start); - add_token(result.markers.per_call_end); - add_token(result.markers.func_name_prefix); - add_token(result.markers.func_name_suffix); - add_token(result.markers.func_close); - add_token(result.markers.arg_name_prefix); - add_token(result.markers.arg_name_suffix); - add_token(result.markers.arg_separator); - add_token(result.markers.arg_value_prefix); - add_token(result.markers.arg_value_suffix); - add_token(result.markers.call_id_prefix); - add_token(result.markers.call_id_suffix); - add_token(result.markers.code_block_marker); + add_token(result.reasoning.start); + add_token(result.reasoning.end); + add_token(result.content.start); + add_token(result.content.end); + add_token(result.tools.format.section_start); + add_token(result.tools.format.section_end); + add_token(result.tools.format.per_call_start); + add_token(result.tools.format.per_call_end); + add_token(result.tools.function.name_prefix); + add_token(result.tools.function.name_suffix); + add_token(result.tools.function.close); + add_token(result.tools.arguments.start); + add_token(result.tools.arguments.end); + add_token(result.tools.arguments.name_prefix); + add_token(result.tools.arguments.name_suffix); + add_token(result.tools.arguments.separator); + add_token(result.tools.arguments.value_prefix); + add_token(result.tools.arguments.value_suffix); + add_token(result.tools.call_id.prefix); + add_token(result.tools.call_id.suffix); } diff --git a/common/chat-diff-analyzer.h b/common/chat-diff-analyzer.h index ce729df0e6..c035203923 100644 --- a/common/chat-diff-analyzer.h +++ b/common/chat-diff-analyzer.h @@ -1,6 +1,7 @@ #pragma once #include "chat.h" +#include "jinja/caps.h" #include "nlohmann/json.hpp" #include @@ -15,11 +16,11 @@ using json = nlohmann::ordered_json; // Parameters for template application // ============================================================================ struct template_params { - json messages; - json tools; - bool add_generation_prompt = false; - bool enable_thinking = true; - std::optional extra_context = std::nullopt; + json messages; + json tools; + bool add_generation_prompt = false; + bool enable_thinking = true; + std::optional extra_context = std::nullopt; }; struct diff_split { @@ -35,9 +36,9 @@ struct diff_split { // Result of compare_variants containing diff and original outputs struct compare_variants_result { - diff_split diff; - std::string output_A; - std::string output_B; + diff_split diff; + std::string output_A; + std::string output_B; }; // ============================================================================ @@ -77,29 +78,23 @@ struct marker_registry { std::string arg_separator; // e.g., "", "\n", "," // === Call ID markers (for non-JSON formats with tool call IDs) === - std::string call_id_prefix; // e.g., "[CALL_ID]" (marker before call ID value) - std::string call_id_suffix; // e.g., "" (marker after call ID value, before next section) - - // === Special markers === - std::string code_block_marker; // e.g., "Action:" (for markdown code block format) - std::string code_block_language; // e.g., "json" - std::string function_namespace; // e.g., "functions." (for prefixed-indexed format) + std::string call_id_prefix; // e.g., "[CALL_ID]" (marker before call ID value) + std::string call_id_suffix; // e.g., "" (marker after call ID value, before next section) }; - // ============================================================================ // Analysis Result Enums // ============================================================================ // Reasoning handling mode (derived from R1-R3 comparisons) enum class reasoning_mode { - NONE, // No reasoning markers detected - TAG_BASED, // Standard tag-based: ... - DELIMITER, // Delimiter-based: [BEGIN FINAL RESPONSE] (reasoning ends at delimiter) - FORCED_OPEN, // Template ends with open reasoning tag (empty start, non-empty end) - FORCED_CLOSED,// Template ends with open reasoning tag on enabled thinking but - // with both opened and closed tag for disabled thinking - TOOLS_ONLY // Only reason on tool calls, not on normal content + NONE, // No reasoning markers detected + TAG_BASED, // Standard tag-based: ... + DELIMITER, // Delimiter-based: [BEGIN FINAL RESPONSE] (reasoning ends at delimiter) + FORCED_OPEN, // Template ends with open reasoning tag (empty start, non-empty end) + FORCED_CLOSED, // Template ends with open reasoning tag on enabled thinking but + // with both opened and closed tag for disabled thinking + TOOLS_ONLY // Only reason on tool calls, not on normal content }; inline std::ostream & operator<<(std::ostream & os, const reasoning_mode & mode) { @@ -143,10 +138,10 @@ inline std::ostream & operator<<(std::ostream & os, const content_mode & mode) { // Call ID position in tool calls (for non-JSON formats) enum class call_id_position { - NONE, // No call ID support detected - PRE_FUNC_NAME, // Call ID before function name: [CALL_ID]id[FUNC]name{args} - BETWEEN_FUNC_AND_ARGS, // Call ID between function and args: [FUNC]name[CALL_ID]id{args} - POST_ARGS, // Call ID after arguments: [FUNC]name{args}[CALL_ID]id + NONE, // No call ID support detected + PRE_FUNC_NAME, // Call ID before function name: [CALL_ID]id[FUNC]name{args} + BETWEEN_FUNC_AND_ARGS, // Call ID between function and args: [FUNC]name[CALL_ID]id{args} + POST_ARGS, // Call ID after arguments: [FUNC]name{args}[CALL_ID]id }; inline std::ostream & operator<<(std::ostream & os, const call_id_position & pos) { @@ -166,10 +161,10 @@ inline std::ostream & operator<<(std::ostream & os, const call_id_position & pos // Tool call format classification (derived from T1-T5, A1-A3 comparisons) enum class tool_format { - NONE, // No tool support detected - JSON_NATIVE, // Pure JSON: {"name": "X", "arguments": {...}} - TAG_WITH_JSON, // Tag-based with JSON args: {...} - TAG_WITH_TAGGED, // Tag-based with tagged args: value + NONE, // No tool support detected + JSON_NATIVE, // Pure JSON: {"name": "X", "arguments": {...}} + TAG_WITH_JSON, // Tag-based with JSON args: {...} + TAG_WITH_TAGGED, // Tag-based with tagged args: value }; inline std::ostream & operator<<(std::ostream & os, const tool_format & format) { @@ -187,33 +182,77 @@ inline std::ostream & operator<<(std::ostream & os, const tool_format & format) } } +struct reasoning_analysis { + reasoning_mode mode = reasoning_mode::NONE; + + std::string start; // e.g., "", "[THINK]", "<|START_THINKING|>", "" + std::string end; // e.g., "", "[BEGIN FINAL RESPONSE]", "<|END_THINKING|>" +}; + +struct content_analysis { + content_mode mode = content_mode::PLAIN; + + std::string start; // e.g., "", ">>>all\n", "" + std::string end; // e.g., "", "" + + bool requires_nonnull_content = false; +}; + +struct tool_format_analysis { + tool_format mode = tool_format::NONE; + + std::string section_start; // e.g., "", "[TOOL_CALLS]", "" + std::string section_end; // e.g., "", "" + std::string per_call_start; // e.g., "<|tool_call_begin|>", "" (for multi-call templates) + std::string per_call_end; // e.g., "<|tool_call_end|>", "" + + bool fun_name_is_key = false; // In JSON format function name is JSON key, i.e. { "": { ... arguments ... } } + bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...] + + std::string function_field = "function"; + std::string name_field = "name"; + std::string args_field = "arguments"; + std::string id_field; + std::string gen_id_field; + std::vector parameter_order; +}; + +struct tool_function_analysis { + std::string name_prefix; // e.g., "", "\"", ":0" + std::string close; // e.g., "", "" (for tag-based) +}; + +struct tool_arguments_analysis { + std::string start; // e.g., "<|tool_call_argument_begin|>", "" + std::string end; // e.g., "<|tool_call_argument_end|>", "" + std::string name_prefix; // e.g., "", "\"" + std::string name_suffix; // e.g., ">", "", "\":" + std::string value_prefix; // e.g., "", "", "" + std::string value_suffix; // e.g., "", "", "" + std::string separator; // e.g., "", "\n", "," +}; + +struct tool_id_analysis { + call_id_position pos = call_id_position::NONE; + + std::string prefix; // e.g., "[CALL_ID]" (marker before call ID value) + std::string suffix; // e.g., "" (marker after call ID value, before next section) +}; + +struct tool_analysis { + tool_format_analysis format; + tool_function_analysis function; + tool_arguments_analysis arguments; + tool_id_analysis call_id; +}; + // Complete result of differential analysis struct diff_analysis_result { - // Classification results - reasoning_mode reasoning = reasoning_mode::NONE; - content_mode content = content_mode::PLAIN; - tool_format tools = tool_format::NONE; - - // All extracted markers - marker_registry markers; - - // JSON field names (for JSON-based formats) - bool fun_name_is_key = false; - std::string function_field = "function"; - std::string name_field = "name"; - std::string args_field = "arguments"; - std::string id_field; - std::string gen_id_field; - std::vector parameter_order; - - // Call ID position (for non-JSON formats) - call_id_position call_id_pos = call_id_position::NONE; - - // Flags - bool supports_tools = false; - bool supports_parallel_calls = false; - bool requires_nonnull_content = false; - bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...] + jinja::caps jinja_caps; + reasoning_analysis reasoning; + content_analysis content; + tool_analysis tools; // Preserved tokens for tokenizer (union of all non-empty markers) std::vector preserved_tokens; @@ -227,94 +266,102 @@ class differential_analyzer { static diff_analysis_result analyze(const common_chat_template & tmpl); // Phase-specific analysis (can be called individually for testing) - static void analyze_reasoning(const common_chat_template & tmpl, diff_analysis_result & result); - static void analyze_content(const common_chat_template & tmpl, diff_analysis_result & result); - static void analyze_tools(const common_chat_template & tmpl, diff_analysis_result & result); - static void analyze_arguments(const common_chat_template & tmpl, diff_analysis_result & result); + static reasoning_analysis analyze_reasoning(const common_chat_template & tmpl, bool supports_tools); + static content_analysis analyze_content(const common_chat_template & tmpl, const reasoning_analysis & reasoning); + static tool_analysis analyze_tools(const common_chat_template & tmpl, + const jinja::caps & caps, + const reasoning_analysis & reasoning); // Factorized differential comparison function (public for testing) // Takes base params and a single modifier lambda to create variant B // Returns compare_variants_result containing diff and both outputs, or std::nullopt on failure static std::optional compare_variants( - const common_chat_template & tmpl, - const template_params & params_A, - const std::function & params_modifier); + const common_chat_template & tmpl, + const template_params & params_A, + const std::function & params_modifier); private: // Comparison helpers (implement the comparison matrix from the plan) - // R1: Extract reasoning markers by comparing with/without reasoning_content - static void compare_reasoning_presence(const common_chat_template & tmpl, diff_analysis_result & result); + // 1. Reasoning analysis: + // Look for reasoning markers in rendered content + static void compare_reasoning_presence(const common_chat_template & tmpl, reasoning_analysis & reasoning); - // R2: Detect forced-open reasoning by comparing enable_thinking=false vs true - static void compare_thinking_enabled(const common_chat_template & tmpl, diff_analysis_result & result); + // Compare generation prompt with enable_thinking=true vs false + static void compare_thinking_enabled(const common_chat_template & tmpl, reasoning_analysis & reasoning); - // R3: Detect reasoning scope (content-only vs with tools) - static void compare_reasoning_scope(const common_chat_template & tmpl, diff_analysis_result & result); + // Check if reasoning is always possible or only in tool calls + static void compare_reasoning_scope(const common_chat_template & tmpl, reasoning_analysis & reasoning); - // C1: Extract content markers by comparing different content values - static void compare_content_values(const common_chat_template & tmpl, diff_analysis_result & result); + // 2. Content (fully inside analyze_content mentioned above) - // T1: Analyze the tool calls - static void analyze_tool_calls(const common_chat_template & tmpl, diff_analysis_result & result); + // 3. Tool calls + // a. format + // Extract tool calling 'haystack' for further analysis and delegate further analysis based on format + static tool_format_analysis analyze_tool_calls(const common_chat_template & tmpl, + const reasoning_analysis & reasoning); - // Analyzes a tool call section to determine the format used (pure JSON, function name markers, or full markers) - static void analyze_tool_call_format(const std::string & haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - diff_analysis_result & result); + // Analyze format based on position of function and argument name in needle + static tool_format_analysis analyze_tool_call_format(const std::string & haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle, + const reasoning_analysis & reasoning); - // Helper functions to handle the two branches of analyze_tool_call_format + // Analyze specifics of JSON native format (entire tool call is a JSON object) static void analyze_tool_call_format_json_native(const std::string & clean_haystack, const std::string & fun_name_needle, const std::string & arg_name_needle, - diff_analysis_result & result); + tool_format_analysis & format); + // Analyze specifics of non-JSON native format (tags for function name or for function name and arguments) static void analyze_tool_call_format_non_json(const std::string & clean_haystack, const std::string & fun_name_needle, - diff_analysis_result & result); + tool_format_analysis & format); - // T2: Check if markers are per call or per section - static void check_per_call_markers(const common_chat_template & tmpl, diff_analysis_result & result); + // Check for and extract specific per-call markers for non-native-JSON templates with parallel call support + static void check_per_call_markers(const common_chat_template & tmpl, tool_format_analysis & result); - // T3: Extract call separator; also outputs second_call_content for per-call detection - static void extract_call_separator(const common_chat_template & tmpl, diff_analysis_result & result, - std::string & second_call_content); + // Logic below is only for non-JSON-native tool calling formats + // 3. b. function name + // Extract function name markers + static tool_function_analysis extract_function_markers(const common_chat_template & tmpl, + const tool_format_analysis & analysis); - // T4: Analyze function name format and extract markers - static void extract_function_markers(const common_chat_template & tmpl, - diff_analysis_result & result); + // 4. c. function arguments + // Delegates to separate functions for: separator analysis, argument name analysis, argument value analysis + static tool_arguments_analysis analyze_arguments(const common_chat_template & tmpl, + const tool_analysis & analysis); - // T5: Extract argument separator - static void extract_argument_separator(const common_chat_template & tmpl, diff_analysis_result & result); + // Extract argument name markers + static void extract_argument_name_markers(const common_chat_template & tmpl, + tool_arguments_analysis & args_analysis); - // T6: Extract args container markers - static void extract_args_markers(const common_chat_template & tmpl, diff_analysis_result & result); + // Extract argument value markers + static void extract_argument_value_markers(const common_chat_template & tmpl, + const tool_analysis & analysis, + tool_arguments_analysis & args_analysis); - // A1: Extract argument name markers - static void extract_argument_name_markers(const common_chat_template & tmpl, diff_analysis_result & result); + // Extract argument separator, if specified (eg. ......) + static void extract_argument_separator(const common_chat_template & tmpl, + tool_arguments_analysis & args_analysis); - // A2: Extract argument value markers - static void extract_argument_value_markers(const common_chat_template & tmpl, diff_analysis_result & result); + // Extract argument wrapper markers, if present (eg. '......') + static void extract_args_markers(const common_chat_template & tmpl, + const tool_analysis & analysis, + tool_arguments_analysis & args_analysis); - // T7: Extract call ID markers (for non-JSON formats) - static void extract_call_id_markers(const common_chat_template & tmpl, diff_analysis_result & result); + // 4. d. function call id + // Extract call ID markers, if present + static tool_id_analysis extract_call_id_markers(const common_chat_template & tmpl, + tool_format_analysis & analysis); - // Classify tool format based on extracted markers - static void classify_tool_format(diff_analysis_result & result); - - // Classification helpers + // Collect tokens from entire analysis to preserve static void collect_preserved_tokens(diff_analysis_result & result); - // Utility: Apply template with given parameters - static std::string apply_template(const common_chat_template & tmpl, - const template_params & params); + static std::string apply_template(const common_chat_template & tmpl, const template_params & params); }; -enum segment_type { - TEXT, - MARKER -}; +enum segment_type { TEXT, MARKER }; inline std::ostream & operator<<(std::ostream & os, const segment_type & type) { switch (type) { @@ -329,7 +376,7 @@ inline std::ostream & operator<<(std::ostream & os, const segment_type & type) { struct segment { segment_type type; - std::string value; + std::string value; segment(segment_type type, std::string value) : type(type), value(std::move(value)) {} }; diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index cb38fb160f..6e58dc6761 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -167,13 +167,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { bool is_content = node.tag == common_chat_peg_builder::CONTENT; if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here - result.reasoning_content += std::string(trim_trailing_space(node.text)); + result.reasoning_content += std::string(node.text); } if (is_content) { // Concatenate content from multiple content nodes (e.g., when reasoning markers // are preserved before content markers in reasoning_format=NONE mode) - result.content += std::string(trim_trailing_space(node.text)); + result.content += std::string(node.text); } } diff --git a/common/chat.cpp b/common/chat.cpp index 3fb17b4e9d..be4e19aebc 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -240,7 +240,7 @@ bool common_chat_templates_support_enable_thinking(const common_chat_templates * ? *chat_templates->template_tool_use : *chat_templates->template_default; diff_analysis_result result = differential_analyzer::analyze(tmpl); - detect |= result.reasoning != reasoning_mode::NONE; + detect |= result.reasoning.mode != reasoning_mode::NONE; return detect; } diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index abd4cd2d9f..aecad6efa6 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -42,7 +42,7 @@ static void caps_try_execute(jinja::program & prog, jinja::runtime runtime(ctx); auto results = runtime.execute(prog); auto parts = jinja::runtime::gather_string_parts(results); - std::string result = parts->as_string().str(); + result = parts->as_string().str(); success = true; } catch (const std::exception & e) { JJ_DEBUG("Exception during execution: %s", e.what()); @@ -95,6 +95,8 @@ caps caps_get(jinja::program & prog) { return v->stats.ops.find(op_name) != v->stats.ops.end(); }; + JJ_DEBUG("%s\n", ">>> Running capability check: typed content"); + // case: typed content support caps_try_execute( prog, @@ -125,6 +127,7 @@ caps caps_get(jinja::program & prog) { } ); + JJ_DEBUG("%s\n", ">>> Running capability check: system prompt"); // case: system prompt support caps_try_execute( @@ -155,6 +158,8 @@ caps caps_get(jinja::program & prog) { } ); + JJ_DEBUG("%s\n", ">>> Running capability check: tool support"); + // case: tools support caps_try_execute( prog, @@ -167,7 +172,7 @@ caps caps_get(jinja::program & prog) { }, { {"role", "assistant"}, - {"content", "Assistant message"}, + {"content", ""}, // Some templates expect content to be empty with tool calls {"tool_calls", json::array({ { {"id", "call00001"}, @@ -260,6 +265,8 @@ caps caps_get(jinja::program & prog) { } ); + JJ_DEBUG("%s\n", ">>> Running capability check: preserve reasoning"); + // case: preserve reasoning content in chat history caps_try_execute( prog, diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index cc012c892f..b7e71115ed 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -114,8 +114,10 @@ value binary_expression::execute_impl(context & ctx) { // Logical operators if (op.value == "and") { + JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str()); return left_val->as_bool() ? right->execute(ctx) : std::move(left_val); } else if (op.value == "or") { + JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str()); return left_val->as_bool() ? std::move(left_val) : right->execute(ctx); } @@ -835,7 +837,7 @@ value call_expression::execute_impl(context & ctx) { for (auto & arg_stmt : this->args) { auto arg_val = arg_stmt->execute(ctx); JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(std::move(arg_val)); + args.push_back(arg_val); } // execute callee value callee_val = callee->execute(ctx); diff --git a/models/templates/deepseek-ai-DeepSeek-V3.1.jinja b/models/templates/deepseek-ai-DeepSeek-V3.1.jinja index 6ef7fb123c..2fd1c415b8 100644 --- a/models/templates/deepseek-ai-DeepSeek-V3.1.jinja +++ b/models/templates/deepseek-ai-DeepSeek-V3.1.jinja @@ -27,7 +27,7 @@ {%- set ns.is_first = false -%} {%- set ns.is_last_user = true -%}{{'<｜User｜>' + message['content']}} {%- endif -%} - {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none -%} + {%- if message['role'] == 'assistant' and message['tool_calls'] -%} {%- if ns.is_last_user -%}{{'<｜Assistant｜>'}} {%- endif -%} {%- set ns.is_last_user = false -%} @@ -35,7 +35,7 @@ {%- set ns.is_tool = false -%} {%- for tool in message['tool_calls'] -%} {%- if not ns.is_first -%} - {%- if message['content'] is none -%}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}} + {%- if not message['content'] -%}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}} {%- else -%}{{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}} {%- endif -%} {%- set ns.is_first = true -%} @@ -43,7 +43,7 @@ {%- endif -%} {%- endfor -%}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} {%- endif -%} - {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) -%} + {%- if message['role'] == 'assistant' and not message['tool_calls'] -%} {%- if ns.is_last_user -%}{{'<｜Assistant｜>'}} {%- if message['prefix'] is defined and message['prefix'] and thinking -%}{{''}} {%- else -%}{{''}} diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 90edaba32d..b2bdab15e9 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -57,7 +57,6 @@ static void test_nemotron_tool_format(testing & t); // CohereForAI template analysis tests static void test_cohere_reasoning_detection(testing & t); -static void test_cohere_tool_format(testing & t); static void test_cohere_analysis(testing & t); // Marker separation @@ -1283,18 +1282,18 @@ static void test_nemotron_reasoning_detection(testing & t) { auto analysis = differential_analyzer::analyze(tmpl); // Check reasoning markers - t.assert_equal("reasoning_start should be ''", "", analysis.markers.reasoning_start); - t.assert_equal("reasoning_end should be ''", "", analysis.markers.reasoning_end); + t.assert_equal("reasoning_start should be ''", "", analysis.reasoning.start); + t.assert_equal("reasoning_end should be ''", "", analysis.reasoning.end); // Check reasoning mode detection // Nemotron uses forced closed reasoning with add_generation_prompt - t.assert_equal("reasoning should be FORCED_CLOSED", reasoning_mode::FORCED_CLOSED, analysis.reasoning); + t.assert_equal("reasoning should be FORCED_CLOSED", reasoning_mode::FORCED_CLOSED, analysis.reasoning.mode); // Make sure reasoning markers don't spill over to content markers - t.assert_equal("content start should be empty", "", analysis.markers.content_start); - t.assert_equal("content end should be empty", "", analysis.markers.content_end); + t.assert_equal("content start should be empty", "", analysis.content.start); + t.assert_equal("content end should be empty", "", analysis.content.end); - t.assert_equal("content should be PLAIN", content_mode::PLAIN, analysis.content); + t.assert_equal("content should be PLAIN", content_mode::PLAIN, analysis.content.mode); } static void test_nemotron_tool_format(testing & t) { @@ -1304,27 +1303,27 @@ static void test_nemotron_tool_format(testing & t) { auto analysis = differential_analyzer::analyze(tmpl); // Check tool markers - Nemotron uses per-call wrapping (each call individually wrapped) - t.assert_equal("tool_section_start should be empty (per-call format)", "", analysis.markers.tool_section_start); - t.assert_equal("tool_section_end should be empty (per-call format)", "", analysis.markers.tool_section_end); - t.assert_equal("per_call_start should be '\\n'", "\n", analysis.markers.per_call_start); - t.assert_equal("per_call_end should be ''", "", analysis.markers.per_call_end); - t.assert_true("should support parallel calls", analysis.supports_parallel_calls); + t.assert_equal("tool_section_start should be empty (per-call format)", "", analysis.tools.format.section_start); + t.assert_equal("tool_section_end should be empty (per-call format)", "", analysis.tools.format.section_end); + t.assert_equal("per_call_start should be '\\n'", "\n", analysis.tools.format.per_call_start); + t.assert_equal("per_call_end should be ''", "", analysis.tools.format.per_call_end); + t.assert_true("should support parallel calls", analysis.jinja_caps.supports_parallel_tool_calls); // Check function markers - t.assert_equal("func_name_prefix should be '\\n'", ">\n", analysis.markers.func_name_suffix); - t.assert_equal("func_close should be '\\n'", "\n", analysis.markers.func_close); + t.assert_equal("func_name_prefix should be '\\n'", ">\n", analysis.tools.function.name_suffix); + t.assert_equal("func_close should be '\\n'", "\n", analysis.tools.function.close); // Check argument markers (note: markers retain trailing newlines for proper parsing) - t.assert_equal("arg_name_prefix should be '\\n'", ">\n", analysis.markers.arg_name_suffix); - t.assert_equal("arg_value_suffix should be '\\n'", "\n", analysis.markers.arg_value_suffix); + t.assert_equal("arg_name_prefix should be '\\n'", ">\n", analysis.tools.arguments.name_suffix); + t.assert_equal("arg_value_suffix should be '\\n'", "\n", analysis.tools.arguments.value_suffix); // Check format classification - t.assert_true("tool format should be TAG_WITH_TAGGED", analysis.tools == tool_format::TAG_WITH_TAGGED); + t.assert_true("tool format should be TAG_WITH_TAGGED", analysis.tools.format.mode == tool_format::TAG_WITH_TAGGED); // Verify tool support - t.assert_true("should support tools", analysis.supports_tools); + t.assert_true("should support tools", analysis.jinja_caps.supports_tools); } static common_chat_template load_cohere_template(testing & t) { @@ -1333,7 +1332,6 @@ static common_chat_template load_cohere_template(testing & t) { static void test_cohere_analysis(testing & t) { t.test("Cohere reasoning detection", test_cohere_reasoning_detection); - t.test("Cohere tool format", test_cohere_tool_format); } static void test_cohere_reasoning_detection(testing & t) { @@ -1343,64 +1341,64 @@ static void test_cohere_reasoning_detection(testing & t) { auto analysis = differential_analyzer::analyze(tmpl); // Check reasoning markers - Cohere uses special token format - t.assert_equal("reasoning_start should be '<|START_THINKING|>'", "<|START_THINKING|>", analysis.markers.reasoning_start); - t.assert_equal("reasoning_end should be '<|END_THINKING|>'", "<|END_THINKING|>", analysis.markers.reasoning_end); + t.assert_equal("reasoning_start should be '<|START_THINKING|>'", "<|START_THINKING|>", analysis.reasoning.start); + t.assert_equal("reasoning_end should be '<|END_THINKING|>'", "<|END_THINKING|>", analysis.reasoning.end); // Check reasoning mode - Cohere only shows reasoning with tool calls (TOOLS_ONLY) - t.assert_equal("reasoning should be TOOLS_ONLY", reasoning_mode::TOOLS_ONLY, analysis.reasoning); + t.assert_equal("reasoning should be TOOLS_ONLY", reasoning_mode::TOOLS_ONLY, analysis.reasoning.mode); // Check content markers - Cohere wraps all content with START/END_RESPONSE - t.assert_equal("content_start should be '<|START_RESPONSE|>'", "<|START_RESPONSE|>", analysis.markers.content_start); - t.assert_equal("content_end should be '<|END_RESPONSE|>'", "<|END_RESPONSE|>", analysis.markers.content_end); + t.assert_equal("content_start should be '<|START_RESPONSE|>'", "<|START_RESPONSE|>", analysis.content.start); + t.assert_equal("content_end should be '<|END_RESPONSE|>'", "<|END_RESPONSE|>", analysis.content.end); // Content is always wrapped (both with and without tools) - t.assert_equal("content should be ALWAYS_WRAPPED", content_mode::ALWAYS_WRAPPED, analysis.content); + t.assert_equal("content should be ALWAYS_WRAPPED", content_mode::ALWAYS_WRAPPED, analysis.content.mode); } -static void test_cohere_tool_format(testing & t) { +static void test_tool_format_cohere(testing & t) { common_chat_template tmpl = load_cohere_template(t); // Run differential analysis auto analysis = differential_analyzer::analyze(tmpl); // Check tool section markers - Cohere uses ACTION markers - t.assert_equal("tool_section_start should be '<|START_ACTION|>'", "<|START_ACTION|>", analysis.markers.tool_section_start); - t.assert_equal("tool_section_end should be '<|END_ACTION|>'", "<|END_ACTION|>", analysis.markers.tool_section_end); + t.assert_equal("tool_section_start should be '<|START_ACTION|>'", "<|START_ACTION|>", analysis.tools.format.section_start); + t.assert_equal("tool_section_end should be '<|END_ACTION|>'", "<|END_ACTION|>", analysis.tools.format.section_end); // JSON_NATIVE format has no per-call markers - t.assert_equal("per_call_start should be empty", "", analysis.markers.per_call_start); - t.assert_equal("per_call_end should be empty", "", analysis.markers.per_call_end); + t.assert_equal("per_call_start should be empty", "", analysis.tools.format.per_call_start); + t.assert_equal("per_call_end should be empty", "", analysis.tools.format.per_call_end); // JSON_NATIVE format has empty function markers (no XML-style markers) - t.assert_equal("func_name_prefix should be empty", "", analysis.markers.func_name_prefix); - t.assert_equal("func_name_suffix should be empty", "", analysis.markers.func_name_suffix); - t.assert_equal("func_close should be empty", "", analysis.markers.func_close); + t.assert_equal("func_name_prefix should be empty", "", analysis.tools.function.name_prefix); + t.assert_equal("func_name_suffix should be empty", "", analysis.tools.function.name_suffix); + t.assert_equal("func_close should be empty", "", analysis.tools.function.close); // JSON_NATIVE format has empty args markers - t.assert_equal("args_start should be empty", "", analysis.markers.args_start); - t.assert_equal("args_end should be empty", "", analysis.markers.args_end); + t.assert_equal("args_start should be empty", "", analysis.tools.arguments.start); + t.assert_equal("args_end should be empty", "", analysis.tools.arguments.end); // JSON_NATIVE format has empty argument markers - t.assert_equal("arg_name_prefix should be empty", "", analysis.markers.arg_name_prefix); - t.assert_equal("arg_name_suffix should be empty", "", analysis.markers.arg_name_suffix); - t.assert_equal("arg_value_prefix should be empty", "", analysis.markers.arg_value_prefix); - t.assert_equal("arg_value_suffix should be empty", "", analysis.markers.arg_value_suffix); - t.assert_equal("arg_separator should be empty", "", analysis.markers.arg_separator); + t.assert_equal("arg_name_prefix should be empty", "", analysis.tools.arguments.name_prefix); + t.assert_equal("arg_name_suffix should be empty", "", analysis.tools.arguments.name_suffix); + t.assert_equal("arg_value_prefix should be empty", "", analysis.tools.arguments.value_prefix); + t.assert_equal("arg_value_suffix should be empty", "", analysis.tools.arguments.value_suffix); + t.assert_equal("arg_separator should be empty", "", analysis.tools.arguments.separator); // Check JSON field names - Cohere uses non-standard names - t.assert_equal("name_field should be 'tool_name'", "tool_name", analysis.name_field); - t.assert_equal("args_field should be 'parameters'", "parameters", analysis.args_field); + t.assert_equal("name_field should be 'tool_name'", "tool_name", analysis.tools.format.name_field); + t.assert_equal("args_field should be 'parameters'", "parameters", analysis.tools.format.args_field); // This isn't a real tool call id field, i.e. with the OpenAI tool call ID format - t.assert_equal("id_field should be 'tool_call_id'", "", analysis.id_field); + t.assert_equal("id_field should be 'tool_call_id'", "", analysis.tools.format.id_field); // Check format classification - t.assert_equal("tool format should be JSON_NATIVE", tool_format::JSON_NATIVE, analysis.tools); + t.assert_equal("tool format should be JSON_NATIVE", tool_format::JSON_NATIVE, analysis.tools.format.mode); // Check flags - t.assert_true("should support tools", analysis.supports_tools); - t.assert_true("should support parallel calls", analysis.supports_parallel_calls); - t.assert_true("should not require nonnull content", !analysis.requires_nonnull_content); - t.assert_true("tools_array_wrapped should be true", analysis.tools_array_wrapped); + t.assert_true("should support tools", analysis.jinja_caps.supports_tools); + t.assert_true("should support parallel calls", analysis.jinja_caps.supports_parallel_tool_calls); + t.assert_true("should not require nonnull content", !analysis.content.requires_nonnull_content); + t.assert_true("tools_array_wrapped should be true", analysis.tools.format.tools_array_wrapped); } // ============================================================================ diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index ae82966699..d59880e3dc 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -530,7 +530,7 @@ static void test_example_qwen3_non_coder(testing & t) { auto mapper = common_chat_peg_unified_mapper(msg); mapper.from_ast(ctx.ast, result); - t.assert_equal("content", "I need to get the weather.", msg.content); + t.assert_equal("content", "I need to get the weather.\n", msg.content); t.assert_equal("reasoning", "", msg.reasoning_content); t.assert_equal("tool calls count", 1u, msg.tool_calls.size()); if (!msg.tool_calls.empty()) { diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index 8b331129b1..91ff83a729 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -32,6 +32,7 @@ static std::string HELP = R"( Usage: test-chat-template [OPTIONS] PATH_TO_TEMPLATE Options: -h, --help Show this help message and exit. + --with-tools Add a tool and a tool call to the default JSON input --json Path to the JSON input file. --stop-on-first-fail Stop testing on the first failure (default: false). --no-common Use direct Jinja engine instead of common chat templates (default: use common). @@ -57,12 +58,65 @@ static std::string DEFAULT_JSON = R"({ "add_generation_prompt": true })"; +static std::string DEFAULT_JSON_WITH_TOOLS = R"({ + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + }, + { + "role": "assistant", + "content": "I am fine, thank you!" + }, + { + "role": "user", + "content": "Call a tool!" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call00001", + "type": "function", + "function": { + "name": "test", + "arguments": { "arg": "hello" } + } + } + ] + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "test", + "description": "Test", + "parameters": { + "type": "object", + "properties": { + "arg": { + "type": "string" + } + } + }, + "required": ["arg"] + } + } + ], + "bos_token": "", + "eos_token": "", + "add_generation_prompt": true +})"; + + int main(int argc, char ** argv) { std::vector args(argv, argv + argc); std::string tmpl_path; std::string json_path; std::string output_path; + std::string & json_to_use = DEFAULT_JSON; bool stop_on_first_fail = false; bool use_common = true; @@ -74,6 +128,8 @@ int main(int argc, char ** argv) { if (args[i] == "--json" && i + 1 < args.size()) { json_path = args[i + 1]; i++; + } else if (args[i] == "--with-tools") { + json_to_use = DEFAULT_JSON_WITH_TOOLS; } else if (args[i] == "--stop-on-first-fail") { stop_on_first_fail = true; } else if (args[i] == "--output" && i + 1 < args.size()) { @@ -106,7 +162,7 @@ int main(int argc, char ** argv) { std::istreambuf_iterator()); input_json = json::parse(content); } else { - input_json = json::parse(DEFAULT_JSON); + input_json = json::parse(json_to_use); } std::filesystem::path p(tmpl_path); diff --git a/tools/parser/debug-template-parser.cpp b/tools/parser/debug-template-parser.cpp index c0e29c548a..c87f3c8e35 100644 --- a/tools/parser/debug-template-parser.cpp +++ b/tools/parser/debug-template-parser.cpp @@ -419,33 +419,33 @@ int main(int argc, char ** argv) { LOG_ERR("\n=== Differential Analysis Results ===\n"); LOG_ERR("\n--- Reasoning & Content Structure ---\n"); - LOG_ERR("reasoning_mode: %s\n", mode_to_str(analysis.reasoning).c_str()); - LOG_ERR("reasoning_start: '%s'\n", analysis.markers.reasoning_start.c_str()); - LOG_ERR("reasoning_end: '%s'\n", analysis.markers.reasoning_end.c_str()); - LOG_ERR("content_mode: %s\n", mode_to_str(analysis.content).c_str()); - LOG_ERR("content_start: '%s'\n", analysis.markers.content_start.c_str()); - LOG_ERR("content_end: '%s'\n", analysis.markers.content_end.c_str()); + LOG_ERR("reasoning_mode: %s\n", mode_to_str(analysis.reasoning.mode).c_str()); + LOG_ERR("reasoning_start: '%s'\n", analysis.reasoning.start.c_str()); + LOG_ERR("reasoning_end: '%s'\n", analysis.reasoning.end.c_str()); + LOG_ERR("content_mode: %s\n", mode_to_str(analysis.content.mode).c_str()); + LOG_ERR("content_start: '%s'\n", analysis.content.start.c_str()); + LOG_ERR("content_end: '%s'\n", analysis.content.end.c_str()); LOG_ERR("\n--- Tool Call Structure ---\n"); - LOG_ERR("tool_mode: %s\n", mode_to_str(analysis.tools).c_str()); - LOG_ERR("supports_tools: %s\n", analysis.supports_tools ? "true" : "false"); - LOG_ERR("supports_parallel_calls: %s\n", analysis.supports_parallel_calls ? "true" : "false"); - LOG_ERR("tool_section_start: '%s'\n", analysis.markers.tool_section_start.c_str()); - LOG_ERR("tool_section_end: '%s'\n", analysis.markers.tool_section_end.c_str()); - LOG_ERR("per_call_start: '%s'\n", analysis.markers.per_call_start.c_str()); - LOG_ERR("per_call_end: '%s'\n", analysis.markers.per_call_end.c_str()); - LOG_ERR("func_name_prefix: '%s'\n", analysis.markers.func_name_prefix.c_str()); - LOG_ERR("func_name_suffix: '%s'\n", analysis.markers.func_name_suffix.c_str()); - LOG_ERR("func_close: '%s'\n", analysis.markers.func_close.c_str()); - LOG_ERR("arg_name_prefix: '%s'\n", analysis.markers.arg_name_prefix.c_str()); - LOG_ERR("arg_name_suffix: '%s'\n", analysis.markers.arg_name_suffix.c_str()); - LOG_ERR("arg_value_prefix: '%s'\n", analysis.markers.arg_value_prefix.c_str()); - LOG_ERR("arg_value_suffix: '%s'\n", analysis.markers.arg_value_suffix.c_str()); - LOG_ERR("name_field: '%s'\n", analysis.name_field.c_str()); - LOG_ERR("args_field: '%s'\n", analysis.args_field.c_str()); - LOG_ERR("id_field: '%s'\n", analysis.id_field.c_str()); - LOG_ERR("gen_id_field: '%s'\n", analysis.gen_id_field.c_str()); - LOG_ERR("parameter_order: '%s'\n", std::accumulate(analysis.parameter_order.begin(), analysis.parameter_order.end(), + LOG_ERR("tool_mode: %s\n", mode_to_str(analysis.tools.format.mode).c_str()); + LOG_ERR("supports_tools: %s\n", analysis.jinja_caps.supports_tools ? "true" : "false"); + LOG_ERR("supports_parallel_calls: %s\n", analysis.jinja_caps.supports_parallel_tool_calls ? "true" : "false"); + LOG_ERR("tool_section_start: '%s'\n", analysis.tools.format.section_start.c_str()); + LOG_ERR("tool_section_end: '%s'\n", analysis.tools.format.section_end.c_str()); + LOG_ERR("per_call_start: '%s'\n", analysis.tools.format.per_call_start.c_str()); + LOG_ERR("per_call_end: '%s'\n", analysis.tools.format.per_call_end.c_str()); + LOG_ERR("func_name_prefix: '%s'\n", analysis.tools.function.name_prefix.c_str()); + LOG_ERR("func_name_suffix: '%s'\n", analysis.tools.function.name_suffix.c_str()); + LOG_ERR("func_close: '%s'\n", analysis.tools.function.close.c_str()); + LOG_ERR("arg_name_prefix: '%s'\n", analysis.tools.arguments.name_prefix.c_str()); + LOG_ERR("arg_name_suffix: '%s'\n", analysis.tools.arguments.name_suffix.c_str()); + LOG_ERR("arg_value_prefix: '%s'\n", analysis.tools.arguments.value_prefix.c_str()); + LOG_ERR("arg_value_suffix: '%s'\n", analysis.tools.arguments.value_suffix.c_str()); + LOG_ERR("name_field: '%s'\n", analysis.tools.format.name_field.c_str()); + LOG_ERR("args_field: '%s'\n", analysis.tools.format.args_field.c_str()); + LOG_ERR("id_field: '%s'\n", analysis.tools.format.id_field.c_str()); + LOG_ERR("gen_id_field: '%s'\n", analysis.tools.format.gen_id_field.c_str()); + LOG_ERR("parameter_order: '%s'\n", std::accumulate(analysis.tools.format.parameter_order.begin(), analysis.tools.format.parameter_order.end(), std::string(""), [] (const std::string & a, const std::string & b) { return a.empty() ? b : a + ", " + b; } ).c_str()); @@ -470,11 +470,13 @@ int main(int argc, char ** argv) { LOG_ERR(" '%s'\n", token.c_str()); } - LOG_ERR("\n=== Verifying created grammar ===\n"); - auto * grammar = llama_grammar_init_impl(nullptr, parser_data.grammar.c_str(), "root", - parser_data.grammar_lazy, nullptr, 0, nullptr, 0); - if (grammar != nullptr) { - LOG_ERR("\n=== Grammar successfully created ===\n"); + if (!parser_data.grammar.empty()) { + LOG_ERR("\n=== Verifying created grammar ===\n"); + auto * grammar = llama_grammar_init_impl(nullptr, parser_data.grammar.c_str(), "root", + parser_data.grammar_lazy, nullptr, 0, nullptr, 0); + if (grammar != nullptr) { + LOG_ERR("\n=== Grammar successfully created ===\n"); + } } } } catch (const std::exception & e) {