From 3605e78569d1e101fda7538eaf1104debd26532e Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 14 Feb 2026 00:17:43 +0100 Subject: [PATCH] Refactor into class-based approach --- common/chat-auto-parser-generator.cpp | 285 +++++++------ common/chat-auto-parser-helpers.cpp | 57 +++ common/chat-auto-parser-helpers.h | 17 + common/chat-auto-parser.h | 35 +- common/chat-diff-analyzer.cpp | 552 +++++++++++-------------- common/chat-diff-analyzer.h | 282 +++++++------ common/chat.cpp | 30 +- common/chat.h | 6 +- tests/test-chat-auto-parser.cpp | 42 +- tools/parser/debug-template-parser.cpp | 8 +- tools/parser/template-analysis.cpp | 36 +- 11 files changed, 706 insertions(+), 644 deletions(-) diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index 13ec14fb64..bc2b1f7bbe 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -19,22 +19,23 @@ static void foreach_function(const json & tools, const std::functionis_always_wrapped()) { + auto wrapped_content = ctx.content->build_optional_wrapped(ctx); + return ctx.reasoning_parser + wrapped_content + tools_parser + p.end(); } - auto content_before_tools = analysis.tools.format.section_start.empty() ? p.eps() : p.until(analysis.tools.format.section_start); - return reasoning + p.optional(p.content(content_before_tools)) + tools_parser + p.end(); + auto content_before_tools = format.section_start.empty() ? p.eps() : p.until(format.section_start); + return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tools_parser + p.end(); } -common_peg_parser universal_peg_generator::build_tool_parser_tag_json( - common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning) { +common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context & ctx) const { + auto & p = ctx.p; + const auto & inputs = ctx.inputs; common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & schema = function.at("parameters"); + const auto & func = tool.at("function"); + std::string name = func.at("name"); + const auto & schema = func.at("parameters"); // Build call_id parser based on position (if supported) common_peg_parser call_id_section = p.eps(); - if (analysis.tools.call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && - !analysis.tools.call_id.prefix.empty() && !analysis.tools.call_id.suffix.empty()) { - call_id_section = p.optional(analysis.tools.call_id.prefix + p.tool_id(p.until(analysis.tools.call_id.suffix))) + analysis.tools.call_id.suffix; + if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && + !call_id.prefix.empty() && !call_id.suffix.empty()) { + call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix))) + call_id.suffix; } - auto func_parser = p.tool_open(analysis.tools.function.name_prefix + p.tool_name(p.literal(name)) + analysis.tools.function.name_suffix) + + auto func_parser = p.tool_open(function.name_prefix + p.tool_name(p.literal(name)) + function.name_suffix) + call_id_section + p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)); - if (!analysis.tools.function.close.empty()) { - func_parser = func_parser + analysis.tools.function.close; + if (!function.close.empty()) { + func_parser = func_parser + function.close; } tool_choice |= p.rule("tool-" + name, func_parser); @@ -221,26 +254,26 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( common_peg_parser tool_calls = p.eps(); - if (!analysis.tools.format.per_call_start.empty()) { - auto wrapped_call = analysis.tools.format.per_call_start + tool_choice + analysis.tools.format.per_call_end; + if (!format.per_call_start.empty()) { + auto wrapped_call = format.per_call_start + tool_choice + format.per_call_end; if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call)); } else { tool_calls = p.trigger_rule("tool-call", wrapped_call); } - if (!analysis.tools.format.section_start.empty()) { - tool_calls = p.trigger_rule("tool-calls", p.literal(analysis.tools.format.section_start) + p.space() + - tool_calls + p.space() + (analysis.tools.format.section_end.empty() ? p.end() : p.literal(analysis.tools.format.section_end))); + if (!format.section_start.empty()) { + tool_calls = p.trigger_rule("tool-calls", p.literal(format.section_start) + p.space() + + tool_calls + p.space() + (format.section_end.empty() ? p.end() : p.literal(format.section_end))); } } else { std::string separator = ", "; // Default if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", - analysis.tools.format.section_start + tool_choice + p.zero_or_more(separator + tool_choice) + analysis.tools.format.section_end); + format.section_start + tool_choice + p.zero_or_more(separator + tool_choice) + format.section_end); } else { tool_calls = p.trigger_rule("tool-call", - analysis.tools.format.section_start + tool_choice + analysis.tools.format.section_end); + format.section_start + tool_choice + format.section_end); } } @@ -248,23 +281,21 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_json( tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !analysis.tools.format.section_start.empty() ? analysis.tools.format.section_start : analysis.tools.format.per_call_start; + std::string trigger_marker = !format.section_start.empty() ? format.section_start : format.per_call_start; auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); - return reasoning + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); + return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } -common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( - common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning) { +common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_context & ctx) const { + auto & p = ctx.p; + const auto & inputs = ctx.inputs; common_peg_parser tool_choice = p.choice(); foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - const auto & params = function.at("parameters"); + const auto & func = tool.at("function"); + std::string name = func.at("name"); + const auto & params = func.at("parameters"); if (!params.contains("properties") || !params.at("properties").is_object()) { return; @@ -283,13 +314,13 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( auto type = param_schema.value("type", "object"); auto arg = p.tool_arg( - p.tool_arg_open(analysis.tools.arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) + analysis.tools.arguments.name_suffix) + analysis.tools.arguments.value_prefix + + p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) + arguments.name_suffix) + arguments.value_prefix + (type == "string" ? - p.tool_arg_string_value(p.schema(p.until(analysis.tools.arguments.value_suffix), + p.tool_arg_string_value(p.schema(p.until(arguments.value_suffix), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, true)) : p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema)) + p.space()) + - p.tool_arg_close(p.literal(analysis.tools.arguments.value_suffix)) + p.tool_arg_close(p.literal(arguments.value_suffix)) ); if (is_required) { @@ -310,23 +341,23 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( // Build call_id parser based on position (if supported) common_peg_parser call_id_section = p.eps(); - if (analysis.tools.call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && - !analysis.tools.call_id.prefix.empty() && !analysis.tools.call_id.suffix.empty()) { - call_id_section = p.optional(analysis.tools.call_id.prefix + p.tool_id(p.until(analysis.tools.call_id.suffix))) + analysis.tools.call_id.suffix; + if (call_id.pos == call_id_position::BETWEEN_FUNC_AND_ARGS && + !call_id.prefix.empty() && !call_id.suffix.empty()) { + call_id_section = p.optional(call_id.prefix + p.tool_id(p.until(call_id.suffix))) + call_id.suffix; } - auto func_parser = p.tool_open(analysis.tools.function.name_prefix + p.tool_name(p.literal(name)) + analysis.tools.function.name_suffix) + + auto func_parser = p.tool_open(function.name_prefix + p.tool_name(p.literal(name)) + function.name_suffix) + call_id_section + p.space() + args_seq; - if (!analysis.tools.function.close.empty()) { - func_parser = func_parser + p.space() + p.tool_close(p.literal(analysis.tools.function.close)); - } else if (!analysis.tools.format.per_call_end.empty()) { + if (!function.close.empty()) { + func_parser = func_parser + p.space() + p.tool_close(p.literal(function.close)); + } else if (!format.per_call_end.empty()) { // When there's no func_close but there is a per_call_end marker, use peek() to ensure // we only emit tool_close when we can actually see the closing marker. This prevents // premature closing during partial parsing when we've seen e.g. "" (end) or "" prefix that failed to match. - func_parser = func_parser + p.tool_close(p.peek(p.literal(analysis.tools.format.per_call_end))); + func_parser = func_parser + p.tool_close(p.peek(p.literal(format.per_call_end))); } else { func_parser = func_parser + p.tool_close(p.space()); // force this to process tool closing callbacks in mapper } @@ -338,26 +369,26 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( common_peg_parser tool_calls = p.eps(); - if (!analysis.tools.format.per_call_start.empty()) { - auto wrapped_call = analysis.tools.format.per_call_start + p.space() + tool_choice + p.space() + analysis.tools.format.per_call_end; + if (!format.per_call_start.empty()) { + auto wrapped_call = format.per_call_start + p.space() + tool_choice + p.space() + format.per_call_end; if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", wrapped_call + p.zero_or_more(p.space() + wrapped_call)); } else { tool_calls = p.trigger_rule("tool-call", wrapped_call); } - if (!analysis.tools.format.section_start.empty()) { - tool_calls = p.trigger_rule("tool-calls", p.literal(analysis.tools.format.section_start) + p.space() + - tool_calls + p.space() + (analysis.tools.format.section_end.empty() ? p.end() : p.literal(analysis.tools.format.section_end))); + if (!format.section_start.empty()) { + tool_calls = p.trigger_rule("tool-calls", p.literal(format.section_start) + p.space() + + tool_calls + p.space() + (format.section_end.empty() ? p.end() : p.literal(format.section_end))); } } else { std::string separator = ", "; // Default if (inputs.parallel_tool_calls) { tool_calls = p.trigger_rule("tool-call", - analysis.tools.format.section_start + p.space() + tool_choice + p.zero_or_more(separator + tool_choice) + p.space() + analysis.tools.format.section_end); + format.section_start + p.space() + tool_choice + p.zero_or_more(separator + tool_choice) + p.space() + format.section_end); } else { tool_calls = p.trigger_rule("tool-call", - analysis.tools.format.section_start + p.space() + tool_choice + p.space() + analysis.tools.format.section_end); + format.section_start + p.space() + tool_choice + p.space() + format.section_end); } } @@ -365,7 +396,9 @@ common_peg_parser universal_peg_generator::build_tool_parser_tag_tagged( tool_calls = p.optional(tool_calls); } - std::string trigger_marker = !analysis.tools.format.section_start.empty() ? analysis.tools.format.section_start : analysis.tools.format.per_call_start; + std::string trigger_marker = !format.section_start.empty() ? format.section_start : format.per_call_start; auto content_before_tools = trigger_marker.empty() ? p.eps() : p.until(trigger_marker); - return reasoning + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); + return ctx.reasoning_parser + p.optional(p.content(content_before_tools)) + tool_calls + p.end(); } + +} // namespace autoparser diff --git a/common/chat-auto-parser-helpers.cpp b/common/chat-auto-parser-helpers.cpp index 0f40d9e813..3be1cbf1b2 100644 --- a/common/chat-auto-parser-helpers.cpp +++ b/common/chat-auto-parser-helpers.cpp @@ -1,6 +1,9 @@ #include "chat-auto-parser-helpers.h" +#include "chat-auto-parser.h" #include "chat-diff-analyzer.h" +#include "chat.h" +#include "log.h" #include "nlohmann/json.hpp" #include @@ -289,3 +292,57 @@ std::vector prune_whitespace_segments(const std::vector & segm return result; } +namespace autoparser { + +std::string apply_template(const common_chat_template & tmpl, const template_params & params) { + templates_params tmpl_params; + tmpl_params.messages = params.messages; + tmpl_params.tools = params.tools; + tmpl_params.add_generation_prompt = params.add_generation_prompt; + tmpl_params.enable_thinking = params.enable_thinking; + + if (params.extra_context) { + tmpl_params.extra_context = *params.extra_context; + } + tmpl_params.extra_context["enable_thinking"] = params.enable_thinking; + + try { + return common_chat_template_direct_apply(tmpl, tmpl_params); + } catch (const std::exception & e) { + LOG_DBG("Template application failed: %s\n", e.what()); + return ""; + } +} + +std::optional compare_variants( + const common_chat_template & tmpl, + const template_params & params_A, + const std::function & params_modifier) { + // Create variant B by copying A + template_params params_B = params_A; + + // Apply modifier to create variant B + if (params_modifier) { + params_modifier(params_B); + } + + // Apply template to both variants + std::string output_A = apply_template(tmpl, params_A); + std::string output_B = apply_template(tmpl, params_B); + + // Check for template application failures + if (output_A.empty() || output_B.empty()) { + return std::nullopt; + } + + // Calculate diff and return result with both outputs + compare_variants_result result; + result.diff = calculate_diff_split(output_A, output_B); + result.output_A = output_A; + result.output_B = output_B; + + return result; +} + +} // namespace autoparser + diff --git a/common/chat-auto-parser-helpers.h b/common/chat-auto-parser-helpers.h index 47e7a2a3d8..c235d63850 100644 --- a/common/chat-auto-parser-helpers.h +++ b/common/chat-auto-parser-helpers.h @@ -1,6 +1,8 @@ #pragma once #include "chat-diff-analyzer.h" +#include +#include #include std::string trim_whitespace(const std::string & str); @@ -54,3 +56,18 @@ std::vector segmentize_markers(const std::string & text); // prune_whitespace_segments(X) -> [ (MARKER, ""), (MARKER, ""), (MARKER, ""), (MARKER, ""), // (MARKER, ""), (MARKER, "") ] std::vector prune_whitespace_segments(const std::vector & segments); + +namespace autoparser { + +// Apply a template with the given parameters, returning the rendered string (empty on failure) +std::string apply_template(const common_chat_template & tmpl, const template_params & params); + +// Factorized differential comparison function +// Takes base params and a single modifier lambda to create variant B +// Returns compare_variants_result containing diff and both outputs, or std::nullopt on failure +std::optional compare_variants( + const common_chat_template & tmpl, + const template_params & params_A, + const std::function & params_modifier); + +} // namespace autoparser diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index 40f1fbe1bb..31ee56dd03 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -10,6 +10,8 @@ using json = nlohmann::ordered_json; +namespace autoparser { + struct templates_params { json messages; json tools; @@ -37,34 +39,7 @@ class universal_peg_generator { static common_chat_params generate_parser(const common_chat_template & tmpl, const struct templates_params & inputs, - const diff_analysis_result & analysis); - - private: - // Build unified parser (single code path for all formats) - static common_peg_arena build_parser(const diff_analysis_result & analysis, - const struct templates_params & inputs, - bool thinking_forced_open, - bool thinking_forced_closed = false); - - // Build tool calling parser based on detected format - static common_peg_parser build_tool_parser(common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning); - - // Per-format tool parser builders - static common_peg_parser build_tool_parser_json_native(common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning); - - static common_peg_parser build_tool_parser_tag_json(common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning); - - static common_peg_parser build_tool_parser_tag_tagged(common_chat_peg_unified_builder & p, - const diff_analysis_result & analysis, - const templates_params & inputs, - const common_peg_parser & reasoning); + const analyze_template & analysis); }; + +} // namespace autoparser diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index a7550a3b6b..2256e48976 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -1,9 +1,7 @@ #include "chat-diff-analyzer.h" #include "chat-auto-parser-helpers.h" -#include "chat-auto-parser.h" #include "chat.h" -#include "llama.h" #include "log.h" #include "nlohmann/json.hpp" @@ -17,10 +15,12 @@ using json = nlohmann::ordered_json; -static std::vector> workarounds( +namespace autoparser { + +static std::vector> workarounds( { // Old reasoning Qwen templates - they don't really display reasoning content, but we still want to // support reasoning on them - [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { + [](const common_chat_template & tmpl, analyze_template & analysis) -> void { if (tmpl.src.find("content.split('')") != std::string::npos && analysis.reasoning.mode == reasoning_mode::NONE) { analysis.reasoning.mode = reasoning_mode::FORCED_OPEN; @@ -32,7 +32,7 @@ static std::vector void { + [](const common_chat_template & tmpl, analyze_template & analysis) -> void { if (tmpl.src.find("Write your thoughts between and write your response between " "") != std::string::npos) { analysis.reasoning.mode = reasoning_mode::TAG_BASED; @@ -49,7 +49,7 @@ static std::vector...<|END_OF_TURN_TOKEN|> - [](const common_chat_template & tmpl, diff_analysis_result & analysis) -> void { + [](const common_chat_template & tmpl, analyze_template & analysis) -> void { if (tmpl.src.find("<|CHATBOT_TOKEN|>") != std::string::npos && tmpl.src.find("<|END_OF_TURN_TOKEN|>") != std::string::npos && analysis.content.start.empty()) { analysis.content.mode = content_mode::ALWAYS_WRAPPED; @@ -61,7 +61,7 @@ static std::vector void { + [](const common_chat_template & tmpl, analyze_template & analysis) -> void { if (tmpl.src.find("set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", " "\"code_interpreter\") | list | length > 0") != std::string::npos) { analysis.content.mode = content_mode::PLAIN; @@ -82,7 +82,7 @@ static std::vector void { + [](const common_chat_template & tmpl, analyze_template & analysis) -> void { if (tmpl.src.find( "{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'") != std::string::npos) { @@ -138,94 +138,76 @@ static json second_tool_call = static json first_tool_call_alt_id = build_tool_call("foofoo", json{{ "first", "XXXX" }, { "second", "YYYY" }}, "call99999"); -std::string differential_analyzer::apply_template(const common_chat_template & tmpl, const template_params & params) { - templates_params tmpl_params; - tmpl_params.messages = params.messages; - tmpl_params.tools = params.tools; - tmpl_params.add_generation_prompt = params.add_generation_prompt; - tmpl_params.enable_thinking = params.enable_thinking; - - if (params.extra_context) { - tmpl_params.extra_context = *params.extra_context; - } - tmpl_params.extra_context["enable_thinking"] = params.enable_thinking; - - try { - return common_chat_template_direct_apply(tmpl, tmpl_params); - } catch (const std::exception & e) { - LOG_DBG("Template application failed: %s\n", e.what()); - return ""; - } -} - -std::optional differential_analyzer::compare_variants( - const common_chat_template & tmpl, - const template_params & params_A, - const std::function & params_modifier) { - // Create variant B by copying A - template_params params_B = params_A; - - // Apply modifier to create variant B - if (params_modifier) { - params_modifier(params_B); - } - - // Apply template to both variants - std::string output_A = apply_template(tmpl, params_A); - std::string output_B = apply_template(tmpl, params_B); - - // Check for template application failures - if (output_A.empty() || output_B.empty()) { - return std::nullopt; - } - - // Calculate diff and return result with both outputs - compare_variants_result result; - result.diff = calculate_diff_split(output_A, output_B); - result.output_A = output_A; - result.output_B = output_B; - - return result; -} - -diff_analysis_result differential_analyzer::analyze(const common_chat_template & tmpl) { - diff_analysis_result result; +// ============================================================================ +// analyze_template +// ============================================================================ +analyze_template::analyze_template(const common_chat_template & tmpl) + : jinja_caps(tmpl.original_caps()) + , reasoning(tmpl, jinja_caps.supports_tool_calls) + , content(tmpl, reasoning) + , tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools()) +{ LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET); - result.jinja_caps = tmpl.original_caps(); - - result.reasoning = analyze_reasoning(tmpl, result.jinja_caps.supports_tool_calls); - result.content = analyze_content(tmpl, result.reasoning); - if (result.jinja_caps.supports_tool_calls) { - result.tools = analyze_tools(tmpl, result.jinja_caps, result.reasoning); - } - collect_preserved_tokens(result); + collect_preserved_tokens(); for (auto & workaround : workarounds) { - workaround(tmpl, result); + workaround(tmpl, *this); } LOG_DBG(ANSI_PURPLE "=== Differential analysis complete ===\n" ANSI_RESET); - - return result; } -reasoning_analysis differential_analyzer::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools) { +void analyze_template::collect_preserved_tokens() { + auto add_token = [this](const std::string & org_token) { + std::string token = trim_whitespace(org_token); + if (!token.empty()) { + // Avoid duplicates + if (std::find(preserved_tokens.begin(), preserved_tokens.end(), token) == preserved_tokens.end()) { + preserved_tokens.push_back(token); + } + } + }; + + add_token(reasoning.start); + add_token(reasoning.end); + add_token(content.start); + add_token(content.end); + add_token(tools.format.section_start); + add_token(tools.format.section_end); + add_token(tools.format.per_call_start); + add_token(tools.format.per_call_end); + add_token(tools.function.name_prefix); + add_token(tools.function.name_suffix); + add_token(tools.function.close); + add_token(tools.arguments.start); + add_token(tools.arguments.end); + add_token(tools.arguments.name_prefix); + add_token(tools.arguments.name_suffix); + add_token(tools.arguments.separator); + add_token(tools.arguments.value_prefix); + add_token(tools.arguments.value_suffix); + add_token(tools.call_id.prefix); + add_token(tools.call_id.suffix); +} + +// ============================================================================ +// analyze_reasoning +// ============================================================================ + +analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools) + : analyze_base(tmpl) { LOG_DBG(ANSI_ORANGE "Phase 1: Reasoning analysis\n" ANSI_RESET); - reasoning_analysis result; - - compare_reasoning_presence(tmpl, result); - compare_thinking_enabled(tmpl, result); + compare_reasoning_presence(); + compare_thinking_enabled(); if (supports_tools) { - compare_reasoning_scope(tmpl, result); + compare_reasoning_scope(); } - - return result; } -void differential_analyzer::compare_reasoning_presence(const common_chat_template & tmpl, reasoning_analysis & reasoning) { +void analyze_reasoning::compare_reasoning_presence() { json user_msg = json{ { "role", "user" }, { "content", "Hello" } @@ -248,7 +230,7 @@ void differential_analyzer::compare_reasoning_presence(const common_chat_templat params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_reasoning }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_reasoning }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed, skipping reasoning detection\n" ANSI_RESET, __func__); @@ -263,22 +245,22 @@ void differential_analyzer::compare_reasoning_presence(const common_chat_templat auto seg = prune_whitespace_segments(segmentize_markers(diff.right)); if (seg.size() >= 3 && trim_whitespace(seg[1].value) == reasoning_content) { // easy one: opening marker - reasoning - closing marker (possibly with trailing whitespace) - reasoning.mode = reasoning_mode::TAG_BASED; - reasoning.start = trim_whitespace(seg[0].value); - reasoning.end = trim_leading_whitespace(seg[2].value); + mode = reasoning_mode::TAG_BASED; + start = trim_whitespace(seg[0].value); + end = trim_leading_whitespace(seg[2].value); for (size_t i = 3; i < seg.size(); i++) { - reasoning.end += seg[i].value; + end += seg[i].value; } // we always truncate because this doesn't really influence correctness but model might not always generate newline - reasoning.end = trim_whitespace(reasoning.end); + end = trim_whitespace(end); } else if (seg.size() >= 2 && trim_whitespace(seg[0].value) == reasoning_content) { // delimited - reasoning.mode = reasoning_mode::DELIMITER; - reasoning.end = trim_leading_whitespace(seg[1].value); + mode = reasoning_mode::DELIMITER; + end = trim_leading_whitespace(seg[1].value); for (size_t i = 2; i < seg.size(); i++) { - reasoning.end += seg[i].value; + end += seg[i].value; } - reasoning.end = trim_whitespace(reasoning.end); + end = trim_whitespace(end); } else if (seg.size() == 1 && trim_whitespace(seg[0].value) == reasoning_content) { // the marker might be in the prefix actually, let's check for case of // left: empty @@ -296,16 +278,16 @@ void differential_analyzer::compare_reasoning_presence(const common_chat_templat if (marker_seg.type == segment_type::TEXT) { marker_seg = pre_seg[pre_seg.size() - 2]; } - reasoning.mode = reasoning_mode::FORCED_CLOSED; - reasoning.start = trim_whitespace(marker_seg.value); - reasoning.end = trim_whitespace(suf_seg[0].value); + mode = reasoning_mode::FORCED_CLOSED; + start = trim_whitespace(marker_seg.value); + end = trim_whitespace(suf_seg[0].value); } } } } } -void differential_analyzer::compare_thinking_enabled(const common_chat_template & tmpl, reasoning_analysis & reasoning) { +void analyze_reasoning::compare_thinking_enabled() { json user_msg = json{ { "role", "user" }, { "content", "Hello" } @@ -316,7 +298,7 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template params.add_generation_prompt = true; params.enable_thinking = false; - auto comparison = compare_variants(tmpl, params, [&](template_params & p) { p.enable_thinking = true; }); + auto comparison = compare_variants(*tmpl, params, [&](template_params & p) { p.enable_thinking = true; }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET , __func__); @@ -333,15 +315,15 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template trim_whitespace(right_trimmed); if (!right_trimmed.empty() && string_ends_with(comparison->output_B, right_trimmed)) { - if (reasoning.start.empty()) { - reasoning.start = right_trimmed; - reasoning.mode = reasoning_mode::FORCED_OPEN; + if (start.empty()) { + start = right_trimmed; + mode = reasoning_mode::FORCED_OPEN; } } } - if (reasoning.start.empty() && !reasoning.end.empty()) { - reasoning.mode = reasoning_mode::DELIMITER; + if (start.empty() && !end.empty()) { + mode = reasoning_mode::DELIMITER; } // Check for FORCED_CLOSED: when enable_thinking=false produces both start and end markers, @@ -353,49 +335,49 @@ void differential_analyzer::compare_thinking_enabled(const common_chat_template // Both should end with the assistant role marker // Check if output_A has both reasoning_start and reasoning_end markers // while output_B has only reasoning_start - if (!reasoning.start.empty()) { + if (!start.empty()) { // Check if output_A contains both start and end markers - bool A_has_start = output_A.find(reasoning.start) != std::string::npos; - bool A_has_end = !reasoning.end.empty() && output_A.find(reasoning.end) != std::string::npos; + bool A_has_start = output_A.find(start) != std::string::npos; + bool A_has_end = !end.empty() && output_A.find(end) != std::string::npos; // Check if output_B contains only the start marker (and not the end marker) - bool B_has_start = output_B.find(reasoning.start) != std::string::npos; - bool B_has_end = !reasoning.end.empty() && output_B.find(reasoning.end) != std::string::npos; + bool B_has_start = output_B.find(start) != std::string::npos; + bool B_has_end = !end.empty() && output_B.find(end) != std::string::npos; // For FORCED_CLOSED: A should have both, B should have only start if (A_has_start && A_has_end && B_has_start && !B_has_end) { - reasoning.mode = reasoning_mode::FORCED_CLOSED; + mode = reasoning_mode::FORCED_CLOSED; } - } else if (!reasoning.end.empty()) { + } else if (!end.empty()) { // We might not have detected the reasoning open marker until now, // but this is another chance to do so auto diff = comparison->diff; auto diff_rt = trim_whitespace(diff.right); auto diff_lt = trim_whitespace(diff.left); - if (diff_rt.empty() && diff_lt == reasoning.end) { + if (diff_rt.empty() && diff_lt == end) { auto seg = segmentize_markers(trim_whitespace(diff.prefix)); if (!seg.empty() && seg[seg.size() - 1].type == MARKER) { // this is FORCED_CLOSED - reasoning.start = seg[seg.size() - 1].value; - reasoning.mode = reasoning_mode::FORCED_CLOSED; + start = seg[seg.size() - 1].value; + mode = reasoning_mode::FORCED_CLOSED; } } } } - if (reasoning.start.empty() && reasoning.end.empty()) { + if (start.empty() && end.empty()) { if (!diff.left.empty() && !diff.right.empty()) { auto seg_A = segmentize_markers(trim_trailing_whitespace(diff.left)); auto seg_B = segmentize_markers(trim_trailing_whitespace(diff.right)); if (seg_A.size() == 1 && seg_B.size() == 1) { - reasoning.mode = reasoning_mode::FORCED_CLOSED; - reasoning.start = seg_B[0].value; - reasoning.end = seg_A[0].value; + mode = reasoning_mode::FORCED_CLOSED; + start = seg_B[0].value; + end = seg_A[0].value; } } } } -void differential_analyzer::compare_reasoning_scope(const common_chat_template & tmpl, reasoning_analysis & reasoning) { +void analyze_reasoning::compare_reasoning_scope() { json assistant_reasoning_content = json{ { "role", "assistant" }, { "content", "Here is my response." }, @@ -417,7 +399,7 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_reasoning_tools }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_reasoning_tools }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); @@ -431,7 +413,7 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & bool reasoning_in_B = comparison->output_B.find(reasoning_content) != std::string::npos; if (!reasoning_in_A && reasoning_in_B) { - reasoning.mode = reasoning_mode::TOOLS_ONLY; + mode = reasoning_mode::TOOLS_ONLY; LOG_DBG("R3: Detected TOOLS_ONLY reasoning mode\n"); // Extract reasoning markers from output_B @@ -446,7 +428,7 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & for (auto & segment : segments_before) { if (segment.type == segment_type::MARKER) { - reasoning.start = segment.value; + start = segment.value; break; } } @@ -458,11 +440,11 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & if (!after_reasoning.empty()) { // Try to find matching end marker - if (!reasoning.start.empty()) { + if (!start.empty()) { auto segments = segmentize_markers(after_reasoning); for (auto & segment : segments) { if (segment.type == segment_type::MARKER) { - reasoning.end = segment.value; + end = segment.value; break; } } @@ -472,10 +454,13 @@ void differential_analyzer::compare_reasoning_scope(const common_chat_template & } } -content_analysis differential_analyzer::analyze_content(const common_chat_template & tmpl, const reasoning_analysis & reasoning) { - LOG_DBG(ANSI_ORANGE "Phase 2: Content analysis\n" ANSI_RESET); +// ============================================================================ +// analyze_content +// ============================================================================ - content_analysis result; +analyze_content::analyze_content(const common_chat_template & tmpl, const analyze_reasoning & reasoning) + : analyze_base(tmpl) { + LOG_DBG(ANSI_ORANGE "Phase 2: Content analysis\n" ANSI_RESET); json assistant_content_only = json{ { "role", "assistant" }, @@ -523,7 +508,7 @@ content_analysis differential_analyzer::analyze_content(const common_chat_templa if (trim_whitespace(diff_reasoning.left) == response || (segments.size() == 2 && trim_whitespace(segments[0].value) == response)) { // We only have the content text in the diff (possibly with a stray EOG marker), so no markers - result.mode = content_mode::PLAIN; + mode = content_mode::PLAIN; found_plain_content = true; } else if (reasoning.mode != reasoning_mode::NONE && !reasoning.end.empty() && diff_reasoning.left.find(reasoning.end) != std::string::npos) { @@ -531,7 +516,7 @@ content_analysis differential_analyzer::analyze_content(const common_chat_templa diff_reasoning.left.find(reasoning.end) + reasoning.end.length()); if (trim_whitespace(post_closed_reasoning) == "Response text") { LOG_DBG("C1: No content markers after stripping reasoning close marker\n"); - result.mode = content_mode::PLAIN; + mode = content_mode::PLAIN; found_plain_content = true; } } @@ -546,48 +531,51 @@ content_analysis differential_analyzer::analyze_content(const common_chat_templa size_t pos = pure_content.find("Response text"); if (pos == std::string::npos) { LOG_DBG(ANSI_ORANGE "%s: Error: response text not found - improper template application?\n" ANSI_RESET, __func__); - return result; + return; } - result.start = trim_leading_whitespace(pure_content.substr(0, pos)); - result.end = trim_leading_whitespace(pure_content.substr(pos + 13)); // 13 - len of "Response text" + start = trim_leading_whitespace(pure_content.substr(0, pos)); + end = trim_leading_whitespace(pure_content.substr(pos + 13)); // 13 - len of "Response text" // TODO: WRAPPED_WITH_REASONING } // Determine content mode - if (!result.start.empty() || !result.end.empty()) { - result.mode = content_mode::ALWAYS_WRAPPED; + if (!start.empty() || !end.empty()) { + mode = content_mode::ALWAYS_WRAPPED; // TODO: END_DELIMITED content mode - delimited at end but not at start? } - - return result; } -tool_analysis differential_analyzer::analyze_tools(const common_chat_template & tmpl, - const jinja::caps & caps, - const reasoning_analysis & reasoning) { - tool_analysis result; +bool analyze_content::is_always_wrapped() const { + return mode == content_mode::ALWAYS_WRAPPED && !start.empty() && !end.empty(); +} + +// ============================================================================ +// analyze_tools +// ============================================================================ + +analyze_tools::analyze_tools(const common_chat_template & tmpl, + const jinja::caps & caps, + const analyze_reasoning & reasoning) + : analyze_base(tmpl) { LOG_DBG(ANSI_ORANGE "Phase 3: Tool call analysis\n" ANSI_RESET); - result.format = analyze_tool_calls(tmpl, reasoning); + analyze_tool_calls(reasoning); - if (result.format.mode != tool_format::NONE && result.format.mode != tool_format::JSON_NATIVE) { + if (format.mode != tool_format::NONE && format.mode != tool_format::JSON_NATIVE) { if (caps.supports_parallel_tool_calls) { - check_per_call_markers(tmpl, result.format); + check_per_call_markers(); } - result.function = extract_function_markers(tmpl, result.format); - if (result.format.mode == tool_format::TAG_WITH_TAGGED) { - result.arguments = analyze_arguments(tmpl, result); + extract_function_markers(); + if (format.mode == tool_format::TAG_WITH_TAGGED) { + analyze_arguments(); } - extract_argument_separator(tmpl, result.arguments); - extract_args_markers(tmpl, result, result.arguments); - result.call_id = extract_call_id_markers(tmpl, result.format); + extract_argument_separator(); + extract_args_markers(); + extract_call_id_markers(); } - - return result; } -tool_format_analysis differential_analyzer::analyze_tool_calls(const common_chat_template & tmpl, - const reasoning_analysis & reasoning) { +void analyze_tools::analyze_tool_calls(const analyze_reasoning & reasoning) { json assistant_no_tools = json{ { "role", "assistant" }, { "content", "Response." } @@ -606,11 +594,11 @@ tool_format_analysis differential_analyzer::analyze_tool_calls(const common_chat params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_tools }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_tools }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); - return tool_format_analysis(); + return; } const auto & diff = comparison->diff; @@ -618,20 +606,18 @@ tool_format_analysis differential_analyzer::analyze_tool_calls(const common_chat std::string tool_section = diff.right; if (tool_section.empty()) { - return tool_format_analysis(); + return; } - return analyze_tool_call_format(tool_section, "foofoo", "first", reasoning); + analyze_tool_call_format(tool_section, "foofoo", "first", reasoning); } -tool_format_analysis differential_analyzer::analyze_tool_call_format(const std::string & haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - const reasoning_analysis & reasoning) { - tool_format_analysis result; - +void analyze_tools::analyze_tool_call_format(const std::string & haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle, + const analyze_reasoning & reasoning) { if (fun_name_needle.empty() || arg_name_needle.empty() || haystack.empty()) { - return result; + return; } auto in_json_haystack = [&haystack](const std::string & needle) -> bool { @@ -656,11 +642,11 @@ tool_format_analysis differential_analyzer::analyze_tool_call_format(const std:: if (in_json_haystack(fun_name_needle)) { // no need to check further, we're in JSON land - result.mode = tool_format::JSON_NATIVE; + format.mode = tool_format::JSON_NATIVE; } else if (in_json_haystack(arg_name_needle)) { - result.mode = tool_format::TAG_WITH_JSON; + format.mode = tool_format::TAG_WITH_JSON; } else { - result.mode = tool_format::TAG_WITH_TAGGED; + format.mode = tool_format::TAG_WITH_TAGGED; } // first, remove any reasoning markers @@ -678,22 +664,19 @@ tool_format_analysis differential_analyzer::analyze_tool_call_format(const std:: } } - if (result.mode == tool_format::JSON_NATIVE) { - analyze_tool_call_format_json_native(clean_haystack, fun_name_needle, arg_name_needle, result); + if (format.mode == tool_format::JSON_NATIVE) { + analyze_tool_call_format_json_native(clean_haystack, fun_name_needle, arg_name_needle); } else { - analyze_tool_call_format_non_json(clean_haystack, fun_name_needle, result); + analyze_tool_call_format_non_json(clean_haystack, fun_name_needle); } // always relax whitespace requirements on ending markers since they don't influence content - result.section_end = trim_whitespace(result.section_end); - result.per_call_end = trim_whitespace(result.per_call_end); - - return result; + format.section_end = trim_whitespace(format.section_end); + format.per_call_end = trim_whitespace(format.per_call_end); } -void differential_analyzer::analyze_tool_call_format_json_native(const std::string & clean_haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - tool_format_analysis & format) { +void analyze_tools::analyze_tool_call_format_json_native(const std::string & clean_haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle) { // we might not have the typical OpenAI tool calling structure int json_start = clean_haystack.find_first_of('{'); int json_end = clean_haystack.find_last_of('}'); @@ -781,9 +764,8 @@ void differential_analyzer::analyze_tool_call_format_json_native(const std::stri } } -void differential_analyzer::analyze_tool_call_format_non_json(const std::string & clean_haystack, - const std::string & fun_name_needle, - tool_format_analysis & format) { +void analyze_tools::analyze_tool_call_format_non_json(const std::string & clean_haystack, + const std::string & fun_name_needle) { // we need to split by markers... auto haystack_split = segmentize_markers(trim_leading_whitespace(clean_haystack)); int where_is_nemo = 0; @@ -871,7 +853,7 @@ void differential_analyzer::analyze_tool_call_format_non_json(const std::string } } -void differential_analyzer::check_per_call_markers(const common_chat_template & tmpl, tool_format_analysis & result) { +void analyze_tools::check_per_call_markers() { json assistant_one_tool = json{ { "role", "assistant" }, { "content", "" }, @@ -891,7 +873,7 @@ void differential_analyzer::check_per_call_markers(const common_chat_template & params.enable_thinking = true; auto one_vs_two = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_tools }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_tools }); }); if (!one_vs_two) { LOG_DBG(ANSI_ORANGE "%s: Generating double tool call comparison failed\n" ANSI_RESET, __func__); @@ -901,18 +883,16 @@ void differential_analyzer::check_per_call_markers(const common_chat_template & diff_split filter_common_call_part = calculate_diff_split(one_vs_two->diff.suffix, one_vs_two->diff.right); std::string second_tool_content = trim_leading_whitespace(filter_common_call_part.right); - if (!result.section_start.empty() && - second_tool_content.find(result.section_start) == 0) { - result.per_call_start = result.section_start; - result.per_call_end = result.section_end; - result.section_start.clear(); - result.section_end.clear(); + if (!format.section_start.empty() && + second_tool_content.find(format.section_start) == 0) { + format.per_call_start = format.section_start; + format.per_call_end = format.section_end; + format.section_start.clear(); + format.section_end.clear(); } } -tool_function_analysis differential_analyzer::extract_function_markers(const common_chat_template & tmpl, const tool_format_analysis & analysis) { - tool_function_analysis result; - +void analyze_tools::extract_function_markers() { json assistant_nocall = json{ { "role", "assistant" }, { "content", "BBBB" }, @@ -937,37 +917,37 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_barbar }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_barbar }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); - return result; + return; } const auto & diff = comparison->diff; if (diff.left.find("foofoo") != std::string::npos && diff.right.find("barbar") != std::string::npos) { std::string prefix_marker; - if (!analysis.per_call_start.empty()) { - prefix_marker = analysis.per_call_start; + if (!format.per_call_start.empty()) { + prefix_marker = format.per_call_start; } else { - prefix_marker = analysis.section_start; + prefix_marker = format.section_start; } if (!prefix_marker.empty() && diff.prefix.rfind(prefix_marker) != std::string::npos) { - result.name_prefix = + function.name_prefix = diff.prefix.substr(diff.prefix.rfind(prefix_marker) + prefix_marker.size()); } auto seg = segmentize_markers(diff.left); for (const auto & s : seg) { if (s.value.find("foofoo") == std::string::npos) { - result.name_prefix += s.value; + function.name_prefix += s.value; } else { size_t pos = s.value.find("foofoo"); std::string pre = s.value.substr(0, pos); std::string post = s.value.substr(pos + 6); // 6 = len("foofoo") - result.name_prefix += pre; - result.name_suffix += post; + function.name_prefix += pre; + function.name_suffix += post; break; } } @@ -977,7 +957,7 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com size_t stop_internal_pos = 0; for (const auto & ss : seg_suf) { bool has_needle = false; - if (analysis.mode == tool_format::TAG_WITH_JSON) { + if (format.mode == tool_format::TAG_WITH_JSON) { has_needle = (ss.type == segment_type::TEXT && ss.value.find_first_of("{[") != std::string::npos); if (has_needle) { stop_internal_pos = ss.value.find_first_of("{["); @@ -993,7 +973,7 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com stop++; } if (stop < seg_suf.size() - 1) { - if (analysis.mode == tool_format::TAG_WITH_TAGGED) { + if (format.mode == tool_format::TAG_WITH_TAGGED) { size_t how_far = 0; if (stop > 0) { if (seg_suf[stop].type == segment_type::MARKER) { @@ -1002,30 +982,30 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com how_far = stop - 1; } for (size_t i = 0; i < how_far; i++) { - result.name_suffix += seg_suf[i].value; + function.name_suffix += seg_suf[i].value; } } } else { for (size_t i = 0; i < stop; i++) { - result.name_suffix += seg_suf[i].value; + function.name_suffix += seg_suf[i].value; } const std::string & stopper = seg_suf[stop].value; - result.name_suffix += stopper.substr(0, stop_internal_pos); + function.name_suffix += stopper.substr(0, stop_internal_pos); } } // now just to find the closer std::string suffix_marker; - if (!analysis.per_call_end.empty()) { - suffix_marker = analysis.per_call_end; + if (!format.per_call_end.empty()) { + suffix_marker = format.per_call_end; } else { - suffix_marker = analysis.section_end; + suffix_marker = format.section_end; } std::string closer_suffix; if (suffix_marker.empty()) { // we'll have to rely on an extra diff with no-calls version auto notool_comp = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_nocall }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_nocall }); }); auto nt_diff = notool_comp->diff; closer_suffix = nt_diff.left.substr(nt_diff.left.find("YYYY") + 4); } else { @@ -1033,18 +1013,18 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com } if (!closer_suffix.empty()) { auto closer_seg = segmentize_markers(closer_suffix); - bool need_to_eat_arg_marker = (analysis.mode == tool_format::TAG_WITH_TAGGED); + bool need_to_eat_arg_marker = (format.mode == tool_format::TAG_WITH_TAGGED); size_t last_arg_seg = closer_seg.size() - 1; for (int i = (int) closer_seg.size() - 1; i >= 0; i--) { if (closer_seg[i].value.find("YYYY") != std::string::npos) { last_arg_seg = i; } } - if (analysis.mode == tool_format::TAG_WITH_JSON) { + if (format.mode == tool_format::TAG_WITH_JSON) { const auto & entire_seg = closer_seg[last_arg_seg].value; size_t pos = entire_seg.find_last_of("}]"); if (pos != std::string::npos && pos < entire_seg.size() - 1) { - result.close = trim_leading_whitespace(entire_seg.substr(pos + 1)); + function.close = trim_leading_whitespace(entire_seg.substr(pos + 1)); } } for (size_t i = last_arg_seg + 1; i < closer_seg.size(); i++) { @@ -1052,31 +1032,25 @@ tool_function_analysis differential_analyzer::extract_function_markers(const com if (need_to_eat_arg_marker) { need_to_eat_arg_marker = false; } else { - result.close += closer_seg[i].value; + function.close += closer_seg[i].value; } } else if (!need_to_eat_arg_marker) { - result.close += closer_seg[i].value; + function.close += closer_seg[i].value; } } } - result.close = trim_leading_whitespace(result.close); + function.close = trim_leading_whitespace(function.close); } - return result; } -tool_arguments_analysis differential_analyzer::analyze_arguments(const common_chat_template & tmpl, const tool_analysis & tool_analysis) { +void analyze_tools::analyze_arguments() { LOG_DBG(ANSI_ORANGE "Phase 4: Argument analysis\n" ANSI_RESET); - tool_arguments_analysis result; - - extract_argument_name_markers(tmpl, result); - extract_argument_value_markers(tmpl, tool_analysis, result); - - return result; + extract_argument_name_markers(); + extract_argument_value_markers(); } -void differential_analyzer::extract_argument_name_markers(const common_chat_template & tmpl, - tool_arguments_analysis & args_analysis) { +void analyze_tools::extract_argument_name_markers() { json assistant_first_arg = json{ { "role", "assistant" }, { "content", "" }, @@ -1096,7 +1070,7 @@ void differential_analyzer::extract_argument_name_markers(const common_chat_temp params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_second_arg }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_second_arg }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); @@ -1125,11 +1099,11 @@ void differential_analyzer::extract_argument_name_markers(const common_chat_temp std::string right_name = right_remainder.substr(0, 6); // 6 = len("second") if (left_name == "first" && right_name == "second") { - args_analysis.name_prefix = trim_whitespace(common_prefix); + arguments.name_prefix = trim_whitespace(common_prefix); std::string suffix_left = left_remainder.substr(5, left_close - 5); std::string suffix_right = right_remainder.substr(6, right_close - 6); if (suffix_left == suffix_right) { - args_analysis.name_suffix = trim_leading_whitespace(suffix_left); + arguments.name_suffix = trim_leading_whitespace(suffix_left); } } } @@ -1137,22 +1111,22 @@ void differential_analyzer::extract_argument_name_markers(const common_chat_temp // we most likely have actual markers for argument names auto pre_seg = segmentize_markers(diff.prefix); for (int i = pre_seg.size() - 1; i >= 0; i--) { - args_analysis.name_prefix = args_analysis.name_prefix + pre_seg[i].value; + arguments.name_prefix = arguments.name_prefix + pre_seg[i].value; if (pre_seg[i].type == segment_type::MARKER) { break; } } auto left_seg = segmentize_markers(diff.left); if (left_seg.size() == 1) { // only the name + maybe extra whitespace / normal chars in differing part - args_analysis.name_suffix = diff.left.substr(5); + arguments.name_suffix = diff.left.substr(5); auto suf_seg= segmentize_markers(diff.suffix); for (size_t i = 0; i < suf_seg.size(); i++) { - args_analysis.name_suffix += suf_seg[i].value; + arguments.name_suffix += suf_seg[i].value; if (suf_seg[i].type == segment_type::MARKER) { if (i < suf_seg.size() - 2 && suf_seg[i + 1].type == segment_type::TEXT && trim_whitespace(suf_seg[i + 1].value).empty()) { // we need to include post-marker whitespace/newlines as well - args_analysis.name_suffix += suf_seg[i + 1].value; + arguments.name_suffix += suf_seg[i + 1].value; } break; } @@ -1165,12 +1139,12 @@ void differential_analyzer::extract_argument_name_markers(const common_chat_temp } else { to_add = left_seg[i].value; } - args_analysis.name_suffix += to_add; + arguments.name_suffix += to_add; if (left_seg[i].type == segment_type::MARKER) { if (i < left_seg.size() - 2 && left_seg[i + 1].type == segment_type::TEXT && trim_whitespace(left_seg[i + 1].value).empty()) { // we need to include post-marker whitespace/newlines as well - args_analysis.name_suffix += left_seg[i + 1].value; + arguments.name_suffix += left_seg[i + 1].value; } break; } @@ -1180,9 +1154,7 @@ void differential_analyzer::extract_argument_name_markers(const common_chat_temp } } -void differential_analyzer::extract_argument_value_markers(const common_chat_template & tmpl, - const tool_analysis & analysis, - tool_arguments_analysis & args_analysis) { +void analyze_tools::extract_argument_value_markers() { json assistant_val_X = json{ { "role", "assistant" }, { "content", "" }, @@ -1202,7 +1174,7 @@ void differential_analyzer::extract_argument_value_markers(const common_chat_tem params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_val_Y }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_val_Y }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); @@ -1212,7 +1184,7 @@ void differential_analyzer::extract_argument_value_markers(const common_chat_tem const auto & diff = comparison->diff; if (diff.left == "XXXX" && diff.right == "YYYY") { - std::string arg_name_ending = "first" + args_analysis.name_suffix; + std::string arg_name_ending = "first" + arguments.name_suffix; std::string prefix = diff.prefix; if (prefix.rfind(arg_name_ending) != std::string::npos) { prefix = prefix.substr(prefix.rfind(arg_name_ending) + arg_name_ending.size()); @@ -1220,7 +1192,7 @@ void differential_analyzer::extract_argument_value_markers(const common_chat_tem if (!prefix.empty()) { auto seg_pre = segmentize_markers(prefix); for (int i = seg_pre.size() - 1; i >= 0; i--) { - args_analysis.value_prefix = seg_pre[i].value + args_analysis.value_prefix; + arguments.value_prefix = seg_pre[i].value + arguments.value_prefix; if (seg_pre[i].type == segment_type::MARKER) { break; } @@ -1228,14 +1200,14 @@ void differential_analyzer::extract_argument_value_markers(const common_chat_tem } std::string value_suffix = diff.suffix; - if (!analysis.function.close.empty()) { - size_t func_close_pos = value_suffix.find(analysis.function.close); + if (!function.close.empty()) { + size_t func_close_pos = value_suffix.find(function.close); if (func_close_pos != std::string::npos) { value_suffix = value_suffix.substr(0, func_close_pos); } - } else if (!analysis.format.per_call_end.empty() || !analysis.format.section_end.empty()) { + } else if (!format.per_call_end.empty() || !format.section_end.empty()) { std::string end_marker = - !analysis.format.per_call_end.empty() ? analysis.format.per_call_end : analysis.format.section_end; + !format.per_call_end.empty() ? format.per_call_end : format.section_end; size_t end_marker_pos = value_suffix.find(end_marker); if (end_marker_pos != std::string::npos) { value_suffix = value_suffix.substr(0, end_marker_pos); @@ -1243,13 +1215,12 @@ void differential_analyzer::extract_argument_value_markers(const common_chat_tem } value_suffix = trim_leading_whitespace(value_suffix); if (!value_suffix.empty()) { - args_analysis.value_suffix = value_suffix; + arguments.value_suffix = value_suffix; } } } -void differential_analyzer::extract_argument_separator(const common_chat_template & tmpl, - tool_arguments_analysis & args_analysis) { +void analyze_tools::extract_argument_separator() { json assistant_one_arg = json{ { "role", "assistant" }, { "content", "" }, @@ -1269,7 +1240,7 @@ void differential_analyzer::extract_argument_separator(const common_chat_templat params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_args }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_args }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); @@ -1280,13 +1251,11 @@ void differential_analyzer::extract_argument_separator(const common_chat_templat if (!diff.right.empty()) { std::string separator = until_common_prefix(diff.right, "first", "second"); - args_analysis.separator = separator; + arguments.separator = separator; } } -void differential_analyzer::extract_args_markers(const common_chat_template & tmpl, - const tool_analysis & analysis, - tool_arguments_analysis & args_analysis) { +void analyze_tools::extract_args_markers() { json assistant_no_args = json{ { "role", "assistant"}, { "content", "" }, @@ -1306,7 +1275,7 @@ void differential_analyzer::extract_args_markers(const common_chat_template & tm params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_args }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_args }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__); @@ -1315,9 +1284,9 @@ void differential_analyzer::extract_args_markers(const common_chat_template & tm const auto & diff = comparison->diff; - if (analysis.format.mode != tool_format::JSON_NATIVE) { - std::string prefix_marker = !analysis.format.section_start.empty() ? analysis.format.section_start : analysis.format.per_call_start; - std::string suffix_marker = !analysis.format.section_end.empty() ? analysis.format.section_end : analysis.format.per_call_end; + if (format.mode != tool_format::JSON_NATIVE) { + std::string prefix_marker = !format.section_start.empty() ? format.section_start : format.per_call_start; + std::string suffix_marker = !format.section_end.empty() ? format.section_end : format.per_call_end; // these might happen earlier in the tools section as an example or somewhere else, so we need to find the closest ones size_t prefix_pos = prefix_marker.empty() ? 0 : diff.prefix.rfind(prefix_marker); size_t suffix_pos = suffix_marker.empty() ? diff.suffix.size() : diff.suffix.find(suffix_marker); @@ -1333,15 +1302,13 @@ void differential_analyzer::extract_args_markers(const common_chat_template & tm std::string args_end = after_common_suffix(suffix_cut, "{}", "\"XXXX\"}"); if (!args_start.empty() || !args_end.empty()) { - args_analysis.start = args_start; - args_analysis.end = args_end; + arguments.start = args_start; + arguments.end = args_end; } } } -tool_id_analysis differential_analyzer::extract_call_id_markers(const common_chat_template & tmpl, tool_format_analysis & analysis) { - tool_id_analysis result; - +void analyze_tools::extract_call_id_markers() { json assistant_id1 = json{ { "role", "assistant" }, { "content", "" }, @@ -1361,17 +1328,17 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha params.enable_thinking = true; auto comparison = compare_variants( - tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_id2 }); }); + *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_id2 }); }); if (!comparison) { LOG_DBG(ANSI_ORANGE "%s: Template application failed for call_id detection\n" ANSI_RESET, __func__); - return result; + return; } const auto & diff = comparison->diff; if (diff.left.empty() && diff.right.empty()) { - return result; + return; } std::string id_value_1 = "call00001"; @@ -1402,7 +1369,7 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha if (args_in_suffix != std::string::npos && (args_in_prefix == std::string::npos || args_in_prefix > diff.prefix.length())) { // Args are in suffix, so call_id is BETWEEN_FUNC_AND_ARGS - result.pos = call_id_position::BETWEEN_FUNC_AND_ARGS; + call_id.pos = call_id_position::BETWEEN_FUNC_AND_ARGS; // The prefix ends with: ... // Segmentize to find the call_id_prefix marker @@ -1427,12 +1394,12 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha } if (!marker_before_id.empty()) { - result.prefix = marker_before_id; + call_id.prefix = marker_before_id; } else { // Fallback: look for the last marker in after_func for (int i = (int) segments.size() - 1; i >= 0; i--) { if (segments[i].type == segment_type::MARKER) { - result.prefix = segments[i].value; + call_id.prefix = segments[i].value; break; } } @@ -1442,7 +1409,7 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha auto suffix_segments = segmentize_markers(diff.suffix); for (size_t i = 0; i < suffix_segments.size(); i++) { if (suffix_segments[i].type == segment_type::MARKER) { - result.suffix = suffix_segments[i].value; + call_id.suffix = suffix_segments[i].value; break; } // Stop if we hit the args @@ -1452,7 +1419,7 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha } } else if (args_in_prefix != std::string::npos) { // Args are in prefix, so call_id is POST_ARGS - result.pos = call_id_position::POST_ARGS; + call_id.pos = call_id_position::POST_ARGS; // Extract markers from between args and the ID std::string after_args = diff.prefix.substr(args_in_prefix); @@ -1462,7 +1429,7 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha auto segments = segmentize_markers(between_args_and_id); for (int i = (int) segments.size() - 1; i >= 0; i--) { if (segments[i].type == segment_type::MARKER) { - result.prefix = segments[i].value; + call_id.prefix = segments[i].value; break; } } @@ -1472,20 +1439,20 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha auto suffix_segments = segmentize_markers(diff.suffix); for (const auto & seg : suffix_segments) { if (seg.type == segment_type::MARKER) { - result.suffix = seg.value; + call_id.suffix = seg.value; break; } } } } else if (func_name_in_suffix != std::string::npos && func_name_in_prefix == std::string::npos) { // Function name is only in suffix - call_id is PRE_FUNC_NAME - result.pos = call_id_position::PRE_FUNC_NAME; + call_id.pos = call_id_position::PRE_FUNC_NAME; // Extract call_id_prefix from prefix (last marker before the common_id_part) auto prefix_segments = segmentize_markers(diff.prefix); for (int i = (int) prefix_segments.size() - 1; i >= 0; i--) { if (prefix_segments[i].type == segment_type::MARKER) { - result.prefix = prefix_segments[i].value; + call_id.prefix = prefix_segments[i].value; break; } } @@ -1495,7 +1462,7 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha auto suffix_segments = segmentize_markers(before_func); for (const auto & seg : suffix_segments) { if (seg.type == segment_type::MARKER) { - result.suffix = seg.value; + call_id.suffix = seg.value; break; } } @@ -1503,45 +1470,10 @@ tool_id_analysis differential_analyzer::extract_call_id_markers(const common_cha // When call_id is detected, per_call_end may have been incorrectly set to include // the call_id_suffix and sample args. Clear it if it starts with call_id_suffix. - if (result.pos != call_id_position::NONE && !result.suffix.empty() && - analysis.per_call_end.find(result.suffix) == 0) { - analysis.per_call_end.clear(); + if (call_id.pos != call_id_position::NONE && !call_id.suffix.empty() && + format.per_call_end.find(call_id.suffix) == 0) { + format.per_call_end.clear(); } - - return result; } -void differential_analyzer::collect_preserved_tokens(diff_analysis_result & result) { - auto & tokens = result.preserved_tokens; - - auto add_token = [&tokens](const std::string & org_token) { - std::string token = trim_whitespace(org_token); - if (!token.empty()) { - // Avoid duplicates - if (std::find(tokens.begin(), tokens.end(), token) == tokens.end()) { - tokens.push_back(token); - } - } - }; - - add_token(result.reasoning.start); - add_token(result.reasoning.end); - add_token(result.content.start); - add_token(result.content.end); - add_token(result.tools.format.section_start); - add_token(result.tools.format.section_end); - add_token(result.tools.format.per_call_start); - add_token(result.tools.format.per_call_end); - add_token(result.tools.function.name_prefix); - add_token(result.tools.function.name_suffix); - add_token(result.tools.function.close); - add_token(result.tools.arguments.start); - add_token(result.tools.arguments.end); - add_token(result.tools.arguments.name_prefix); - add_token(result.tools.arguments.name_suffix); - add_token(result.tools.arguments.separator); - add_token(result.tools.arguments.value_prefix); - add_token(result.tools.arguments.value_suffix); - add_token(result.tools.call_id.prefix); - add_token(result.tools.call_id.suffix); -} +} // namespace autoparser diff --git a/common/chat-diff-analyzer.h b/common/chat-diff-analyzer.h index b1bfc83283..a94c40459c 100644 --- a/common/chat-diff-analyzer.h +++ b/common/chat-diff-analyzer.h @@ -2,6 +2,7 @@ #include "chat.h" #include "jinja/caps.h" +#include "peg-parser.h" #include "nlohmann/json.hpp" #include @@ -12,6 +13,8 @@ using json = nlohmann::ordered_json; +class common_chat_peg_unified_builder; + // ============================================================================ // Parameters for template application // ============================================================================ @@ -41,6 +44,10 @@ struct compare_variants_result { std::string output_B; }; +namespace autoparser { + +struct templates_params; + // ============================================================================ // Marker Registry: All markers extracted via differential analysis // ============================================================================ @@ -182,21 +189,9 @@ inline std::ostream & operator<<(std::ostream & os, const tool_format & format) } } -struct reasoning_analysis { - reasoning_mode mode = reasoning_mode::NONE; - - std::string start; // e.g., "", "[THINK]", "<|START_THINKING|>", "" - std::string end; // e.g., "", "[BEGIN FINAL RESPONSE]", "<|END_THINKING|>" -}; - -struct content_analysis { - content_mode mode = content_mode::PLAIN; - - std::string start; // e.g., "", ">>>all\n", "" - std::string end; // e.g., "", "" - - bool requires_nonnull_content = false; -}; +// ============================================================================ +// Sub-structs for tool analysis +// ============================================================================ struct tool_format_analysis { tool_format mode = tool_format::NONE; @@ -240,127 +235,176 @@ struct tool_id_analysis { std::string suffix; // e.g., "" (marker after call ID value, before next section) }; -struct tool_analysis { +// ============================================================================ +// Parser build context (shared interface for build_parser methods) +// ============================================================================ + +struct analyze_content; + +struct parser_build_context { + common_chat_peg_unified_builder & p; + const templates_params & inputs; + common_peg_parser reasoning_parser; + bool extracting_reasoning = false; + const analyze_content * content = nullptr; + + parser_build_context(common_chat_peg_unified_builder & p, const templates_params & inputs); +}; + +// ============================================================================ +// Base class for analyzers with parser building +// ============================================================================ + +struct analyze_base { + virtual ~analyze_base() = default; + virtual common_peg_parser build_parser(parser_build_context & ctx) const = 0; + + protected: + const common_chat_template * tmpl = nullptr; + + analyze_base() = default; + explicit analyze_base(const common_chat_template & tmpl) : tmpl(&tmpl) {} +}; + +// ============================================================================ +// Reasoning analyzer +// ============================================================================ + +struct analyze_reasoning : analyze_base { + reasoning_mode mode = reasoning_mode::NONE; + + std::string start; // e.g., "", "[THINK]", "<|START_THINKING|>", "" + std::string end; // e.g., "", "[BEGIN FINAL RESPONSE]", "<|END_THINKING|>" + + analyze_reasoning() = default; + analyze_reasoning(const common_chat_template & tmpl, bool supports_tools); + + common_peg_parser build_parser(parser_build_context & ctx) const override; + + private: + // Look for reasoning markers in rendered content + void compare_reasoning_presence(); + + // Compare generation prompt with enable_thinking=true vs false + void compare_thinking_enabled(); + + // Check if reasoning is always possible or only in tool calls + void compare_reasoning_scope(); +}; + +// ============================================================================ +// Content analyzer +// ============================================================================ + +struct analyze_content : analyze_base { + content_mode mode = content_mode::PLAIN; + + std::string start; // e.g., "", ">>>all\n", "" + std::string end; // e.g., "", "" + + bool requires_nonnull_content = false; + + analyze_content() = default; + analyze_content(const common_chat_template & tmpl, const analyze_reasoning & reasoning); + + common_peg_parser build_parser(parser_build_context & ctx) const override; + + bool is_always_wrapped() const; + common_peg_parser build_optional_wrapped(parser_build_context & ctx) const; +}; + +// ============================================================================ +// Tool analyzer +// ============================================================================ + +struct analyze_tools : analyze_base { tool_format_analysis format; tool_function_analysis function; tool_arguments_analysis arguments; tool_id_analysis call_id; + + analyze_tools() = default; + analyze_tools(const common_chat_template & tmpl, + const jinja::caps & caps, + const analyze_reasoning & reasoning); + + common_peg_parser build_parser(parser_build_context & ctx) const override; + + private: + // Extract tool calling 'haystack' for further analysis and delegate further analysis based on format + void analyze_tool_calls(const analyze_reasoning & reasoning); + + // Analyze format based on position of function and argument name in needle + void analyze_tool_call_format(const std::string & haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle, + const analyze_reasoning & reasoning); + + // Analyze specifics of JSON native format (entire tool call is a JSON object) + void analyze_tool_call_format_json_native(const std::string & clean_haystack, + const std::string & fun_name_needle, + const std::string & arg_name_needle); + + // Analyze specifics of non-JSON native format (tags for function name or for function name and arguments) + void analyze_tool_call_format_non_json(const std::string & clean_haystack, + const std::string & fun_name_needle); + + // Check for and extract specific per-call markers for non-native-JSON templates with parallel call support + void check_per_call_markers(); + + // Extract function name markers + void extract_function_markers(); + + // Delegates to separate functions for: separator analysis, argument name analysis, argument value analysis + void analyze_arguments(); + + // Extract argument name markers + void extract_argument_name_markers(); + + // Extract argument value markers + void extract_argument_value_markers(); + + // Extract argument separator, if specified (eg. ......) + void extract_argument_separator(); + + // Extract argument wrapper markers, if present (eg. '......') + void extract_args_markers(); + + // Extract call ID markers, if present + void extract_call_id_markers(); + + // Per-format tool parser builders + common_peg_parser build_tool_parser_json_native(parser_build_context & ctx) const; + common_peg_parser build_tool_parser_tag_json(parser_build_context & ctx) const; + common_peg_parser build_tool_parser_tag_tagged(parser_build_context & ctx) const; }; -// Complete result of differential analysis -struct diff_analysis_result { +// ============================================================================ +// Top-level template analyzer (merges differential_analyzer + diff_analysis_result) +// ============================================================================ + +struct analyze_template { jinja::caps jinja_caps; - reasoning_analysis reasoning; - content_analysis content; - tool_analysis tools; + analyze_reasoning reasoning; + analyze_content content; + analyze_tools tools; // Preserved tokens for tokenizer (union of all non-empty markers) std::vector preserved_tokens; -}; -// Performs systematic differential analysis on chat templates -// Uses comparison matrix to extract markers without heuristics -class differential_analyzer { - public: - // Main entry point: Run full differential analysis on a template - static diff_analysis_result analyze(const common_chat_template & tmpl); + // Constructor: runs full differential analysis on a template + explicit analyze_template(const common_chat_template & tmpl); - // Phase-specific analysis (can be called individually for testing) - static reasoning_analysis analyze_reasoning(const common_chat_template & tmpl, bool supports_tools); - static content_analysis analyze_content(const common_chat_template & tmpl, const reasoning_analysis & reasoning); - static tool_analysis analyze_tools(const common_chat_template & tmpl, - const jinja::caps & caps, - const reasoning_analysis & reasoning); - - // Factorized differential comparison function (public for testing) - // Takes base params and a single modifier lambda to create variant B - // Returns compare_variants_result containing diff and both outputs, or std::nullopt on failure - static std::optional compare_variants( - const common_chat_template & tmpl, - const template_params & params_A, - const std::function & params_modifier); + // Build the unified PEG parser for this template + common_peg_arena build_parser(const templates_params & inputs) const; private: - // Comparison helpers (implement the comparison matrix from the plan) - - // 1. Reasoning analysis: - // Look for reasoning markers in rendered content - static void compare_reasoning_presence(const common_chat_template & tmpl, reasoning_analysis & reasoning); - - // Compare generation prompt with enable_thinking=true vs false - static void compare_thinking_enabled(const common_chat_template & tmpl, reasoning_analysis & reasoning); - - // Check if reasoning is always possible or only in tool calls - static void compare_reasoning_scope(const common_chat_template & tmpl, reasoning_analysis & reasoning); - - // 2. Content (fully inside analyze_content mentioned above) - - // 3. Tool calls - // a. format - // Extract tool calling 'haystack' for further analysis and delegate further analysis based on format - static tool_format_analysis analyze_tool_calls(const common_chat_template & tmpl, - const reasoning_analysis & reasoning); - - // Analyze format based on position of function and argument name in needle - static tool_format_analysis analyze_tool_call_format(const std::string & haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - const reasoning_analysis & reasoning); - - // Analyze specifics of JSON native format (entire tool call is a JSON object) - static void analyze_tool_call_format_json_native(const std::string & clean_haystack, - const std::string & fun_name_needle, - const std::string & arg_name_needle, - tool_format_analysis & format); - - // Analyze specifics of non-JSON native format (tags for function name or for function name and arguments) - static void analyze_tool_call_format_non_json(const std::string & clean_haystack, - const std::string & fun_name_needle, - tool_format_analysis & format); - - // Check for and extract specific per-call markers for non-native-JSON templates with parallel call support - static void check_per_call_markers(const common_chat_template & tmpl, tool_format_analysis & result); - - // Logic below is only for non-JSON-native tool calling formats - // 3. b. function name - // Extract function name markers - static tool_function_analysis extract_function_markers(const common_chat_template & tmpl, - const tool_format_analysis & analysis); - - // 4. c. function arguments - // Delegates to separate functions for: separator analysis, argument name analysis, argument value analysis - static tool_arguments_analysis analyze_arguments(const common_chat_template & tmpl, - const tool_analysis & analysis); - - // Extract argument name markers - static void extract_argument_name_markers(const common_chat_template & tmpl, - tool_arguments_analysis & args_analysis); - - // Extract argument value markers - static void extract_argument_value_markers(const common_chat_template & tmpl, - const tool_analysis & analysis, - tool_arguments_analysis & args_analysis); - - // Extract argument separator, if specified (eg. ......) - static void extract_argument_separator(const common_chat_template & tmpl, - tool_arguments_analysis & args_analysis); - - // Extract argument wrapper markers, if present (eg. '......') - static void extract_args_markers(const common_chat_template & tmpl, - const tool_analysis & analysis, - tool_arguments_analysis & args_analysis); - - // 4. d. function call id - // Extract call ID markers, if present - static tool_id_analysis extract_call_id_markers(const common_chat_template & tmpl, - tool_format_analysis & analysis); - // Collect tokens from entire analysis to preserve - static void collect_preserved_tokens(diff_analysis_result & result); - - static std::string apply_template(const common_chat_template & tmpl, const template_params & params); + void collect_preserved_tokens(); }; +} // namespace autoparser + enum segment_type { TEXT, MARKER }; inline std::ostream & operator<<(std::ostream & os, const segment_type & type) { diff --git a/common/chat.cpp b/common/chat.cpp index be4e19aebc..abc93392aa 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -239,8 +239,8 @@ bool common_chat_templates_support_enable_thinking(const common_chat_templates * const auto & tmpl = chat_templates->template_tool_use ? *chat_templates->template_tool_use : *chat_templates->template_default; - diff_analysis_result result = differential_analyzer::analyze(tmpl); - detect |= result.reasoning.mode != reasoning_mode::NONE; + autoparser::analyze_template result(tmpl); + detect |= result.reasoning.mode != autoparser::reasoning_mode::NONE; return detect; } @@ -752,7 +752,7 @@ static void foreach_parameter(const json & std::string common_chat_template_direct_apply( const common_chat_template & tmpl, - const struct templates_params & inputs, + const autoparser::templates_params & inputs, const std::optional & messages_override, const std::optional & tools_override, const std::optional & additional_context) { @@ -803,7 +803,7 @@ std::string common_chat_template_direct_apply( } static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, - const struct templates_params & inputs) { + const autoparser::templates_params & inputs) { common_chat_params data; // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja @@ -917,7 +917,7 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ } static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, - const struct templates_params & inputs) { + const autoparser::templates_params & inputs) { common_chat_params data; // Copy reasoning to the "thinking" field as expected by the gpt-oss template @@ -1063,7 +1063,7 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp // Functionary v3.2 - uses recipient-based format: >>>recipient\n{content} static common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, - const struct templates_params & inputs) { + const autoparser::templates_params & inputs) { common_chat_params data; data.prompt = common_chat_template_direct_apply(tmpl, inputs); @@ -1116,16 +1116,14 @@ static common_chat_params common_chat_params_init_functionary_v3_2(const common_ if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { if (inputs.parallel_tool_calls) { return p.choice({ content_and_tools, tools_only }) + p.end(); - } else { - return p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); } - } else { - if (inputs.parallel_tool_calls) { - return p.choice({ content_and_tools, content_only, tools_only }) + p.end(); - } - auto content_and_tool = content_until_tool + tool_choice; - return p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); + return p.choice({ content_until_tool + tool_choice, tools_only }) + p.end(); } + if (inputs.parallel_tool_calls) { + return p.choice({ content_and_tools, content_only, tools_only }) + p.end(); + } + auto content_and_tool = content_until_tool + tool_choice; + return p.choice({ content_and_tool, content_only, tool_choice }) + p.end(); }); data.parser = parser.save(); @@ -1204,7 +1202,7 @@ static void func_args_not_string(json & messages) { static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls, const struct common_chat_templates_inputs & inputs) { - templates_params params; + autoparser::templates_params params; params.tools = common_chat_tools_to_json_oaicompat(inputs.tools); const auto & tmpl = params.tools.is_array() && tmpls->template_tool_use ? *tmpls->template_tool_use @@ -1282,7 +1280,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ try { LOG_DBG("Using differential autoparser\n"); - auto auto_params = universal_peg_generator::generate_parser(tmpl, params); + auto auto_params = autoparser::universal_peg_generator::generate_parser(tmpl, params); return auto_params; } catch (const std::exception & e) { LOG_WRN("Automatic parser generation failed: %s\n", e.what()); diff --git a/common/chat.h b/common/chat.h index 00f8eb62b6..0492b82c44 100644 --- a/common/chat.h +++ b/common/chat.h @@ -23,6 +23,10 @@ using json = nlohmann::ordered_json; struct common_chat_templates; +namespace autoparser { +struct templates_params; +} // namespace autoparser + struct common_chat_tool_call { std::string name; std::string arguments; @@ -294,7 +298,7 @@ std::map common_chat_templates_get_caps(const common_chat_tem std::string common_chat_template_direct_apply( const common_chat_template & tmpl, - const struct templates_params & inputs, + const autoparser::templates_params & inputs, const std::optional & messages_override = std::nullopt, const std::optional & tools_override = std::nullopt, const std::optional & additional_context = std::nullopt); diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 4f3f7f5ec2..c78428491f 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -10,6 +10,8 @@ #include #include +using namespace autoparser; + static void test_calculate_diff_split_basic(testing & t); static void test_calculate_diff_split_identical(testing & t); static void test_calculate_diff_split_common_prefix(testing & t); @@ -591,7 +593,7 @@ static void test_compare_variants_basic(testing & t) { p.messages[0]["content"] = "World"; }; - auto result = differential_analyzer::compare_variants(tmpl, params, modifier); + auto result = autoparser::compare_variants(tmpl, params, modifier); if (!t.assert_true("result should have value", result.has_value())) { return; @@ -614,7 +616,7 @@ static void test_compare_variants_messages_modifier(testing & t) { p.messages[0]["content"] = "B"; }; - std::optional result = differential_analyzer::compare_variants(tmpl, params, modifier); + std::optional result = autoparser::compare_variants(tmpl, params, modifier); if (!t.assert_true("result should have value", result.has_value())) { return; @@ -637,7 +639,7 @@ static void test_compare_variants_tools_modifier(testing & t) { p.tools[0]["name"] = "bar"; }; - auto result = differential_analyzer::compare_variants(tmpl, params, modifier); + auto result = autoparser::compare_variants(tmpl, params, modifier); if (!t.assert_true("result should have value", result.has_value())) { return; @@ -661,7 +663,7 @@ static void test_compare_variants_both_modifiers(testing & t) { p.messages[0]["role"] = "newuser"; }; - auto result = differential_analyzer::compare_variants(tmpl, params, modifier); + auto result = autoparser::compare_variants(tmpl, params, modifier); if (!t.assert_true("result should have value", result.has_value())) { return; @@ -684,7 +686,7 @@ static void test_compare_variants_template_failure(testing & t) { p.messages[0]["content"] = "World"; }; - auto result = differential_analyzer::compare_variants(tmpl, params, modifier); + auto result = autoparser::compare_variants(tmpl, params, modifier); t.assert_true("result should be nullopt on template failure", !result.has_value()); } @@ -699,7 +701,7 @@ static void test_compare_variants_identity(testing & t) { }); // No modifier - should use identity - auto result = differential_analyzer::compare_variants(tmpl, params, nullptr); + auto result = autoparser::compare_variants(tmpl, params, nullptr); if (!t.assert_true("result should have value", result.has_value())) { return; @@ -810,7 +812,7 @@ static void test_seed_oss_tool_presence(testing & t) { params_with_tools.add_generation_prompt = false; params_with_tools.enable_thinking = true; - auto result = differential_analyzer::compare_variants(tmpl, params_no_tools, + auto result = autoparser::compare_variants(tmpl, params_no_tools, [&](template_params & p) { p.messages = params_with_tools.messages; }); @@ -872,7 +874,7 @@ static void test_seed_oss_call_count(testing & t) { params_one.add_generation_prompt = false; params_one.enable_thinking = true; - auto result = differential_analyzer::compare_variants(tmpl, params_one, + auto result = autoparser::compare_variants(tmpl, params_one, [&](template_params & p) { p.messages = json::array({user_msg, assistant_two_calls}); }); @@ -964,7 +966,7 @@ static void test_seed_oss_function_names(testing & t) { params_alpha.add_generation_prompt = false; params_alpha.enable_thinking = true; - auto result = differential_analyzer::compare_variants(tmpl, params_alpha, + auto result = autoparser::compare_variants(tmpl, params_alpha, [&](template_params & p) { p.messages = json::array({user_msg, assistant_func_beta}); }); @@ -1068,7 +1070,7 @@ static void test_seed_oss_argument_count(testing & t) { params_zero.add_generation_prompt = false; params_zero.enable_thinking = true; - auto result_zero_one = differential_analyzer::compare_variants(tmpl, params_zero, + auto result_zero_one = autoparser::compare_variants(tmpl, params_zero, [&](template_params & p) { p.messages = json::array({user_msg, assistant_one_arg}); }); @@ -1086,7 +1088,7 @@ static void test_seed_oss_argument_count(testing & t) { params_one.add_generation_prompt = false; params_one.enable_thinking = true; - auto result_one_two = differential_analyzer::compare_variants(tmpl, params_one, + auto result_one_two = autoparser::compare_variants(tmpl, params_one, [&](template_params & p) { p.messages = json::array({user_msg, assistant_two_args}); }); @@ -1144,7 +1146,7 @@ static void test_seed_oss_args_presence(testing & t) { params_same.enable_thinking = true; // Test same arg vs other arg - auto result_same_other = differential_analyzer::compare_variants(tmpl, params_same, + auto result_same_other = autoparser::compare_variants(tmpl, params_same, [&](template_params & p) { p.messages = json::array({user_msg, assistant_other_arg}); }); @@ -1163,7 +1165,7 @@ static void test_seed_oss_args_presence(testing & t) { diff5a.right.find("value2") != std::string::npos || diff5a.prefix.find("value2") != std::string::npos || diff5a.suffix.find("value2") != std::string::npos); // Test same arg vs both args - auto result_same_both = differential_analyzer::compare_variants(tmpl, params_same, + auto result_same_both = autoparser::compare_variants(tmpl, params_same, [&](template_params & p) { p.messages = json::array({user_msg, assistant_both_args}); }); @@ -1212,7 +1214,7 @@ static void test_seed_oss_tool_with_reasoning(testing & t) { params_tool_only.add_generation_prompt = false; params_tool_only.enable_thinking = true; - auto result = differential_analyzer::compare_variants(tmpl, params_tool_only, + auto result = autoparser::compare_variants(tmpl, params_tool_only, [&](template_params & p) { p.messages = json::array({user_msg, assistant_tool_with_reasoning}); }); @@ -1285,7 +1287,7 @@ static void test_nemotron_reasoning_detection(testing & t) { params.enable_thinking = true; // Run differential analysis - auto analysis = differential_analyzer::analyze(tmpl); + auto analysis = autoparser::analyze_template(tmpl); // Check reasoning markers t.assert_equal("reasoning_start should be ''", "", analysis.reasoning.start); @@ -1306,7 +1308,7 @@ static void test_nemotron_tool_format(testing & t) { common_chat_template tmpl = load_nemotron_template(t); // Run differential analysis - auto analysis = differential_analyzer::analyze(tmpl); + auto analysis = autoparser::analyze_template(tmpl); // Check tool markers - Nemotron uses per-call wrapping (each call individually wrapped) t.assert_equal("tool_section_start should be empty (per-call format)", "", analysis.tools.format.section_start); @@ -1344,7 +1346,7 @@ static void test_cohere_reasoning_detection(testing & t) { common_chat_template tmpl = load_cohere_template(t); // Run differential analysis - auto analysis = differential_analyzer::analyze(tmpl); + auto analysis = autoparser::analyze_template(tmpl); // Check reasoning markers - Cohere uses special token format t.assert_equal("reasoning_start should be '<|START_THINKING|>'", "<|START_THINKING|>", analysis.reasoning.start); @@ -1365,7 +1367,7 @@ static void test_tool_format_cohere(testing & t) { common_chat_template tmpl = load_cohere_template(t); // Run differential analysis - auto analysis = differential_analyzer::analyze(tmpl); + auto analysis = autoparser::analyze_template(tmpl); // Check tool section markers - Cohere uses ACTION markers t.assert_equal("tool_section_start should be '<|START_ACTION|>'", "<|START_ACTION|>", analysis.tools.format.section_start); @@ -1772,12 +1774,12 @@ static void test_tagged_args_with_embedded_quotes(testing & t) { auto tool_choice = p.choice(); for (const auto & tool_def : tools) { - if (!tool_def.contains("function")) continue; + if (!tool_def.contains("function")) { continue; } const auto & function = tool_def.at("function"); std::string name = function.at("name"); const auto & params = function.at("parameters"); - if (!params.contains("properties") || !params.at("properties").is_object()) continue; + if (!params.contains("properties") || !params.at("properties").is_object()) { continue; } const auto & properties = params.at("properties"); diff --git a/tools/parser/debug-template-parser.cpp b/tools/parser/debug-template-parser.cpp index c87f3c8e35..17cbdfea18 100644 --- a/tools/parser/debug-template-parser.cpp +++ b/tools/parser/debug-template-parser.cpp @@ -279,7 +279,7 @@ static void render_scenario(const common_chat_template & tmpl, LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str()); try { - templates_params inputs; + autoparser::templates_params inputs; inputs.messages = final_messages; inputs.add_generation_prompt = add_generation_prompt; inputs.extra_context["enable_thinking"] = enable_thinking; @@ -395,10 +395,10 @@ int main(int argc, char ** argv) { LOG_ERR(" TEMPLATE ANALYSIS\n"); LOG_ERR("================================================================================\n"); - diff_analysis_result analysis = differential_analyzer::analyze(chat_template); + autoparser::analyze_template analysis(chat_template); // Generate Parser - templates_params params; + autoparser::templates_params params; params.messages = json::array(); params.reasoning_format = opts.enable_reasoning ? COMMON_REASONING_FORMAT_DEEPSEEK : COMMON_REASONING_FORMAT_NONE; @@ -414,7 +414,7 @@ int main(int argc, char ** argv) { } params.parallel_tool_calls = false; - auto parser_data = universal_peg_generator::generate_parser(chat_template, params, analysis); + auto parser_data = autoparser::universal_peg_generator::generate_parser(chat_template, params, analysis); LOG_ERR("\n=== Differential Analysis Results ===\n"); diff --git a/tools/parser/template-analysis.cpp b/tools/parser/template-analysis.cpp index deb2bafa20..a92e104ac0 100644 --- a/tools/parser/template-analysis.cpp +++ b/tools/parser/template-analysis.cpp @@ -400,12 +400,12 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_no_tools; + autoparser::templates_params params_no_tools; params_no_tools.messages = json::array({ user_msg }); params_no_tools.add_generation_prompt = false; params_no_tools.tools = json::array(); - templates_params params_with_tools = params_no_tools; + autoparser::templates_params params_with_tools = params_no_tools; params_with_tools.tools = tools; std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools); @@ -419,12 +419,12 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_no_prompt; + autoparser::templates_params params_no_prompt; params_no_prompt.messages = json::array({ user_msg }); params_no_prompt.add_generation_prompt = false; params_no_prompt.tools = json::array(); - templates_params params_with_prompt = params_no_prompt; + autoparser::templates_params params_with_prompt = params_no_prompt; params_with_prompt.add_generation_prompt = true; std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt); @@ -438,12 +438,12 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_no_reasoning; + autoparser::templates_params params_no_reasoning; params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() }); params_no_reasoning.add_generation_prompt = false; params_no_reasoning.enable_thinking = true; - templates_params params_with_reasoning = params_no_reasoning; + autoparser::templates_params params_with_reasoning = params_no_reasoning; params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() }); std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning); @@ -458,12 +458,12 @@ static void analyze_template(const std::string & template_path) { json user_msg = make_user_msg(); json user_msg2 = make_user_msg2(); - templates_params params_no_reasoning; + autoparser::templates_params params_no_reasoning; params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 }); params_no_reasoning.add_generation_prompt = false; params_no_reasoning.enable_thinking = true; - templates_params params_with_reasoning = params_no_reasoning; + autoparser::templates_params params_with_reasoning = params_no_reasoning; params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 }); std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning); @@ -477,12 +477,12 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_no_tool; + autoparser::templates_params params_no_tool; params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() }); params_no_tool.add_generation_prompt = false; params_no_tool.tools = tools; - templates_params params_with_tool = params_no_tool; + autoparser::templates_params params_with_tool = params_no_tool; params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() }); std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool); @@ -497,12 +497,12 @@ static void analyze_template(const std::string & template_path) { json user_msg = make_user_msg(); json user_msg2 = make_user_msg2_continue(); - templates_params params_no_tool; + autoparser::templates_params params_no_tool; params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 }); params_no_tool.add_generation_prompt = false; params_no_tool.tools = tools; - templates_params params_with_tool = params_no_tool; + autoparser::templates_params params_with_tool = params_no_tool; params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 }); std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool); @@ -516,12 +516,12 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_one_tool; + autoparser::templates_params params_one_tool; params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() }); params_one_tool.add_generation_prompt = false; params_one_tool.tools = tools; - templates_params params_two_tools = params_one_tool; + autoparser::templates_params params_two_tools = params_one_tool; params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() }); std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool); @@ -536,12 +536,12 @@ static void analyze_template(const std::string & template_path) { json user_msg = make_user_msg(); json user_msg2 = make_user_msg2_continue(); - templates_params params_one_tool; + autoparser::templates_params params_one_tool; params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 }); params_one_tool.add_generation_prompt = false; params_one_tool.tools = tools; - templates_params params_two_tools = params_one_tool; + autoparser::templates_params params_two_tools = params_one_tool; params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 }); std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool); @@ -555,13 +555,13 @@ static void analyze_template(const std::string & template_path) { { json user_msg = make_user_msg(); - templates_params params_no_reasoning; + autoparser::templates_params params_no_reasoning; params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() }); params_no_reasoning.add_generation_prompt = false; params_no_reasoning.tools = tools; params_no_reasoning.enable_thinking = true; - templates_params params_with_reasoning = params_no_reasoning; + autoparser::templates_params params_with_reasoning = params_no_reasoning; params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() }); std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);