") -> ""
+// until_common_prefix("some text", "1234", "abcd") -> ""
+// until_common_prefix("one arg two args three args four", "argument alpha", "argument beta") -> "one ""
+std::string until_common_prefix(const std::string & full, const std::string & left, const std::string & right);
+
+// Returns the suffix of `full` after the last occurrence of the common suffix of `left` and `right`
+// Returns empty string if there's no common suffix
+// Mirror function of `until_common_prefix`
+// * eg.:
+// after_common_suffix("really want a FUNCTION call", "first FUNCTION", "second FUNCTION") -> " call"
+// after_common_suffix("one arg two-args three args four", "alpha-args", "beta-args") -> " three args four"
+std::string after_common_suffix(const std::string & full, const std::string & left, const std::string & right);
+
+// Segmentize text into markers and non-marker fragments
+// * eg.:
+// segmentize_markers("The site titleHere's some content
" ->
+// [ (MARKER, ""), (MARKER, ""), (MARKER, ""), (TEXT, "The site title"), (MARKER, ""),
+// (MARKER, ""), (MARKER, ""), (TEXT, "Here's some "), (MARKER, ""), (TEXT, "content"), (MARKER, ""),
+// (MARKER, "
"), (MARKER, ""), (MARKER, "")
+// ]
+// segmentize_markers("<|tool_call|>[args]{ are here }[/args]<|tool_call_end|>") ->
+// [ (MARKER, "<|tool_call|>"), (MARKER, "[args]"), (TEXT, "{ are here }"), (MARKER, "[/args]"), (MARKER, "<|tool_call_end|>") ]
+std::vector segmentize_markers(const std::string & text);
+
+// Prune whitespace-only segments from a vector of segments
+// * eg.:
+// segmentize_markers("\n\n\n \n\n\n") ->
+// X = [ (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n \n"),
+// (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n"), (MARKER, "") ]
+// prune_whitespace_segments(X) -> [ (MARKER, ""), (MARKER, ""), (MARKER, ""), (MARKER, ""),
+// (MARKER, ""), (MARKER, "") ]
+std::vector prune_whitespace_segments(const std::vector & segments);
+
+namespace autoparser {
+
+// Apply a template with the given parameters, returning the rendered string (empty on failure)
+std::string apply_template(const common_chat_template & tmpl, const template_params & params);
+
+// Factorized differential comparison function
+// Takes base params and a single modifier lambda to create variant B
+// Returns compare_variants_result containing diff and both outputs, or std::nullopt on failure
+std::optional compare_variants(
+ const common_chat_template & tmpl,
+ const template_params & params_A,
+ const std::function & params_modifier);
+
+} // namespace autoparser
diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h
new file mode 100644
index 0000000000..99dd9f063c
--- /dev/null
+++ b/common/chat-auto-parser.h
@@ -0,0 +1,434 @@
+#pragma once
+
+#include "chat.h"
+#include "common.h"
+#include "jinja/caps.h"
+#include "peg-parser.h"
+#include "nlohmann/json.hpp"
+
+#include
+#include
+#include
+#include
+#include
+
+using json = nlohmann::ordered_json;
+
+class common_chat_peg_builder;
+
+// ============================================================================
+// Parameters for template application (low-level, used by diff analysis)
+// ============================================================================
+struct template_params {
+ json messages;
+ json tools;
+ bool add_generation_prompt = false;
+ bool enable_thinking = true;
+ std::optional extra_context = std::nullopt;
+};
+
+struct diff_split {
+ std::string prefix;
+ std::string suffix;
+ std::string left;
+ std::string right;
+
+ bool operator==(struct diff_split & other) const {
+ return prefix == other.prefix && suffix == other.suffix && left == other.left && right == other.right;
+ }
+};
+
+// Result of compare_variants containing diff and original outputs
+struct compare_variants_result {
+ diff_split diff;
+ std::string output_A;
+ std::string output_B;
+};
+
+namespace autoparser {
+
+// ============================================================================
+// High-level params for parser generation
+// ============================================================================
+
+struct generation_params {
+ json messages;
+ json tools;
+ common_chat_tool_choice tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;
+ json json_schema;
+ bool parallel_tool_calls = true;
+ common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_AUTO;
+ bool stream = true;
+ std::string grammar;
+ bool add_generation_prompt = false;
+ bool enable_thinking = true;
+ std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
+ std::string generation_prompt;
+ json extra_context;
+ bool add_bos = false;
+ bool add_eos = false;
+ bool is_inference = true;
+ bool add_inference = false;
+ bool mark_input = true; // whether to mark input strings in the jinja context
+};
+
+// ============================================================================
+// Analysis Result Enums
+// ============================================================================
+
+// Reasoning handling mode (derived from R1-R3 comparisons)
+enum class reasoning_mode {
+ NONE, // No reasoning markers detected
+ TAG_BASED, // Tag-based: ... (start can be empty for delimiter-style)
+ TOOLS_ONLY // Only reason on tool calls, not on normal content
+};
+
+inline std::ostream & operator<<(std::ostream & os, const reasoning_mode & mode) {
+ switch (mode) {
+ case reasoning_mode::NONE:
+ return os << "NONE";
+ case reasoning_mode::TAG_BASED:
+ return os << "TAG_BASED";
+ case reasoning_mode::TOOLS_ONLY:
+ return os << "TOOLS_ONLY";
+ default:
+ return os << "UNKNOWN";
+ }
+}
+
+// Content wrapping mode (derived from C1 comparison)
+enum class content_mode {
+ PLAIN, // No content markers
+ ALWAYS_WRAPPED, // Content always wrapped with markers
+ WRAPPED_WITH_REASONING, // Content wrapped only when reasoning present
+};
+
+inline std::ostream & operator<<(std::ostream & os, const content_mode & mode) {
+ switch (mode) {
+ case content_mode::PLAIN:
+ return os << "PLAIN";
+ case content_mode::ALWAYS_WRAPPED:
+ return os << "ALWAYS_WRAPPED";
+ case content_mode::WRAPPED_WITH_REASONING:
+ return os << "WRAPPED_WITH_REASONING";
+ default:
+ return os << "UNKNOWN";
+ }
+}
+
+// Call ID position in tool calls (for non-JSON formats)
+enum class call_id_position {
+ NONE, // No call ID support detected
+ PRE_FUNC_NAME, // Call ID before function name: [CALL_ID]id[FUNC]name{args}
+ BETWEEN_FUNC_AND_ARGS, // Call ID between function and args: [FUNC]name[CALL_ID]id{args}
+ POST_ARGS, // Call ID after arguments: [FUNC]name{args}[CALL_ID]id
+};
+
+inline std::ostream & operator<<(std::ostream & os, const call_id_position & pos) {
+ switch (pos) {
+ case call_id_position::NONE:
+ return os << "NONE";
+ case call_id_position::PRE_FUNC_NAME:
+ return os << "PRE_FUNC_NAME";
+ case call_id_position::BETWEEN_FUNC_AND_ARGS:
+ return os << "BETWEEN_FUNC_AND_ARGS";
+ case call_id_position::POST_ARGS:
+ return os << "POST_ARGS";
+ default:
+ return os << "UNKNOWN";
+ }
+}
+
+// Tool call format classification (derived from T1-T5, A1-A3 comparisons)
+enum class tool_format {
+ NONE, // No tool support detected
+ JSON_NATIVE, // Pure JSON: {"name": "X", "arguments": {...}}
+ TAG_WITH_JSON, // Tag-based with JSON args: {...}
+ TAG_WITH_TAGGED, // Tag-based with tagged args: value
+};
+
+inline std::ostream & operator<<(std::ostream & os, const tool_format & format) {
+ switch (format) {
+ case tool_format::NONE:
+ return os << "NONE";
+ case tool_format::JSON_NATIVE:
+ return os << "JSON_NATIVE";
+ case tool_format::TAG_WITH_JSON:
+ return os << "TAG_WITH_JSON";
+ case tool_format::TAG_WITH_TAGGED:
+ return os << "TAG_WITH_TAGGED";
+ default:
+ return os << "UNKNOWN";
+ }
+}
+
+// ============================================================================
+// Sub-structs for tool analysis
+// ============================================================================
+
+struct tool_format_analysis {
+ tool_format mode = tool_format::NONE;
+
+ std::string section_start; // e.g., "", "[TOOL_CALLS]", ""
+ std::string section_end; // e.g., "", ""
+ std::string per_call_start; // e.g., "<|tool_call_begin|>", "" (for multi-call templates)
+ std::string per_call_end; // e.g., "<|tool_call_end|>", ""
+
+ bool fun_name_is_key = false; // In JSON format function name is JSON key, i.e. { "": { ... arguments ... } }
+ bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...]
+
+ std::string function_field = "function";
+ std::string name_field = "name";
+ std::string args_field = "arguments";
+ std::string id_field;
+ std::string gen_id_field;
+ std::vector parameter_order;
+};
+
+struct tool_function_analysis {
+ std::string name_prefix; // e.g., "", "\"", ":0"
+ std::string close; // e.g., "", "" (for tag-based)
+};
+
+struct tool_arguments_analysis {
+ std::string start; // e.g., "<|tool_call_argument_begin|>", ""
+ std::string end; // e.g., "<|tool_call_argument_end|>", ""
+ std::string name_prefix; // e.g., "", "\""
+ std::string name_suffix; // e.g., ">", "