#include "chat-auto-parser-helpers.h"
#include "chat-auto-parser.h"
#include "chat.h"
#include "log.h"
#include "nlohmann/json.hpp"

using json = nlohmann::ordered_json;

template_analysis_result template_analyzer::analyze_template(const common_chat_template & tmpl) {
    LOG_DBG("=== STARTING UNIFIED TEMPLATE ANALYSIS ===\n");

    template_analysis_result result;

    // Phase 1: Analyze content and reasoning structure (no tools involved)
    result.content = analyze_content_structure(tmpl);

    // Phase 2: Analyze tool call structure (layered on Phase 1)
    result.tools = analyze_tool_structure(tmpl, result.content);

    // Post-processing: Extract reasoning markers from tool_section_start if Phase 1 didn't detect them
    // Some templates (like Command-R7B) include reasoning markers in tool outputs but not in prompts
    if (result.content.reasoning_start.empty() && !result.tools.tool_section_start.empty()) {
        // Known reasoning end marker patterns that might be embedded in tool_section_start
        std::vector<std::pair<std::string, std::string>> reasoning_patterns = {
            { "<|START_THINKING|>", "<|END_THINKING|>" },
            { "<|START_THOUGHT|>",  "<|END_THOUGHT|>"  },
            { "<|START_REASON|>",   "<|END_REASON|>"   },
            { "<think>",            "</think>"         },
            { "<thinking>",         "</thinking>"      },
        };

        for (const auto & [start_marker, end_marker] : reasoning_patterns) {
            size_t end_pos = result.tools.tool_section_start.find(end_marker);
            if (end_pos != std::string::npos) {
                // Found reasoning end marker in tool_section_start
                // Extract it and clean up tool_section_start
                result.content.reasoning_start = start_marker;
                result.content.reasoning_end   = end_marker;
                result.content.reasoning_mode  = content_structure::REASONING_OPTIONAL;

                // Clean up tool_section_start: remove everything before and including the end marker
                size_t after_end = end_pos + end_marker.length();
                if (after_end < result.tools.tool_section_start.length()) {
                    result.tools.tool_section_start = result.tools.tool_section_start.substr(after_end);
                    // Trim leading whitespace
                    size_t first_non_ws             = result.tools.tool_section_start.find_first_not_of(" \t\n\r");
                    if (first_non_ws != std::string::npos && first_non_ws > 0) {
                        result.tools.tool_section_start = result.tools.tool_section_start.substr(first_non_ws);
                    }
                }

                LOG_DBG("Post-processing: Extracted reasoning markers from tool_section_start\n");
                LOG_DBG("  reasoning_start: '%s', reasoning_end: '%s'\n", result.content.reasoning_start.c_str(),
                        result.content.reasoning_end.c_str());
                LOG_DBG("  cleaned tool_section_start: '%s'\n", result.tools.tool_section_start.c_str());
                break;
            }
        }
    }

    // Post-processing: Detect content markers for recipient-based format
    // For recipient-based format, content is prefixed with tool_call_start_marker + recipient_name + \n
    // (e.g., ">>>all\n"). We need to detect and extract this as the content_start marker.
    if (result.tools.function_format == tool_call_structure::FUNC_RECIPIENT_BASED &&
        result.content.content_start.empty() && !result.tools.tool_section_start.empty()) {
        // Render template with content only (no tools) to detect the content marker
        templates_params inputs;
        inputs.messages = {
            { { "role", "user" },      { "content", "Hello" }               },
            { { "role", "assistant" }, { "content", "ACTUAL_CONTENT_HERE" } }
        };
        inputs.add_generation_prompt = true;

        std::string output;
        try {
            output = common_chat_template_direct_apply(tmpl, inputs);
        } catch (...) {
            output = "";
        }

        if (!output.empty()) {
            // Find where the actual content starts
            size_t content_pos = output.find("ACTUAL_CONTENT_HERE");

            if (content_pos != std::string::npos) {
                // For recipient-based format, find the last occurrence of tool_call_start_marker
                // before the content. The marker is from that position to the content (including the newline).
                size_t marker_pos = output.rfind(result.tools.tool_section_start, content_pos);

                if (marker_pos != std::string::npos && marker_pos < content_pos) {
                    // Find the newline after the marker
                    size_t newline_pos = output.find('\n', marker_pos);

                    if (newline_pos != std::string::npos && newline_pos < content_pos) {
                        // Extract everything up to and including the newline after the marker
                        std::string detected_marker = output.substr(marker_pos, newline_pos - marker_pos + 1);

                        // Verify the marker starts with tool_call_start_marker
                        if (detected_marker.find(result.tools.tool_section_start) == 0) {
                            result.content.content_start = detected_marker;
                            result.content.content_mode  = content_structure::CONTENT_ALWAYS_WRAPPED;
                            LOG_DBG("Post-processing: Detected recipient-based content marker: '%s'\n",
                                    result.content.content_start.c_str());
                        }
                    }
                }
            }
        }
    }

    // Collect preserved tokens from both phases
    collect_preserved_tokens(result);

    LOG_DBG("=== UNIFIED TEMPLATE ANALYSIS COMPLETE ===\n");
    LOG_DBG("Content structure:\n");
    LOG_DBG("  reasoning_mode: %d\n", static_cast<int>(result.content.reasoning_mode));
    LOG_DBG("  reasoning_start: '%s'\n", result.content.reasoning_start.c_str());
    LOG_DBG("  reasoning_end: '%s'\n", result.content.reasoning_end.c_str());
    LOG_DBG("  content_mode: %d\n", static_cast<int>(result.content.content_mode));
    LOG_DBG("  content_start: '%s'\n", result.content.content_start.c_str());
    LOG_DBG("  content_end: '%s'\n", result.content.content_end.c_str());
    LOG_DBG("Tool structure:\n");
    LOG_DBG("  supports_tools: %s\n", result.tools.supports_tools ? "true" : "false");
    LOG_DBG("  function_format: %d\n", static_cast<int>(result.tools.function_format));
    LOG_DBG("  argument_format: %d\n", static_cast<int>(result.tools.argument_format));
    LOG_DBG("  tool_section_start: '%s'\n", result.tools.tool_section_start.c_str());
    LOG_DBG("  tool_section_end: '%s'\n", result.tools.tool_section_end.c_str());

    return result;
}

content_structure template_analyzer::analyze_content_structure(const common_chat_template & tmpl) {
    LOG_DBG("=== PHASE 1: ANALYZING CONTENT STRUCTURE ===\n");

    content_structure cs;

    // Step 1: Detect reasoning markers by toggling enable_thinking
    detect_reasoning_markers(tmpl, cs);

    // Step 2: Detect content wrapping markers
    detect_content_markers(tmpl, cs);

    // Step 3: Determine reasoning mode (NONE, OPTIONAL, FORCED_OPEN)
    templates_params inputs;
    inputs.messages = {
        { { "role", "user" }, { "content", "Hello" } }
    };
    inputs.add_generation_prompt            = true;
    inputs.enable_thinking                  = true;

    std::string prompt;
    try {
        prompt = common_chat_template_direct_apply(tmpl, inputs);
    } catch (...) {
        LOG_DBG("Failed to render template for reasoning mode detection\n");
        return cs;
    }

    cs.reasoning_mode = detect_reasoning_mode(cs, prompt);

    LOG_DBG("Phase 1 complete: reasoning_mode=%d, content_mode=%d\n", static_cast<int>(cs.reasoning_mode),
            static_cast<int>(cs.content_mode));

    return cs;
}

void template_analyzer::detect_reasoning_markers(const common_chat_template & tmpl, content_structure & cs) {
    LOG_DBG("=== DETECTING REASONING MARKERS ===\n");

    // Method 1: Compare outputs with reasoning_content field present vs absent
    json reasoning_msg = {
        { "role",              "assistant"      },
        { "content",           "CONTENT_MARKER" },
        { "reasoning_content", "THOUGHT_MARKER" }
    };

    json base_msg = {
        { "role",    "assistant"      },
        { "content", "CONTENT_MARKER" }
    };

    templates_params inputs;

    inputs.messages = { reasoning_msg };
    std::string reasoning_output;
    try {
        reasoning_output = common_chat_template_direct_apply(tmpl, inputs);
    } catch (...) {
        LOG_DBG("Failed to render template with reasoning_content\n");
        reasoning_output = "";
    }

    inputs.messages = { base_msg };
    std::string base_output;
    try {
        base_output = common_chat_template_direct_apply(tmpl, inputs);
    } catch (...) {
        LOG_DBG("Failed to render base template\n");
        base_output = "";
    }

    // If outputs differ and we can find THOUGHT_MARKER, extract the reasoning markers
    if (!reasoning_output.empty() && reasoning_output != base_output) {
        size_t thought_pos = reasoning_output.find("THOUGHT_MARKER");
        size_t content_pos = reasoning_output.find("CONTENT_MARKER");

        if (thought_pos != std::string::npos && content_pos != std::string::npos && content_pos > thought_pos) {
            // Extract what's between THOUGHT_MARKER and CONTENT_MARKER as the end marker
            size_t thought_end = thought_pos + strlen("THOUGHT_MARKER");
            cs.reasoning_end   = reasoning_output.substr(thought_end, content_pos - thought_end);

            // Find what's before THOUGHT_MARKER by comparing with base_output
            size_t diff_start = 0;
            while (diff_start < base_output.length() && diff_start < reasoning_output.length() &&
                   base_output[diff_start] == reasoning_output[diff_start]) {
                diff_start++;
            }

            // If diff_start is in the middle of a tag (previous char is '<'), back up to include it
            // This handles cases like base="</think>" vs reasoning="<think>" where both share '<'
            if (diff_start > 0 && diff_start < reasoning_output.length() &&
                reasoning_output[diff_start - 1] == '<') {
                diff_start--;
            }

            if (diff_start < thought_pos) {
                cs.reasoning_start = reasoning_output.substr(diff_start, thought_pos - diff_start);
            }

            trim_whitespace(cs.reasoning_start);
            trim_whitespace(cs.reasoning_end);

            // If we found reasoning_end but not reasoning_start, try to derive it from reasoning_end
            // For example: </think> -> <think>, </|END_THINKING|> -> <|START_THINKING|>
            if (cs.reasoning_start.empty() && !cs.reasoning_end.empty()) {
                // First, try to derive directly from the closing tag format
                if (cs.reasoning_end.length() > 3 && cs.reasoning_end[0] == '<' && cs.reasoning_end[1] == '/') {
                    // Standard XML closing tag like </think> -> <think>
                    size_t tag_end_pos = cs.reasoning_end.find('>');
                    if (tag_end_pos != std::string::npos) {
                        std::string tag_name = cs.reasoning_end.substr(2, tag_end_pos - 2);
                        cs.reasoning_start   = "<" + tag_name + ">";
                        LOG_DBG("Method 1: Derived reasoning_start from closing tag format\n");
                        LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
                    }
                } else if (cs.reasoning_end.find("<|END_") == 0 || cs.reasoning_end.find("<|/") == 0) {
                    // Special token format like <|END_THINKING|> -> <|START_THINKING|>
                    // or <|/think|> -> <|think|>
                    if (cs.reasoning_end.find("<|END_") == 0) {
                        std::string core   = cs.reasoning_end.substr(6);  // Remove "<|END_"
                        cs.reasoning_start = "<|START_" + core;
                    } else {
                        std::string core   = cs.reasoning_end.substr(3);  // Remove "<|/"
                        cs.reasoning_start = "<|" + core;
                    }
                    LOG_DBG("Method 1: Derived reasoning_start from special token format\n");
                    LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
                }
            }

            if (!cs.reasoning_start.empty()) {
                LOG_DBG("Method 1: Found reasoning markers via reasoning_content field\n");
                LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
            }
        }
    }

    // Method 2: Compare prompts with enable_thinking true vs false
    if (cs.reasoning_start.empty()) {
        LOG_DBG("Method 1 failed, trying Method 2 (enable_thinking toggle)\n");

        json user_msg = {
            { "role",    "user"  },
            { "content", "Hello" }
        };

        templates_params inputs_prompt;
        inputs_prompt.messages              = { user_msg };
        inputs_prompt.add_generation_prompt = true;
        inputs_prompt.enable_thinking = false;
        std::string prompt_no_think;
        try {
            prompt_no_think = common_chat_template_direct_apply(tmpl, inputs_prompt);
        } catch (...) {
            prompt_no_think = "";
        }

        inputs_prompt.enable_thinking = true;
        std::string prompt_think;
        try {
            prompt_think = common_chat_template_direct_apply(tmpl, inputs_prompt);
        } catch (...) {
            prompt_think = "";
        }

        if (!prompt_think.empty() && prompt_think != prompt_no_think) {
            // Find the difference - this should be the reasoning start marker
            size_t diff_pos = 0;
            while (diff_pos < prompt_no_think.length() && diff_pos < prompt_think.length() &&
                   prompt_no_think[diff_pos] == prompt_think[diff_pos]) {
                diff_pos++;
            }

            // Check which direction has extra content
            if (prompt_think.length() > prompt_no_think.length()) {
                // Normal case: enable_thinking=true adds content (e.g., <think> at the end)
                std::string diff = prompt_think.substr(diff_pos);

                // Only use if it looks like a tag
                if (diff.find('<') != std::string::npos || diff.find('[') != std::string::npos) {
                    cs.reasoning_start = diff;
                    cs.reasoning_end   = create_closing_tag(diff);
                    trim_whitespace(cs.reasoning_start);
                    trim_whitespace(cs.reasoning_end);

                    LOG_DBG("Method 2: Found reasoning markers via enable_thinking toggle\n");
                    LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
                }
            } else {
                // Reverse case: enable_thinking=false adds content (e.g., GLM-4.6 adds <think></think>)
                // This means the template adds an empty thinking block when thinking is disabled
                std::string diff = prompt_no_think.substr(diff_pos);

                // Look for adjacent opening and closing tags like <think></think>
                size_t open_start = diff.find('<');
                if (open_start != std::string::npos) {
                    size_t open_end = diff.find('>', open_start);
                    if (open_end != std::string::npos) {
                        std::string opening_tag = diff.substr(open_start, open_end - open_start + 1);
                        // Skip if it looks like a role marker
                        if (opening_tag.find("assistant") == std::string::npos &&
                            opening_tag.find("user") == std::string::npos &&
                            opening_tag.find("system") == std::string::npos) {
                            std::string expected_close = create_closing_tag(opening_tag);
                            // Check if the closing tag follows immediately (empty thinking block)
                            size_t      close_pos      = diff.find(expected_close, open_end + 1);
                            if (close_pos != std::string::npos) {
                                // Verify only whitespace between tags
                                std::string between = diff.substr(open_end + 1, close_pos - open_end - 1);
                                bool        only_ws = true;
                                for (char c : between) {
                                    if (!std::isspace(static_cast<unsigned char>(c))) {
                                        only_ws = false;
                                        break;
                                    }
                                }
                                if (only_ws) {
                                    cs.reasoning_start = opening_tag;
                                    cs.reasoning_end   = expected_close;
                                    trim_whitespace(cs.reasoning_start);
                                    trim_whitespace(cs.reasoning_end);

                                    LOG_DBG("Method 2: Found reasoning markers via enable_thinking toggle (reverse)\n");
                                    LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(),
                                            cs.reasoning_end.c_str());
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    // Method 3: Check if the prompt ends with an unclosed reasoning tag
    if (cs.reasoning_start.empty()) {
        LOG_DBG("Method 2 failed, trying Method 3 (prompt ending with open tag)\n");

        json user_msg = {
            { "role",    "user"  },
            { "content", "Hello" }
        };

        templates_params inputs_prompt;
        inputs_prompt.messages                         = { user_msg };
        inputs_prompt.add_generation_prompt            = true;
        inputs_prompt.enable_thinking                  = true;

        std::string prompt;
        try {
            prompt = common_chat_template_direct_apply(tmpl, inputs_prompt);
        } catch (...) {
            prompt = "";
        }

        if (!prompt.empty()) {
            // Save trailing whitespace before trimming
            std::string trailing_ws;
            size_t      end_pos = prompt.length();
            while (end_pos > 0 && (prompt[end_pos - 1] == '\n' || prompt[end_pos - 1] == '\r')) {
                trailing_ws = prompt[end_pos - 1] + trailing_ws;
                end_pos--;
            }

            trim_trailing_newlines(prompt);

            // Find the last tag in the prompt
            size_t last_open_angle  = prompt.rfind('<');
            size_t last_close_angle = prompt.rfind('>');

            // Check for closed tags at the end
            if (last_open_angle != std::string::npos && last_close_angle != std::string::npos &&
                last_close_angle == prompt.length() - 1 && last_close_angle > last_open_angle) {
                std::string tag = prompt.substr(last_open_angle);

                // Check if this looks like a reasoning tag (not a role marker)
                std::vector<std::string> blacklisted_tags = {
                    "<|CHATBOT_TOKEN|>", "<|SYSTEM_TOKEN|>",  "<|USER_TOKEN|>",  "<|ASSISTANT_TOKEN|>", "<|im_start|>",
                    "<|im_end|>",        "<|start_of_role|>", "<|end_of_role|>", "<|end_of_text|>",     "<|end|>",
                    "<|assistant|>",     "<|user|>",          "<|system|>",      "<assistant>",         "<user>",
                    "<system>"
                };

                bool is_blacklisted = false;
                for (const auto & blacklisted : blacklisted_tags) {
                    if (tag == blacklisted) {
                        is_blacklisted = true;
                        break;
                    }
                }

                // Check if it looks like a thinking/reasoning tag
                std::string lower_tag = tag;
                std::transform(lower_tag.begin(), lower_tag.end(), lower_tag.begin(), ::tolower);
                bool looks_like_reasoning = lower_tag.find("think") != std::string::npos ||
                                            lower_tag.find("reason") != std::string::npos ||
                                            lower_tag.find("thought") != std::string::npos;

                if (!is_blacklisted && looks_like_reasoning) {
                    // Check if the detected tag is a close tag (starts with </)
                    // This handles templates like DeepSeek-V3.1 that end with </think> when thinking is disabled
                    bool is_close_tag = (tag.size() > 2 && tag[0] == '<' && tag[1] == '/');

                    if (is_close_tag) {
                        // The tag is a close tag (e.g., </think>)
                        // Derive the open tag by removing the '/'
                        std::string tag_name = extract_tag_name(tag);  // Returns "/think" for </think>
                        if (!tag_name.empty() && tag_name[0] == '/') {
                            tag_name = tag_name.substr(1);             // Remove leading '/'
                        }
                        cs.reasoning_start = "<" + tag_name + ">";
                        cs.reasoning_end   = tag;
                        trim_whitespace(cs.reasoning_start);
                        trim_whitespace(cs.reasoning_end);

                        LOG_DBG("Method 3: Found reasoning markers via prompt ending with CLOSE tag\n");
                        LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());

                        // Note: The prompt ends with the close tag, meaning thinking is disabled.
                        // The reasoning_mode will be set in detect_reasoning_mode() which will
                        // correctly identify this as NOT forced open since the prompt ends with
                        // the end marker, not the start marker.
                    } else {
                        // Standard case: open tag at the end (e.g., <think>)
                        cs.reasoning_start = tag + trailing_ws;
                        cs.reasoning_end   = create_closing_tag(tag) + trailing_ws;
                        trim_whitespace(cs.reasoning_start);
                        trim_whitespace(cs.reasoning_end);

                        LOG_DBG("Method 3: Found reasoning markers via prompt ending with tag\n");
                        LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
                    }
                }
            }
        }
    }

    // Method 4: Look for adjacent opening/closing tag pairs with common content in prompt
    // This detects patterns like <think></think>, <|START_THINKING|><|END_THINKING|>, [think][/think]
    if (cs.reasoning_start.empty()) {
        LOG_DBG("Method 3 failed, trying Method 4 (adjacent tag pairs with common content)\n");

        json user_msg = {
            { "role",    "user"  },
            { "content", "Hello" }
        };

        templates_params inputs_prompt;
        inputs_prompt.messages                         = { user_msg };
        inputs_prompt.add_generation_prompt            = true;
        // Try with thinking disabled - templates may output empty thinking blocks
        inputs_prompt.enable_thinking                  = false;

        std::string prompt;
        try {
            prompt = common_chat_template_direct_apply(tmpl, inputs_prompt);
        } catch (...) {
            prompt = "";
        }

        if (!prompt.empty()) {
            // Look for patterns like <tag1><tag2> or <tag1>...<tag2> where tag1 and tag2 share a common word
            // Common patterns:
            //   <think></think>
            //   <|START_THINKING|><|END_THINKING|>
            //   [think][/think]

            // Find potential tag pairs by looking for closing tags that immediately follow opening tags
            // Pattern: opening tag followed by closing tag with same keyword
            std::vector<std::tuple<std::string, std::string, std::string>> tag_patterns = {
                // (opening pattern, closing pattern, keyword to match)
                { "<|START_",   "<|END_",      "THINKING" },
                { "<|START_",   "<|END_",      "THOUGHT"  },
                { "<|START_",   "<|END_",      "REASON"   },
                { "<think>",    "</think>",    ""         },
                { "<Think>",    "</Think>",    ""         },
                { "<THINK>",    "</THINK>",    ""         },
                { "[think]",    "[/think]",    ""         },
                { "[THINK]",    "[/THINK]",    ""         },
                { "<thinking>", "</thinking>", ""         },
                { "<THINKING>", "</THINKING>", ""         },
                { "<|think|>",  "<|/think|>",  ""         },
            };

            for (const auto & [open_prefix, close_prefix, keyword] : tag_patterns) {
                size_t open_pos = prompt.find(open_prefix);
                if (open_pos == std::string::npos) {
                    continue;
                }

                std::string start_tag;
                std::string end_tag;

                if (!keyword.empty()) {
                    // Pattern like <|START_THINKING|><|END_THINKING|>
                    std::string full_open     = open_prefix + keyword;
                    size_t      full_open_pos = prompt.find(full_open);
                    if (full_open_pos == std::string::npos) {
                        continue;
                    }

                    // Find the end of this tag (look for |> or >)
                    size_t tag_end = prompt.find("|>", full_open_pos + full_open.length());
                    if (tag_end == std::string::npos) {
                        tag_end = prompt.find('>', full_open_pos + full_open.length());
                    }
                    if (tag_end == std::string::npos) {
                        continue;
                    }

                    start_tag =
                        prompt.substr(full_open_pos, tag_end - full_open_pos + (prompt[tag_end] == '|' ? 2 : 1));

                    // Look for the corresponding end tag
                    std::string expected_close = close_prefix + keyword;
                    size_t      close_pos      = prompt.find(expected_close, tag_end);
                    if (close_pos == std::string::npos) {
                        continue;
                    }

                    // Find end of close tag
                    size_t close_end = prompt.find("|>", close_pos + expected_close.length());
                    if (close_end == std::string::npos) {
                        close_end = prompt.find('>', close_pos + expected_close.length());
                    }
                    if (close_end == std::string::npos) {
                        continue;
                    }

                    end_tag = prompt.substr(close_pos, close_end - close_pos + (prompt[close_end] == '|' ? 2 : 1));
                } else {
                    // Simple pattern like <think></think>
                    start_tag        = open_prefix;
                    size_t close_pos = prompt.find(close_prefix, open_pos + start_tag.length());
                    if (close_pos == std::string::npos) {
                        continue;
                    }
                    end_tag = close_prefix;
                }

                // Verify the tags are adjacent or nearly adjacent (only whitespace between)
                size_t start_end_pos = prompt.find(start_tag) + start_tag.length();
                size_t end_start_pos = prompt.find(end_tag, start_end_pos);
                if (end_start_pos != std::string::npos) {
                    std::string between         = prompt.substr(start_end_pos, end_start_pos - start_end_pos);
                    // Allow only whitespace between the tags (empty thinking block)
                    bool        only_whitespace = true;
                    for (char c : between) {
                        if (!std::isspace(static_cast<unsigned char>(c))) {
                            only_whitespace = false;
                            break;
                        }
                    }

                    if (only_whitespace) {
                        cs.reasoning_start = start_tag;
                        cs.reasoning_end   = end_tag;
                        LOG_DBG("Method 4: Found reasoning markers via adjacent tag pairs\n");
                        LOG_DBG("  start: '%s', end: '%s'\n", cs.reasoning_start.c_str(), cs.reasoning_end.c_str());
                        break;
                    }
                }
            }
        }
    }

    if (cs.reasoning_start.empty()) {
        LOG_DBG("No reasoning markers detected\n");
    }
}

void template_analyzer::detect_content_markers(const common_chat_template & tmpl, content_structure & cs) {
    LOG_DBG("=== DETECTING CONTENT MARKERS ===\n");

    // Render template with a unique content marker
    json user_msg = {
        { "role",    "user"  },
        { "content", "Hello" }
    };
    json assistant_msg = {
        { "role",    "assistant"            },
        { "content", "UNIQUE_CONTENT_12345" }
    };

    templates_params inputs;
    inputs.messages                         = { user_msg, assistant_msg };
    // Try with thinking enabled first (some templates only wrap content when reasoning is present)
    inputs.extra_context["thinking"]        = true;
    inputs.enable_thinking                  = true;

    std::string output_with_thinking;
    try {
        output_with_thinking = common_chat_template_direct_apply(tmpl, inputs);
    } catch (...) {
        output_with_thinking = "";
    }

    // Also render without thinking
    inputs.extra_context["thinking"]        = false;
    inputs.enable_thinking                  = false;

    std::string output_no_thinking;
    try {
        output_no_thinking = common_chat_template_direct_apply(tmpl, inputs);
    } catch (...) {
        output_no_thinking = "";
    }

    // Check both outputs for content markers
    auto find_content_markers = [&](const std::string & output) -> std::pair<std::string, std::string> {
        size_t marker_pos = output.find("UNIQUE_CONTENT_12345");
        if (marker_pos == std::string::npos) {
            return { "", "" };
        }

        // Known content marker patterns
        std::vector<std::pair<std::string, std::string>> patterns = {
            { "<|START_RESPONSE|>", "<|END_RESPONSE|>"      },
            { "<|response|>",       "<|/response|>"         },
            { "<response>",         "</response>"           },
            { "<output>",           "</output>"             },
            { "<answer>",           "</answer>"             },
            { "<|CHATBOT_TOKEN|>",  "<|END_OF_TURN_TOKEN|>" },
        };

        for (const auto & [start_pattern, end_pattern] : patterns) {
            size_t start_pos = output.rfind(start_pattern, marker_pos);
            if (start_pos != std::string::npos) {
                // Check that there's only whitespace between the start pattern and our marker
                std::string between =
                    output.substr(start_pos + start_pattern.length(), marker_pos - start_pos - start_pattern.length());
                size_t first_non_ws = between.find_first_not_of(" \t\n\r");
                if (first_non_ws == std::string::npos) {
                    // Found valid start marker, look for end marker
                    size_t marker_end = marker_pos + strlen("UNIQUE_CONTENT_12345");
                    size_t end_pos    = output.find(end_pattern, marker_end);
                    if (end_pos != std::string::npos) {
                        std::string after              = output.substr(marker_end, end_pos - marker_end);
                        size_t      first_non_ws_after = after.find_first_not_of(" \t\n\r");
                        if (first_non_ws_after == std::string::npos) {
                            return { start_pattern, end_pattern };
                        }
                    }
                }
            }
        }

        return { "", "" };
    };

    auto [start_with_thinking, end_with_thinking] = find_content_markers(output_with_thinking);
    auto [start_no_thinking, end_no_thinking]     = find_content_markers(output_no_thinking);

    if (!start_with_thinking.empty() && !start_no_thinking.empty()) {
        // Content is always wrapped
        cs.content_mode  = content_structure::CONTENT_ALWAYS_WRAPPED;
        cs.content_start = start_with_thinking;
        cs.content_end   = end_with_thinking;
        LOG_DBG("Content markers found in both thinking modes (ALWAYS_WRAPPED)\n");
    } else if (!start_with_thinking.empty() && start_no_thinking.empty()) {
        // Content is wrapped only when reasoning is present
        cs.content_mode  = content_structure::CONTENT_WRAPPED_WITH_REASONING;
        cs.content_start = start_with_thinking;
        cs.content_end   = end_with_thinking;
        LOG_DBG("Content markers found only with thinking enabled (WRAPPED_WITH_REASONING)\n");
    } else if (!start_no_thinking.empty()) {
        // Unusual: content wrapped without thinking but not with? Use what we found
        cs.content_mode  = content_structure::CONTENT_ALWAYS_WRAPPED;
        cs.content_start = start_no_thinking;
        cs.content_end   = end_no_thinking;
        LOG_DBG("Content markers found only without thinking (treating as ALWAYS_WRAPPED)\n");
    } else {
        cs.content_mode = content_structure::CONTENT_PLAIN;
        LOG_DBG("No content markers detected (PLAIN)\n");
    }

    LOG_DBG("Content markers: start='%s', end='%s'\n", cs.content_start.c_str(), cs.content_end.c_str());
}

content_structure::reasoning_mode_type template_analyzer::detect_reasoning_mode(const content_structure & cs,
                                                                                const std::string &       prompt) {
    LOG_DBG("=== DETECTING REASONING MODE ===\n");

    // If both markers are empty, mode is NONE
    if (cs.reasoning_start.empty() && cs.reasoning_end.empty()) {
        LOG_DBG("No reasoning markers, mode=REASONING_NONE\n");
        return content_structure::REASONING_NONE;
    }

    // Handle case with end marker but no start marker (implicit start)
    if (cs.reasoning_start.empty() && !cs.reasoning_end.empty()) {
        LOG_DBG("Reasoning end marker present but no start marker, mode=REASONING_FORCED_OPEN\n");
        return content_structure::REASONING_FORCED_OPEN;
    }

    // Check if the prompt ends with the reasoning start marker (forced open)
    std::string trimmed_prompt = prompt;
    trim_trailing_newlines(trimmed_prompt);

    std::string trimmed_marker = cs.reasoning_start;
    trim_whitespace(trimmed_marker);

    if (string_ends_with(trimmed_prompt, trimmed_marker)) {
        LOG_DBG("Prompt ends with reasoning start marker, mode=REASONING_FORCED_OPEN\n");
        return content_structure::REASONING_FORCED_OPEN;
    }

    // Otherwise, reasoning is optional
    LOG_DBG("Reasoning markers present but not forced, mode=REASONING_OPTIONAL\n");
    return content_structure::REASONING_OPTIONAL;
}

tool_call_structure template_analyzer::analyze_tool_structure(const common_chat_template & tmpl,
                                                              const content_structure &    content) {
    (void) content;  // May be used in future for better tool detection

    LOG_DBG("=== PHASE 2: ANALYZING TOOL STRUCTURE ===\n");

    tool_call_structure ts;

    // Use differential analysis to detect tool patterns
    // This now includes a robust test that renders two payloads:
    // 1. Tool definitions + content only
    // 2. Tool definitions + content + tool calls
    // If outputs are identical, the template doesn't support tool calls
    auto discovered = analyze_by_differential(tmpl);
    auto format     = determine_format_from_patterns(discovered);

    // Strip EOS tokens from discovered patterns (handles both standard <|eos|> and fullwidth <｜end▁of▁sentence｜>)
    if (!discovered.tool_call_closer.empty()) {
        LOG_DBG("Before stripping: tool_call_closer='%s' (len=%zu)\n", discovered.tool_call_closer.c_str(),
                discovered.tool_call_closer.length());
        discovered.tool_call_closer = strip_eos_token(discovered.tool_call_closer);
        LOG_DBG("After stripping: tool_call_closer='%s'\n", discovered.tool_call_closer.c_str());
    }
    if (!discovered.tool_call_end_marker.empty()) {
        discovered.tool_call_end_marker = strip_eos_token(discovered.tool_call_end_marker);
    }

    if (format == FORMAT_UNKNOWN) {
        LOG_DBG("Template does not support tool calls (differential analysis returned no patterns)\n");
        ts.supports_tools = false;
        return ts;
    }

    // Propagate requires_nonnull_content flag from differential analysis
    ts.requires_nonnull_content = discovered.requires_nonnull_content;
    if (ts.requires_nonnull_content) {
        LOG_DBG("Template requires non-null content (renders null as 'None')\n");
    }

    // Check if minja reports tool call support (for informational purposes)
    auto caps = tmpl.original_caps();
    if (!caps.supports_tool_calls) {
        LOG_DBG("Note: minja caps indicate no tool support, but differential analysis found patterns\n");
    }

    if (format == FORMAT_JSON_NATIVE) {
        analyze_json_format(ts, discovered);
    } else if (format == FORMAT_XML_CONSTRUCTED) {
        analyze_xml_format(ts, discovered);
    } else if (format == FORMAT_BRACKET_TAG) {
        analyze_bracket_tag_format(ts, discovered);
    } else if (format == FORMAT_RECIPIENT_BASED) {
        analyze_recipient_based_format(ts, discovered);
    } else if (format == FORMAT_MARKDOWN_CODE_BLOCK) {
        analyze_markdown_code_block_format(ts, discovered);
    }

    return ts;
}

void template_analyzer::collect_preserved_tokens(template_analysis_result & result) {
    LOG_DBG("=== COLLECTING PRESERVED TOKENS ===\n");

    std::vector<std::string> tokens;

    // Add reasoning markers
    if (!result.content.reasoning_start.empty()) {
        tokens.push_back(result.content.reasoning_start);
    }
    if (!result.content.reasoning_end.empty()) {
        tokens.push_back(result.content.reasoning_end);
    }

    // Add content markers
    if (!result.content.content_start.empty()) {
        tokens.push_back(result.content.content_start);
    }
    if (!result.content.content_end.empty()) {
        tokens.push_back(result.content.content_end);
    }

    // Add tool section markers
    if (!result.tools.tool_section_start.empty()) {
        tokens.push_back(result.tools.tool_section_start);
    }
    if (!result.tools.tool_section_end.empty()) {
        tokens.push_back(result.tools.tool_section_end);
    }

    // Add function markers for tag-based formats
    if (result.tools.function_format == tool_call_structure::FUNC_TAG_WITH_NAME) {
        if (!result.tools.function_prefix.empty()) {
            tokens.push_back(result.tools.function_prefix);
        }
        if (!result.tools.function_close.empty()) {
            tokens.push_back(result.tools.function_close);
        }
    }

    // Add markers for prefixed-indexed formats (e.g., Kimi-K2)
    if (result.tools.function_format == tool_call_structure::FUNC_PREFIXED_INDEXED) {
        if (!result.tools.per_call_start.empty()) {
            tokens.push_back(result.tools.per_call_start);
        }
        if (!result.tools.args_marker.empty()) {
            tokens.push_back(result.tools.args_marker);
        }
        if (!result.tools.per_call_end.empty()) {
            tokens.push_back(result.tools.per_call_end);
        }
    }

    // Add argument markers for tagged formats
    if (result.tools.argument_format == tool_call_structure::ARGS_TAGGED) {
        if (!result.tools.arg_prefix.empty()) {
            tokens.push_back(result.tools.arg_prefix);
        }
        if (!result.tools.arg_close.empty()) {
            tokens.push_back(result.tools.arg_close);
        }
    }

    // Add markers for markdown code block format (Cohere Command-R Plus)
    if (result.tools.function_format == tool_call_structure::FUNC_MARKDOWN_CODE_BLOCK) {
        if (!result.tools.code_block_marker.empty()) {
            tokens.push_back(result.tools.code_block_marker);
        }
        if (!result.tools.tool_section_end.empty()) {
            tokens.push_back(result.tools.tool_section_end);  // Closing code fence ```
        }
    }

    result.preserved_tokens = tokens;
    LOG_DBG("Collected %zu preserved tokens\n", tokens.size());
}

void template_analyzer::analyze_json_format(tool_call_structure & ts, const internal_discovered_pattern & discovered) {
    ts.supports_tools     = true;
    ts.function_format    = tool_call_structure::FUNC_JSON_OBJECT;
    ts.argument_format    = tool_call_structure::ARGS_JSON;
    ts.tool_section_start = discovered.tool_call_start_marker;
    ts.tool_section_end   = discovered.tool_call_end_marker;
    ts.name_field         = discovered.tool_name_field;
    ts.args_field         = discovered.tool_args_field;
    ts.id_field           = discovered.tool_id_field;

    // Check for FUNC_NAME_AS_KEY format (e.g. Apertus: {"function_name": args})
    // This is characterized by the opener ending in {" and no explicit name field found yet
    if (!discovered.tool_call_opener.empty() && discovered.tool_call_opener.length() >= 2 &&
        discovered.tool_call_opener.substr(discovered.tool_call_opener.length() - 2) == "{\"") {
        LOG_DBG("Detected FUNC_NAME_AS_KEY format from tool_call_opener ending in '{\"' \n");
        ts.function_format = tool_call_structure::FUNC_NAME_AS_KEY;
    }

    // For JSON_NATIVE format, clean up tool_section_end to only include the closing tag
    // The differential analysis may include JSON closing braces (e.g., "}}\n</tool_call>")
    // but the parser handles JSON separately, so we only need the tag marker
    if (!ts.tool_section_end.empty()) {
        size_t tag_start = ts.tool_section_end.find("</");
        if (tag_start != std::string::npos) {
            size_t tag_end = ts.tool_section_end.find('>', tag_start);
            if (tag_end != std::string::npos) {
                // Check if there is a closing bracket ']' before the tag
                size_t bracket_pos = ts.tool_section_end.rfind(']', tag_start);
                if (bracket_pos != std::string::npos) {
                    // Include the bracket
                    ts.tool_section_end = ts.tool_section_end.substr(bracket_pos, tag_end - bracket_pos + 1);
                } else {
                    ts.tool_section_end = ts.tool_section_end.substr(tag_start, tag_end - tag_start + 1);
                }
            }
        } else {
            // Try other closing patterns like ]<|END_ACTION|>
            tag_start = ts.tool_section_end.find("<|");
            if (tag_start != std::string::npos) {
                size_t tag_end = ts.tool_section_end.find("|>", tag_start);
                if (tag_end != std::string::npos) {
                    // Include the opening bracket if present
                    size_t bracket_pos = ts.tool_section_end.rfind(']', tag_start);
                    if (bracket_pos != std::string::npos && bracket_pos + 1 == tag_start) {
                        ts.tool_section_end = ts.tool_section_end.substr(bracket_pos, tag_end - bracket_pos + 2);
                    } else {
                        ts.tool_section_end = ts.tool_section_end.substr(tag_start, tag_end - tag_start + 2);
                    }
                }
            }
        }
    }
}

void template_analyzer::analyze_xml_format(tool_call_structure & ts, const internal_discovered_pattern & discovered) {
    ts.supports_tools     = true;
    ts.function_format    = tool_call_structure::FUNC_TAG_WITH_NAME;
    ts.tool_section_start = discovered.tool_call_start_marker;
    ts.tool_section_end   = discovered.tool_call_end_marker;

    // Extract function tag patterns
    if (!discovered.function_opener.empty()) {
        char first = discovered.function_opener[0];
        if (first != '<' && first != '{' && first != '[') {
            // Non-XML/JSON prefix format (e.g., ">>>", "##", etc.)
            // Function name follows prefix directly, ends with newline
            ts.function_prefix = discovered.function_opener;
            ts.function_suffix = "\n";  // Function name typically ends with newline
            ts.function_close  = "";    // No closing tag for prefix formats
        } else {
            size_t eq_pos = discovered.function_opener.find('=');
            if (eq_pos != std::string::npos) {
                // Check if there's a quote after the equals sign
                if (eq_pos + 1 < discovered.function_opener.length() &&
                    (discovered.function_opener[eq_pos + 1] == '"' || discovered.function_opener[eq_pos + 1] == '\'')) {
                    ts.function_prefix = discovered.function_opener.substr(0, eq_pos + 2);
                } else {
                    ts.function_prefix = discovered.function_opener.substr(0, eq_pos + 1);
                }
                ts.function_suffix = discovered.function_name_suffix;

                // For formats like <function=name>{args}</function>, where function_prefix
                // IS the section start (no separate wrapper), tool_section_end is the function close.
                // But for nested formats like <tool_call><function=name>...</function></tool_call>,
                // the function_close is separate from tool_section_end.
                // We detect the non-nested case when tool_section_start matches function_prefix
                // (or tool_section_start was already cleared because it matched).
                bool section_start_matches_prefix = ts.tool_section_start.empty() ||
                                                    ts.tool_section_start.find(ts.function_prefix) == 0 ||
                                                    ts.function_prefix.find(ts.tool_section_start) == 0;
                if (section_start_matches_prefix && ts.function_prefix.find('<') == 0 && !ts.tool_section_end.empty() &&
                    ts.tool_section_end.find("</") == 0) {
                    ts.function_close   = ts.tool_section_end;
                    ts.tool_section_end = "";  // Clear to avoid double wrapping
                } else {
                    ts.function_close = discovered.function_closer;
                }
            } else if (!discovered.function_opener.empty() && discovered.function_opener[0] == '<') {
                // Check for FUNC_PREFIXED_INDEXED format
                // Detected by: function_opener ends with "." (namespace separator)
                //              AND function_name_suffix starts with ":" followed by digit (index)
                // Example: <|tool_call_begin|>functions.name:0<|tool_call_argument_begin|>
                size_t namespace_dot = discovered.function_opener.rfind('.');
                bool   has_namespace =
                    (namespace_dot != std::string::npos && namespace_dot == discovered.function_opener.length() - 1);

                bool has_index =
                    (!discovered.function_name_suffix.empty() && discovered.function_name_suffix[0] == ':' &&
                     discovered.function_name_suffix.length() > 1 &&
                     std::isdigit(static_cast<unsigned char>(discovered.function_name_suffix[1])));

                if (has_namespace && has_index) {
                    LOG_DBG("Detected FUNC_PREFIXED_INDEXED format: namespace ends with '.', suffix has ':N' index\n");
                    ts.function_format = tool_call_structure::FUNC_PREFIXED_INDEXED;

                    // Split function_opener into per_call_start and function_namespace
                    // e.g., "<|tool_call_begin|>functions." -> "<|tool_call_begin|>" + "functions."
                    // Find where the namespace starts (after the last '>' before the '.')
                    size_t namespace_start = discovered.function_opener.rfind('>');
                    if (namespace_start != std::string::npos && namespace_start < namespace_dot) {
                        ts.per_call_start     = discovered.function_opener.substr(0, namespace_start + 1);
                        ts.function_namespace = discovered.function_opener.substr(namespace_start + 1);
                    } else {
                        // Fallback: namespace is just the part ending with '.'
                        ts.per_call_start     = discovered.function_opener.substr(0, namespace_dot);
                        ts.function_namespace = ".";
                    }

                    // Extract args_marker from function_name_suffix
                    // Format: ":0<|some_marker|>" -> index is ":0", args_marker is "<|some_marker|>"
                    size_t args_marker_start = discovered.function_name_suffix.find('<');
                    if (args_marker_start != std::string::npos) {
                        size_t args_marker_end = discovered.function_name_suffix.find('>', args_marker_start);
                        if (args_marker_end != std::string::npos) {
                            ts.args_marker = discovered.function_name_suffix.substr(
                                args_marker_start, args_marker_end - args_marker_start + 1);
                        }
                    }

                    // Derive per_call_end from tool_call_closer by finding corresponding end marker
                    // tool_call_closer contains per_call_end + tool_section_end
                    // We find per_call_end by looking for a marker that structurally matches per_call_start
                    if (!discovered.tool_call_closer.empty() && !ts.per_call_start.empty()) {
                        // Extract structural pattern from per_call_start
                        // e.g., "<|tool_call_begin|>" -> look for "<|tool_call_...|>" in closer
                        size_t start_marker_begin = ts.per_call_start.find("<|");
                        size_t start_marker_end   = ts.per_call_start.rfind("|>");
                        if (start_marker_begin != std::string::npos && start_marker_end != std::string::npos) {
                            // Find the base pattern (e.g., "<|tool_call" from "<|tool_call_begin|>")
                            std::string start_content = ts.per_call_start.substr(
                                start_marker_begin + 2, start_marker_end - start_marker_begin - 2);
                            // Find a related marker in the closer
                            size_t closer_pos = discovered.tool_call_closer.find("<|");
                            while (closer_pos != std::string::npos) {
                                size_t closer_end = discovered.tool_call_closer.find("|>", closer_pos);
                                if (closer_end != std::string::npos) {
                                    std::string candidate =
                                        discovered.tool_call_closer.substr(closer_pos, closer_end - closer_pos + 2);
                                    // Check if this marker shares a common prefix with per_call_start
                                    // (ignoring _begin vs _end suffix differences)
                                    std::string candidate_content = candidate.substr(2, candidate.length() - 4);
                                    // Find common prefix between start_content and candidate_content
                                    size_t      common_len        = 0;
                                    while (common_len < start_content.length() &&
                                           common_len < candidate_content.length() &&
                                           start_content[common_len] == candidate_content[common_len]) {
                                        common_len++;
                                    }
                                    // If substantial overlap (>50%), this is likely the per_call_end
                                    if (common_len > start_content.length() / 2 &&
                                        candidate_content.find("end") != std::string::npos) {
                                        ts.per_call_end = candidate;
                                        break;
                                    }
                                }
                                closer_pos = discovered.tool_call_closer.find("<|", closer_pos + 1);
                            }
                        }
                    }

                    // Derive tool_section_end from tool_section_start by finding matching end marker
                    // For FUNC_PREFIXED_INDEXED, we always derive this to get the correct marker
                    // (the default discovered.tool_call_end_marker may contain extra content)
                    if (!ts.tool_section_start.empty()) {
                        size_t start_marker_begin = ts.tool_section_start.find("<|");
                        size_t start_marker_end   = ts.tool_section_start.rfind("|>");
                        if (start_marker_begin != std::string::npos && start_marker_end != std::string::npos) {
                            std::string start_content = ts.tool_section_start.substr(
                                start_marker_begin + 2, start_marker_end - start_marker_begin - 2);
                            size_t closer_pos = discovered.tool_call_closer.find("<|");
                            while (closer_pos != std::string::npos) {
                                size_t closer_end = discovered.tool_call_closer.find("|>", closer_pos);
                                if (closer_end != std::string::npos) {
                                    std::string candidate =
                                        discovered.tool_call_closer.substr(closer_pos, closer_end - closer_pos + 2);
                                    std::string candidate_content = candidate.substr(2, candidate.length() - 4);
                                    size_t      common_len        = 0;
                                    while (common_len < start_content.length() &&
                                           common_len < candidate_content.length() &&
                                           start_content[common_len] == candidate_content[common_len]) {
                                        common_len++;
                                    }
                                    if (common_len > start_content.length() / 2 &&
                                        candidate_content.find("end") != std::string::npos) {
                                        ts.tool_section_end = candidate;
                                        break;
                                    }
                                }
                                closer_pos = discovered.tool_call_closer.find("<|", closer_pos + 1);
                            }
                        }
                    }

                    LOG_DBG(
                        "FUNC_PREFIXED_INDEXED: per_call_start='%s', namespace='%s', args_marker='%s', "
                        "per_call_end='%s'\n",
                        ts.per_call_start.c_str(), ts.function_namespace.c_str(), ts.args_marker.c_str(),
                        ts.per_call_end.c_str());
                } else {
                    // Other formats like <|tool_call_begin|>name (non-indexed)
                    // Use function_opener as default, but try to use full tool_call_opener if it contains more
                    ts.function_prefix = discovered.function_opener;
                    LOG_DBG("Initial function_prefix: '%s', tool_call_opener: '%s', tool_section_start: '%s'\n",
                            ts.function_prefix.c_str(), discovered.tool_call_opener.c_str(),
                            ts.tool_section_start.c_str());
                    if (!ts.tool_section_start.empty() &&
                        discovered.tool_call_opener.find(ts.tool_section_start) == 0) {
                        std::string remainder = discovered.tool_call_opener.substr(ts.tool_section_start.length());
                        LOG_DBG("Derived remainder: '%s'\n", remainder.c_str());
                        if (remainder.length() > ts.function_prefix.length()) {
                            ts.function_prefix = remainder;
                        }
                    }
                    ts.function_suffix = discovered.function_name_suffix;
                    ts.function_close  = discovered.function_closer;
                }
            }
        }
    }

    // Fix for templates where tool_section_start matches function_prefix (double wrapping)
    // e.g. Functionary: tool_section_start="<function=", function_prefix="<function="
    if (!ts.tool_section_start.empty() && !ts.function_prefix.empty() && ts.tool_section_start == ts.function_prefix) {
        LOG_DBG("tool_section_start matches function_prefix, clearing section start to avoid double wrapping\n");
        ts.tool_section_start = "";
    }

    // Similar check for tool_section_end matching function_close
    if (!ts.tool_section_end.empty() && !ts.function_close.empty() && ts.tool_section_end == ts.function_close) {
        LOG_DBG("tool_section_end matches function_close, clearing section end to avoid double wrapping\n");
        ts.tool_section_end = "";
    }

    // Handle nested container markers (e.g., DeepSeek R1 style)
    // If function_suffix contains markdown code block (```), the template uses nested markers
    // tool_section_start might be: <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function
    // We need to derive tool_section_end from the outer marker pattern
    if (ts.function_suffix.find("```") != std::string::npos && !ts.tool_section_start.empty()) {
        // Check if tool_section_start contains nested markers (both outer and per-call)
        // Pattern: <X_calls_begin><X_call_begin>...
        // We look for "calls" pattern which indicates an outer container
        size_t calls_pos = ts.tool_section_start.find("calls");
        if (calls_pos != std::string::npos && calls_pos < ts.tool_section_start.length()) {
            // Find where the outer marker ends (after the first >)
            size_t first_close = ts.tool_section_start.find('>', calls_pos);
            if (first_close != std::string::npos && first_close < ts.tool_section_start.length() - 1) {
                // Extract the outer marker (e.g., "<｜tool▁calls▁begin｜>")
                std::string outer_start = ts.tool_section_start.substr(0, first_close + 1);
                // Derive the outer end marker by replacing "begin" with "end"
                size_t      begin_pos   = outer_start.find("begin");
                if (begin_pos != std::string::npos) {
                    std::string outer_end =
                        outer_start.substr(0, begin_pos) + "end" + outer_start.substr(begin_pos + 5);
                    ts.tool_section_end = outer_end;

                    // Strip outer marker from function_prefix and function_opener if they were combined
                    if (ts.tool_section_start.find(outer_start) == 0) {
                        std::string remainder    = ts.tool_section_start.substr(outer_start.length());
                        // Trim leading whitespace from remainder
                        size_t      first_non_ws = remainder.find_first_not_of(" \t\n\r");
                        if (first_non_ws != std::string::npos && first_non_ws > 0) {
                            remainder = remainder.substr(first_non_ws);
                        }

                        // Concatenate with existing function_prefix (e.g. separator tag)
                        // but avoid double-concatenation if already present
                        if (!remainder.empty() && ts.function_prefix.find(remainder) == std::string::npos) {
                            ts.function_prefix = remainder + ts.function_prefix;
                        }
                    }

                    // Update tool_section_start to be just the outer marker
                    ts.tool_section_start = outer_start;

                    // Check if there's a fence in tool_call_closer that should be in function_close
                    // (DeepSeek R1 wraps JSON in markdown blocks within the custom tags)
                    if (discovered.tool_call_closer.find("```") != std::string::npos) {
                        size_t fence_pos = discovered.tool_call_closer.find("```");
                        // Include leading newlines if present before the fence
                        while (fence_pos > 0 && (discovered.tool_call_closer[fence_pos - 1] == '\n' ||
                                                 discovered.tool_call_closer[fence_pos - 1] == '\r')) {
                            fence_pos--;
                        }
                        ts.function_close = discovered.tool_call_closer.substr(fence_pos);

                        // Clip function_close to not include tool_section_end (if they were combined in differential analysis)
                        if (!ts.tool_section_end.empty()) {
                            size_t end_pos = ts.function_close.find(ts.tool_section_end);
                            if (end_pos != std::string::npos) {
                                ts.function_close = ts.function_close.substr(0, end_pos);
                            }
                        }

                        // Further trim any trailing EOS or prompt garbage
                        ts.function_close     = strip_eos_token(ts.function_close);
                        size_t prompt_garbage = ts.function_close.find("<｜");
                        if (prompt_garbage != std::string::npos && prompt_garbage > 0 &&
                            ts.function_close.substr(prompt_garbage).find("Assistant") != std::string::npos) {
                            ts.function_close = ts.function_close.substr(0, prompt_garbage);
                        }
                    }
                }
            }
        }
    }

    // General cleanup for tool_section_end when tool_section_start uses token markers (<|...|> or <｜...｜>)
    // If tool_section_start contains a token marker with "begin" and tool_section_end is messy (contains }
    // or multiple markers), derive tool_section_end by finding matching end marker in tool_call_closer
    if (!ts.tool_section_start.empty() && !discovered.tool_call_closer.empty()) {
        // Check if tool_section_start contains a token marker
        size_t start_opener_pos = find_token_opener(ts.tool_section_start, 0);
        size_t start_closer_pos = find_token_closer(ts.tool_section_start, start_opener_pos);
        if (start_opener_pos != std::string::npos && start_closer_pos != std::string::npos) {
            size_t      opener_len    = get_token_opener_length(ts.tool_section_start, start_opener_pos);
            // Extract the token content (between opener and closer)
            std::string start_content = ts.tool_section_start.substr(start_opener_pos + opener_len,
                                                                     start_closer_pos - start_opener_pos - opener_len);

            // Check if tool_section_end needs cleanup (starts with } or contains multiple markers)
            bool needs_cleanup = false;
            if (!ts.tool_section_end.empty() && ts.tool_section_end[0] == '}') {
                needs_cleanup = true;
            }
            // Count tokens in tool_section_end
            size_t token_count = 0;
            size_t pos         = 0;
            while ((pos = find_token_opener(ts.tool_section_end, pos)) != std::string::npos) {
                token_count++;
                pos += get_token_opener_length(ts.tool_section_end, pos);
            }
            if (token_count > 1) {
                needs_cleanup = true;
            }

            if (needs_cleanup) {
                // Find matching end marker in tool_call_closer
                // Look for a token that has similar content but with "end" instead of "begin"
                pos = 0;
                while ((pos = find_token_opener(discovered.tool_call_closer, pos)) != std::string::npos) {
                    size_t end_closer_pos = find_token_closer(discovered.tool_call_closer, pos);
                    if (end_closer_pos != std::string::npos) {
                        size_t      op_len    = get_token_opener_length(discovered.tool_call_closer, pos);
                        size_t      cl_len    = get_token_closer_length(discovered.tool_call_closer, end_closer_pos);
                        std::string candidate = discovered.tool_call_closer.substr(pos, end_closer_pos + cl_len - pos);
                        std::string candidate_content =
                            discovered.tool_call_closer.substr(pos + op_len, end_closer_pos - pos - op_len);

                        // Check if this candidate matches our start marker structure
                        // Start content might be "tool▁calls▁begin" and candidate might be "tool▁calls▁end"
                        size_t begin_in_start   = start_content.find("begin");
                        size_t end_in_candidate = candidate_content.find("end");
                        if (begin_in_start != std::string::npos && end_in_candidate != std::string::npos) {
                            // Check if they share a common prefix (e.g., "tool▁calls▁")
                            std::string start_base = start_content.substr(0, begin_in_start);
                            std::string cand_base  = candidate_content.substr(0, end_in_candidate);
                            if (start_base == cand_base) {
                                ts.tool_section_end = candidate;
                                LOG_DBG(
                                    "Derived tool_section_end='%s' from tool_section_start='%s' using token matching\n",
                                    ts.tool_section_end.c_str(), ts.tool_section_start.c_str());
                                break;
                            }
                        }
                    }
                    pos += get_token_opener_length(discovered.tool_call_closer, pos);
                }
            }
        }
    }

    // Determine argument format
    if (!discovered.parameter_key_prefix.empty() && discovered.parameter_key_prefix.find('<') != std::string::npos) {
        ts.argument_format = tool_call_structure::ARGS_TAGGED;
        ts.arg_prefix      = discovered.parameter_key_prefix;
        ts.arg_suffix      = discovered.parameter_key_suffix;
        ts.arg_close       = discovered.parameter_closer;
        ts.arg_separator   = discovered.argument_separator;

        // Check for specific GLM-4 style key-value tags
        // Format: <arg_key>key</arg_key>\n<arg_value>value</arg_value>
        // Analyzer detects suffix as: </arg_key>\n<arg_value>
        if (ts.arg_suffix.find("<arg_value>") != std::string::npos) {
            ts.argument_format = tool_call_structure::ARGS_KEY_VALUE_TAGS;

            // Clean up suffix to be just the key closer
            size_t val_opener = ts.arg_suffix.find("<arg_value>");
            if (val_opener != std::string::npos) {
                // Extract just the </arg_key> part (trimming whitespace/newlines before <arg_value>)
                std::string key_closer = ts.arg_suffix.substr(0, val_opener);
                // Trim trailing whitespace/newlines
                while (!key_closer.empty() &&
                       (key_closer.back() == '\n' || key_closer.back() == '\r' || key_closer.back() == ' ')) {
                    key_closer.pop_back();
                }
                ts.arg_suffix = key_closer;
            }
        }
    } else {
        ts.argument_format = tool_call_structure::ARGS_JSON;
    }

    LOG_DBG("%s: final markers: section_start='%s', section_end='%s', prefix='%s', close='%s'\n", __func__,
            ts.tool_section_start.c_str(), ts.tool_section_end.c_str(), ts.function_prefix.c_str(),
            ts.function_close.c_str());
}

void template_analyzer::analyze_bracket_tag_format(tool_call_structure &               ts,
                                                   const internal_discovered_pattern & discovered) {
    // Bracket-tag format: [TOOL_CALLS]name[CALL_ID]id[ARGS]{...} (Mistral Small 3.2)
    ts.supports_tools  = true;
    ts.function_format = tool_call_structure::FUNC_BRACKET_TAG;
    ts.argument_format = tool_call_structure::ARGS_JSON;

    // The function_opener contains the bracket tag before the function name (e.g., "[TOOL_CALLS]")
    // Each tool call starts with this tag, so it's the per_call_start, not a section wrapper
    // tool_section_start/end should be empty since there's no overall section wrapper
    ts.tool_section_start = "";
    ts.tool_section_end   = "";
    ts.per_call_start     = discovered.function_opener;

    // Extract markers from function_name_suffix (e.g., "[CALL_ID]call_0001[ARGS]" or just "[ARGS]")
    // Pattern: [ID_MARKER]...[ARGS_MARKER] or just [ARGS_MARKER]
    if (!discovered.function_name_suffix.empty()) {
        // Find all bracket tags in the suffix
        std::vector<std::string> tags;
        size_t                   pos = 0;
        while ((pos = discovered.function_name_suffix.find('[', pos)) != std::string::npos) {
            size_t end = discovered.function_name_suffix.find(']', pos);
            if (end != std::string::npos) {
                tags.push_back(discovered.function_name_suffix.substr(pos, end - pos + 1));
                pos = end + 1;
            } else {
                break;
            }
        }

        // Classify tags: args marker contains "ARG", id marker contains "ID" or "CALL"
        for (const auto & tag : tags) {
            std::string upper_tag = tag;
            for (auto & c : upper_tag) {
                c = static_cast<char>(std::toupper(static_cast<unsigned char>(c)));
            }
            if (upper_tag.find("ARG") != std::string::npos) {
                ts.args_marker = tag;
            } else if (upper_tag.find("ID") != std::string::npos || upper_tag.find("CALL") != std::string::npos) {
                ts.id_marker = tag;
            }
        }
    }

    LOG_DBG("FUNC_BRACKET_TAG: per_call_start='%s', id_marker='%s', args_marker='%s'\n", ts.per_call_start.c_str(),
            ts.id_marker.c_str(), ts.args_marker.c_str());
}

void template_analyzer::analyze_recipient_based_format(tool_call_structure &               ts,
                                                       const internal_discovered_pattern & discovered) {
    // Recipient-based format (Functionary v3.2): >>>recipient\n{content}
    // where recipient is either "all" (for content) or a function name (for tools)
    ts.supports_tools  = true;
    ts.function_format = tool_call_structure::FUNC_RECIPIENT_BASED;
    ts.argument_format = tool_call_structure::ARGS_JSON;  // Python dict format, parse as JSON

    // The tool_call_start_marker is used as the recipient delimiter
    ts.tool_section_start = discovered.tool_call_start_marker;
    ts.tool_section_end   = "";

    // For recipient-based format, content is wrapped in tool_call_start_marker + "all\n"
    // This needs to be detected and stripped. We detect this by checking if the
    // content_start marker (from phase 1 analysis) starts with tool_call_start_marker
    // If not already detected, infer it from the pattern.
    // Note: This is set on the ContentStructure result, not ToolCallStructure
    // The caller (analyze_template) will have the ContentStructure to modify

    LOG_DBG("FUNC_RECIPIENT_BASED: delimiter='%s'\n", ts.tool_section_start.c_str());
}

void template_analyzer::analyze_markdown_code_block_format(tool_call_structure &               ts,
                                                           const internal_discovered_pattern & discovered) {
    // Markdown code block format (Cohere Command-R Plus):
    // Action:
    // ```json
    // [
    //     {
    //         "tool_name": "...",
    //         "parameters": {...}
    //     }
    // ]
    // ```
    ts.supports_tools  = true;
    ts.function_format = tool_call_structure::FUNC_MARKDOWN_CODE_BLOCK;
    ts.argument_format = tool_call_structure::ARGS_JSON;

    // Extract the code block marker (e.g., "Action:")
    // The tool_call_start_marker should contain "Action:" followed by newline
    if (!discovered.tool_call_start_marker.empty()) {
        // Extract just the marker text (e.g., "Action:")
        // The marker may be followed by whitespace/newline in the template
        size_t marker_end = discovered.tool_call_start_marker.find_first_of(" \n\r\t");
        if (marker_end != std::string::npos) {
            ts.code_block_marker = discovered.tool_call_start_marker.substr(0, marker_end);
        } else {
            ts.code_block_marker = discovered.tool_call_start_marker;
        }
    }

    // Extract the code block language (e.g., "json")
    // For Command-R Plus format: Action:\n```json\n[...]
    // The code fence is in tool_call_opener (before the function name), not function_name_suffix
    if (!discovered.function_name_suffix.empty() && discovered.function_name_suffix.find("```") != std::string::npos) {
        // Format: ```json or ```json\n
        size_t code_fence_pos = discovered.function_name_suffix.find("```");
        size_t lang_start     = code_fence_pos + 3;
        // Find the end of the language identifier (newline, space, or end of string)
        size_t lang_end       = discovered.function_name_suffix.find_first_of(" \n\r\t", lang_start);
        if (lang_end != std::string::npos && lang_end > lang_start) {
            ts.code_block_language = discovered.function_name_suffix.substr(lang_start, lang_end - lang_start);
        } else {
            // No language identifier after ```, will use "json" as default
            ts.code_block_language = "json";
        }
    } else if (!discovered.tool_call_opener.empty() && discovered.tool_call_opener.find("```") != std::string::npos) {
        // Code fence is in tool_call_opener (before the function name)
        // Format: Action:\n```json\n[...
        size_t code_fence_pos = discovered.tool_call_opener.find("```");
        size_t lang_start     = code_fence_pos + 3;
        // Find the end of the language identifier (newline, space, or end of string)
        size_t lang_end       = discovered.tool_call_opener.find_first_of(" \n\r\t", lang_start);
        if (lang_end != std::string::npos && lang_end > lang_start) {
            ts.code_block_language = discovered.tool_call_opener.substr(lang_start, lang_end - lang_start);
        } else {
            // No language identifier after ```, will use "json" as default
            ts.code_block_language = "json";
        }
    } else {
        // Default to "json" if no code fence found
        ts.code_block_language = "json";
    }

    // The tool_section_end should be the closing code fence: ```
    if (!discovered.tool_call_closer.empty() && discovered.tool_call_closer.find("```") != std::string::npos) {
        // Extract just the closing code fence (may have trailing content)
        size_t fence_pos = discovered.tool_call_closer.find("```");
        size_t fence_end = fence_pos + 3;
        // Include any non-newline characters after ``` (like language identifier if present)
        while (fence_end < discovered.tool_call_closer.length() && discovered.tool_call_closer[fence_end] != '\n' &&
               discovered.tool_call_closer[fence_end] != '\r') {
            fence_end++;
        }
        ts.tool_section_end = discovered.tool_call_closer.substr(fence_pos, fence_end - fence_pos);
    } else {
        // Default closing code fence
        ts.tool_section_end = "```";
    }

    // JSON array format for function calls
    ts.name_field = discovered.tool_name_field;
    ts.args_field = discovered.tool_args_field;
    ts.id_field   = discovered.tool_id_field;

    LOG_DBG("FUNC_MARKDOWN_CODE_BLOCK: marker='%s', language='%s', section_end='%s'\n", ts.code_block_marker.c_str(),
            ts.code_block_language.c_str(), ts.tool_section_end.c_str());
}