diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index b206d83222..aab13121a2 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -1,4 +1,5 @@ #include "chat-auto-parser.h" +#include "chat-auto-parser-helpers.h" #include "chat-peg-parser.h" #include "chat.h" #include "common.h" @@ -54,21 +55,13 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & const auto & r_start = autoparser.reasoning.start; const auto & r_end = autoparser.reasoning.end; - auto rtrim = [](std::string s) { - while (!s.empty() && (s.back() == ' ' || s.back() == '\n' || - s.back() == '\r' || s.back() == '\t')) { - s.pop_back(); - } - return s; - }; - - auto prompt_trimmed = rtrim(data.prompt); - auto r_end_trimmed = rtrim(r_end); - auto r_start_trimmed = rtrim(r_start); + auto prompt_trimmed = trim_trailing_whitespace(data.prompt); + auto r_end_trimmed = trim_trailing_whitespace(r_end); + auto r_start_trimmed = trim_trailing_whitespace(r_start); if (!r_start_trimmed.empty()) { if (string_ends_with(prompt_trimmed, r_end_trimmed)) { - auto before_end = rtrim(prompt_trimmed.substr(0, prompt_trimmed.size() - r_end_trimmed.size())); + auto before_end = trim_trailing_whitespace(prompt_trimmed.substr(0, prompt_trimmed.size() - r_end_trimmed.size())); if (string_ends_with(before_end, r_start_trimmed)) { // Start+end at prompt end — use canonical markers to preserve whitespace. data.reasoning_prefill = r_start + r_end; @@ -185,7 +178,7 @@ common_peg_parser analyze_reasoning::build_parser(parser_build_context & ctx) co // Standard tag-based: optional(reasoning) return p.optional(start + p.reasoning(p.until(end)) + end); } - // Delimiter-style (empty start): optional(reasoning[DELIMITER]) + // Delimiter-style (empty start) return p.optional(p.reasoning(p.until(end)) + end); } } diff --git a/docs/autoparser.md b/docs/autoparser.md index 905393b08d..b2374391a1 100644 --- a/docs/autoparser.md +++ b/docs/autoparser.md @@ -266,8 +266,8 @@ Text is segmentized into markers and non-marker fragments using `segmentize_mark - Searches `diff.right` (output with reasoning) for the reasoning content needle - Uses PEG parsers to find surrounding markers: - - If both pre/post markers found in `diff.right` → `TAG_BASED` (both tags visible in diff = no forced close) - - If both found but post marker only in the full output B → `FORCED_CLOSED` + - If both pre/post markers found in `diff.right` → `TAG_BASED` + - If both found but post marker only in the full output B → `TAG_BASED` (template forces markers; handled via prefill) - If only post marker found → `TAG_BASED` (delimiter-style, empty start) - Sets `reasoning.start` and `reasoning.end` @@ -349,7 +349,7 @@ Classification logic: A workaround array in `common/chat-diff-analyzer.cpp` applies post-hoc patches after analysis. Each workaround is a lambda that inspects the template source and overrides analysis results. Current workarounds: -1. **Old Qwen/DeepSeek thinking templates** — source contains `content.split('')`: sets `reasoning.mode = FORCED_OPEN` with ``/`` markers if no reasoning was detected +1. **Old Qwen/DeepSeek thinking templates** — source contains `content.split('')`: sets `reasoning.mode = TAG_BASED` with ``/`` markers if no reasoning was detected 2. **Granite 3.3** — source contains specific "Write your thoughts" text: forces `TAG_BASED` reasoning with ``/`` and `WRAPPED_WITH_REASONING` content with ``/`` 3. **Cohere Command R+** — source contains `<|CHATBOT_TOKEN|>`: sets `ALWAYS_WRAPPED` content mode if no content start is already set 4. **Functionary 3.1** — source contains `set has_code_interpreter`: forces `PLAIN` content, specific `per_call_start/end`, clears preserved tokens to only keep Functionary-specific markers diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 491522324a..e140eb8ebf 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -1295,7 +1295,7 @@ static void test_nemotron_reasoning_detection(testing & t) { t.assert_equal("reasoning_end should be '\\n'", "\n", analysis.reasoning.end); // Check reasoning mode detection - // Nemotron uses tag-based reasoning (formerly FORCED_CLOSED; prefill handles the template's forced markers) + // Nemotron uses tag-based reasoning; prefill handles the template's forced markers t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode); // Make sure reasoning markers don't spill over to content markers