diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp
index b206d83222..aab13121a2 100644
--- a/common/chat-auto-parser-generator.cpp
+++ b/common/chat-auto-parser-generator.cpp
@@ -1,4 +1,5 @@
#include "chat-auto-parser.h"
+#include "chat-auto-parser-helpers.h"
#include "chat-peg-parser.h"
#include "chat.h"
#include "common.h"
@@ -54,21 +55,13 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
const auto & r_start = autoparser.reasoning.start;
const auto & r_end = autoparser.reasoning.end;
- auto rtrim = [](std::string s) {
- while (!s.empty() && (s.back() == ' ' || s.back() == '\n' ||
- s.back() == '\r' || s.back() == '\t')) {
- s.pop_back();
- }
- return s;
- };
-
- auto prompt_trimmed = rtrim(data.prompt);
- auto r_end_trimmed = rtrim(r_end);
- auto r_start_trimmed = rtrim(r_start);
+ auto prompt_trimmed = trim_trailing_whitespace(data.prompt);
+ auto r_end_trimmed = trim_trailing_whitespace(r_end);
+ auto r_start_trimmed = trim_trailing_whitespace(r_start);
if (!r_start_trimmed.empty()) {
if (string_ends_with(prompt_trimmed, r_end_trimmed)) {
- auto before_end = rtrim(prompt_trimmed.substr(0, prompt_trimmed.size() - r_end_trimmed.size()));
+ auto before_end = trim_trailing_whitespace(prompt_trimmed.substr(0, prompt_trimmed.size() - r_end_trimmed.size()));
if (string_ends_with(before_end, r_start_trimmed)) {
// Start+end at prompt end — use canonical markers to preserve whitespace.
data.reasoning_prefill = r_start + r_end;
@@ -185,7 +178,7 @@ common_peg_parser analyze_reasoning::build_parser(parser_build_context & ctx) co
// Standard tag-based: optional(reasoning)
return p.optional(start + p.reasoning(p.until(end)) + end);
}
- // Delimiter-style (empty start): optional(reasoning[DELIMITER])
+ // Delimiter-style (empty start)
return p.optional(p.reasoning(p.until(end)) + end);
}
}
diff --git a/docs/autoparser.md b/docs/autoparser.md
index 905393b08d..b2374391a1 100644
--- a/docs/autoparser.md
+++ b/docs/autoparser.md
@@ -266,8 +266,8 @@ Text is segmentized into markers and non-marker fragments using `segmentize_mark
- Searches `diff.right` (output with reasoning) for the reasoning content needle
- Uses PEG parsers to find surrounding markers:
- - If both pre/post markers found in `diff.right` → `TAG_BASED` (both tags visible in diff = no forced close)
- - If both found but post marker only in the full output B → `FORCED_CLOSED`
+ - If both pre/post markers found in `diff.right` → `TAG_BASED`
+ - If both found but post marker only in the full output B → `TAG_BASED` (template forces markers; handled via prefill)
- If only post marker found → `TAG_BASED` (delimiter-style, empty start)
- Sets `reasoning.start` and `reasoning.end`
@@ -349,7 +349,7 @@ Classification logic:
A workaround array in `common/chat-diff-analyzer.cpp` applies post-hoc patches after analysis. Each workaround is a lambda that inspects the template source and overrides analysis results. Current workarounds:
-1. **Old Qwen/DeepSeek thinking templates** — source contains `content.split('')`: sets `reasoning.mode = FORCED_OPEN` with ``/`` markers if no reasoning was detected
+1. **Old Qwen/DeepSeek thinking templates** — source contains `content.split('')`: sets `reasoning.mode = TAG_BASED` with ``/`` markers if no reasoning was detected
2. **Granite 3.3** — source contains specific "Write your thoughts" text: forces `TAG_BASED` reasoning with ``/`` and `WRAPPED_WITH_REASONING` content with ``/``
3. **Cohere Command R+** — source contains `<|CHATBOT_TOKEN|>`: sets `ALWAYS_WRAPPED` content mode if no content start is already set
4. **Functionary 3.1** — source contains `set has_code_interpreter`: forces `PLAIN` content, specific `per_call_start/end`, clears preserved tokens to only keep Functionary-specific markers
diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp
index 491522324a..e140eb8ebf 100644
--- a/tests/test-chat-auto-parser.cpp
+++ b/tests/test-chat-auto-parser.cpp
@@ -1295,7 +1295,7 @@ static void test_nemotron_reasoning_detection(testing & t) {
t.assert_equal("reasoning_end should be '\\n'", "\n", analysis.reasoning.end);
// Check reasoning mode detection
- // Nemotron uses tag-based reasoning (formerly FORCED_CLOSED; prefill handles the template's forced markers)
+ // Nemotron uses tag-based reasoning; prefill handles the template's forced markers
t.assert_equal("reasoning should be TAG_BASED", reasoning_mode::TAG_BASED, analysis.reasoning.mode);
// Make sure reasoning markers don't spill over to content markers