happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

common : gracefully handle incomplete output (#20191) 

* common : handle incomplete UTF-8 at end of input in PEG parser

* cont : if reached end prematurely, emit needs_more_input to propagate partial output

* cont: refactor peg parse context to add lenient flag

* cont : remove partial flag, keep lenient flag
This commit is contained in:
Aldehir Rojas 2026-03-08 11:17:02 -05:00 committed by GitHub
parent 9b24886f78
commit 451ef08432
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 139 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
common/chat-peg-parser.cpp
View File

@ -167,8 +167,8 @@ void tag_based_peg_mapper::from_ast(const common_peg_ast_arena & arena, const co
});
}
tagged_parse_result tagged_peg_parser::parse_and_extract(const std::string & input, bool is_partial) const {
common_peg_parse_context ctx(input, is_partial);
tagged_parse_result tagged_peg_parser::parse_and_extract(const std::string & input, common_peg_parse_flags extra_flags) const {
common_peg_parse_context ctx(input, flags | extra_flags);
auto parse_result = arena.parse(ctx);
tag_based_peg_mapper mapper;
@ -179,11 +179,10 @@ tagged_parse_result tagged_peg_parser::parse_and_extract(const std::string & inp
tagged_parse_result tagged_peg_parser::parse_anywhere_and_extract(const std::string & input) const {
if (input.empty()) {
return parse_and_extract(input, false);
return parse_and_extract(input);
}
for (size_t i = 0; i < input.size(); i++) {
common_peg_parse_context ctx(input, false);
ctx.debug = debug;
common_peg_parse_context ctx(input, flags);
auto parse_result = arena.parse(ctx, i);
if (parse_result.success() || i == input.size() - 1) {
tag_based_peg_mapper mapper;

8
common/chat-peg-parser.h
View File

@ -155,19 +155,19 @@ struct tagged_parse_result {
struct tagged_peg_parser {
common_peg_arena arena;
bool debug = false;
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE;
tagged_peg_parser & withDebug() {
debug = true;
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
return *this;
}
tagged_peg_parser & withoutDebug() {
debug = false;
flags = flags & ~COMMON_PEG_PARSE_FLAG_DEBUG;
return *this;
}
tagged_parse_result parse_and_extract(const std::string & input, bool is_partial = false) const;
tagged_parse_result parse_and_extract(const std::string & input, common_peg_parse_flags extra_flags = COMMON_PEG_PARSE_FLAG_NONE) const;
tagged_parse_result parse_anywhere_and_extract(const std::string & input) const;
};

12
common/chat.cpp
View File

@ -1527,8 +1527,12 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), input.c_str());
common_peg_parse_context ctx(input, is_partial);
ctx.debug = params.debug;
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
if (params.debug) {
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
}
common_peg_parse_context ctx(input, flags);
auto result = parser.parse(ctx);
if (result.fail()) {
@ -1541,7 +1545,7 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "\nAST for partial parse (fail):\n%s\n", ctx.ast.dump().c_str());
fflush(stderr);
}
@ -1557,7 +1561,7 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
auto mapper = common_chat_peg_mapper(msg);
mapper.from_ast(ctx.ast, result);
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "\nAST for %s parse:\n%s\n", is_partial ? "partial" : "full", ctx.ast.dump().c_str());
fflush(stderr);
}

75
common/peg-parser.cpp
View File

@ -349,7 +349,7 @@ struct parser_executor {
auto pos = start_pos;
for (auto i = 0u; i < p.literal.size(); ++i) {
if (pos >= ctx.input.size()) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -364,7 +364,7 @@ struct parser_executor {
}
common_peg_parse_result operator()(const common_peg_sequence_parser & p) {
if (ctx.debug) {
if (ctx.is_debug()) {
LOG_DBG("%sSEQ start at %zu '%s' (%zu children)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.children.size());
}
@ -375,26 +375,19 @@ struct parser_executor {
for (size_t i = 0; i < p.children.size(); i++) {
const auto & child_id = p.children[i];
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ child %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str());
}
auto result = arena.parse(child_id, ctx, pos);
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ child %zu: %s at %zu->%zu\n", debug_indent().c_str(), i,
common_peg_parse_result_type_name(result.type), result.start, result.end);
}
if (result.fail()) {
ctx.parse_depth--;
if (ctx.is_partial && result.end >= ctx.input.size()) {
if (ctx.debug) {
fprintf(stderr, "%sSEQ -> NEED_MORE (child failed at end)\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end,
std::move(nodes));
}
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> FAIL\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, result.end);
@ -406,7 +399,7 @@ struct parser_executor {
if (result.need_more_input()) {
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes));
@ -416,14 +409,14 @@ struct parser_executor {
}
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes));
}
common_peg_parse_result operator()(const common_peg_choice_parser & p) {
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE start at %zu '%s' (%zu options)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.children.size());
}
@ -432,17 +425,17 @@ struct parser_executor {
auto pos = start_pos;
for (size_t i = 0; i < p.children.size(); i++) {
const auto & child_id = p.children[i];
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i, arena.dump(child_id).c_str());
}
auto result = arena.parse(child_id, ctx, pos);
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE option %zu: %s\n", debug_indent().c_str(), i,
common_peg_parse_result_type_name(result.type));
}
if (!result.fail()) {
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE -> %s (option %zu)\n", debug_indent().c_str(),
common_peg_parse_result_type_name(result.type), i);
}
@ -451,14 +444,14 @@ struct parser_executor {
}
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sCHOICE -> FAIL (no options matched)\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
common_peg_parse_result operator()(const common_peg_repetition_parser & p) {
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT start at %zu '%s' (min=%d, max=%d)\n", debug_indent().c_str(), start_pos,
debug_input_snippet(start_pos).c_str(), p.min_count, p.max_count);
}
@ -471,7 +464,7 @@ struct parser_executor {
// Try to match up to max_count times (or unlimited if max_count is -1)
while (p.max_count == -1 || match_count < p.max_count) {
if (pos >= ctx.input.size()) {
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT: at end of input, count=%d\n", debug_indent().c_str(), match_count);
}
break;
@ -479,7 +472,7 @@ struct parser_executor {
auto result = arena.parse(p.child, ctx, pos);
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT iter %d: %s at %zu->%zu, nodes=%zu\n", debug_indent().c_str(), match_count,
common_peg_parse_result_type_name(result.type), result.start, result.end, result.nodes.size());
fprintf(stderr, "%sREPEAT CHILD: %s\n", debug_indent().c_str(), arena.dump(p.child).c_str());
@ -488,7 +481,7 @@ struct parser_executor {
if (result.success()) {
// Prevent infinite loop on empty matches
if (result.end == pos) {
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%s REPEAT: empty match, stopping\n", debug_indent().c_str());
}
break;
@ -509,7 +502,7 @@ struct parser_executor {
}
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(),
match_count, nodes.size());
}
@ -517,7 +510,7 @@ struct parser_executor {
}
// Child failed - stop trying
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT: child failed, stopping\n", debug_indent().c_str());
}
break;
@ -526,14 +519,14 @@ struct parser_executor {
// Check if we got enough matches
if (p.min_count > 0 && match_count < p.min_count) {
ctx.parse_depth--;
if (pos >= ctx.input.size() && ctx.is_partial) {
if (ctx.debug) {
if (pos >= ctx.input.size() && ctx.is_lenient()) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(),
match_count, p.min_count);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes));
}
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count,
p.min_count);
}
@ -541,7 +534,7 @@ struct parser_executor {
}
ctx.parse_depth--;
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count,
nodes.size());
}
@ -576,7 +569,7 @@ struct parser_executor {
auto result = common_parse_utf8_codepoint(ctx.input, start_pos);
if (result.status == utf8_parse_result::INCOMPLETE) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos);
@ -615,7 +608,7 @@ struct parser_executor {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
}
// Not enough matches yet
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -656,7 +649,7 @@ struct parser_executor {
// Check if we got enough matches
if (match_count < p.min_count) {
if (pos >= ctx.input.size() && ctx.is_partial) {
if (pos >= ctx.input.size() && ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
@ -668,7 +661,7 @@ struct parser_executor {
static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos) {
++pos; // consume '\'
if (pos >= ctx.input.size()) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
@ -698,7 +691,7 @@ struct parser_executor {
++pos; // consume 'u'
for (int i = 0; i < 4; ++i) {
if (pos >= ctx.input.size()) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);
@ -732,7 +725,7 @@ struct parser_executor {
auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);
if (utf8_result.status == utf8_parse_result::INCOMPLETE) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -747,7 +740,7 @@ struct parser_executor {
}
// Reached end without finding closing quote
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -774,7 +767,7 @@ struct parser_executor {
auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);
if (utf8_result.status == utf8_parse_result::INCOMPLETE) {
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -789,7 +782,7 @@ struct parser_executor {
}
// Reached end without finding closing quote
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos);
@ -807,7 +800,7 @@ struct parser_executor {
if (utf8_result.status == utf8_parse_result::INCOMPLETE) {
// Incomplete UTF-8 sequence
if (!ctx.is_partial) {
if (!ctx.is_lenient()) {
// Input is complete but UTF-8 is incomplete = malformed
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
@ -837,7 +830,7 @@ struct parser_executor {
last_valid_pos = pos;
}
if (last_valid_pos == ctx.input.size() && ctx.is_partial) {
if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) {
// Reached the end of a partial stream, there might still be more input that we need to consume.
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos);
}
@ -876,7 +869,7 @@ struct parser_executor {
common_peg_parse_result operator()(const common_peg_tag_parser & p) {
// Parse the child
if (ctx.debug) {
if (ctx.is_debug()) {
fprintf(stderr, "%sTAG: %s\n", debug_indent().c_str(), p.tag.c_str());
}
auto result = arena.parse(p.child, ctx, start_pos);

37
common/peg-parser.h
View File

@ -139,22 +139,43 @@ struct common_peg_parse_result {
bool success() const { return type == COMMON_PEG_PARSE_RESULT_SUCCESS; }
};
enum common_peg_parse_flags {
COMMON_PEG_PARSE_FLAG_NONE = 0,
COMMON_PEG_PARSE_FLAG_LENIENT = 1 << 0,
COMMON_PEG_PARSE_FLAG_DEBUG = 1 << 1,
};
inline common_peg_parse_flags operator|(common_peg_parse_flags a, common_peg_parse_flags b) {
return static_cast<common_peg_parse_flags>(int(a) | int(b));
}
inline common_peg_parse_flags & operator|=(common_peg_parse_flags & a, common_peg_parse_flags b) {
return a = a | b;
}
inline common_peg_parse_flags operator&(common_peg_parse_flags a, common_peg_parse_flags b) {
return static_cast<common_peg_parse_flags>(int(a) & int(b));
}
inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
return static_cast<common_peg_parse_flags>(~int(a));
}
struct common_peg_parse_context {
std::string input;
bool is_partial;
bool debug = false; // Enable debug output for parser tracing
common_peg_parse_flags flags;
common_peg_ast_arena ast;
int parse_depth;
common_peg_parse_context()
: is_partial(false), parse_depth(0) {}
common_peg_parse_context(common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
: flags(flags), parse_depth(0) {}
common_peg_parse_context(const std::string & input)
: input(input), is_partial(false), parse_depth(0) {}
common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
: input(input), flags(flags), parse_depth(0) {}
common_peg_parse_context(const std::string & input, bool is_partial)
: input(input), is_partial(is_partial), parse_depth(0) {}
bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
};
class common_peg_arena;

54
tests/peg-parser/test-basic.cpp
View File

@ -120,7 +120,7 @@ void test_basic(testing & t) {
return p.literal("hello") + p.optional(p.literal(" world"));
});
auto ctx = common_peg_parse_context("hello", false);
auto ctx = common_peg_parse_context("hello");
auto result = parser.parse(ctx);
t.assert_equal("optional_absent", true, result.success());
t.assert_equal("optional_absent_end", 5u, result.end);
@ -132,7 +132,7 @@ void test_basic(testing & t) {
return p.literal("hello") + p.optional(p.literal(" world"));
});
auto ctx = common_peg_parse_context("hello ", true);
auto ctx = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("partial_match_need_more", true, result.need_more_input());
});
@ -215,7 +215,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<thi", true);
auto ctx = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_1", true, result.need_more_input());
});
@ -224,7 +224,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });
auto ctx = common_peg_parse_context("begin", true);
auto ctx = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_2", true, result.need_more_input());
});
@ -233,7 +233,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_3", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think></", true);
auto ctx = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_3", true, result.need_more_input());
});
@ -242,7 +242,7 @@ void test_basic(testing & t) {
t.test("sequence_full_match", [&](testing & t) {
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });
auto ctx = common_peg_parse_context("helloworld", false);
auto ctx = common_peg_parse_context("helloworld");
auto result = common_chat_combinator_parser.parse(ctx);
t.assert_equal("sequence_full_match", true, result.success());
});
@ -251,7 +251,7 @@ void test_basic(testing & t) {
t.test("sequence_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", true);
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_no_match", true, result.fail());
});
@ -260,7 +260,7 @@ void test_basic(testing & t) {
t.test("choices_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });
auto ctx = common_peg_parse_context("opt", true);
auto ctx = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("choices_partial_match_1", true, result.need_more_input());
});
@ -270,7 +270,7 @@ void test_basic(testing & t) {
auto parser =
build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });
auto ctx = common_peg_parse_context("choice", true);
auto ctx = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("choices_partial_match_2", true, result.need_more_input());
});
@ -279,7 +279,7 @@ void test_basic(testing & t) {
t.test("choices_full_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });
auto ctx = common_peg_parse_context("first", false);
auto ctx = common_peg_parse_context("first");
auto result = parser.parse(ctx);
t.assert_equal("choices_full_match_1", true, result.success());
});
@ -288,7 +288,7 @@ void test_basic(testing & t) {
t.test("choices_full_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });
auto ctx = common_peg_parse_context("beta", false);
auto ctx = common_peg_parse_context("beta");
auto result = parser.parse(ctx);
t.assert_equal("choices_full_match_2", true, result.success());
});
@ -297,7 +297,7 @@ void test_basic(testing & t) {
t.test("choices_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });
auto ctx = common_peg_parse_context("best", false);
auto ctx = common_peg_parse_context("best");
auto result = parser.parse(ctx);
t.assert_equal("choices_no_match", true, result.fail());
});
@ -306,7 +306,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("a", true);
auto ctx = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input());
});
@ -315,7 +315,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });
auto ctx = common_peg_parse_context("xyx", true);
auto ctx = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input());
});
@ -324,7 +324,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_full_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });
auto ctx = common_peg_parse_context("test", false);
auto ctx = common_peg_parse_context("test");
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_full_match", true, result.success());
});
@ -333,7 +333,7 @@ void test_basic(testing & t) {
t.test("one_or_more_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });
auto ctx = common_peg_parse_context("rep", true);
auto ctx = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input());
});
@ -342,7 +342,7 @@ void test_basic(testing & t) {
t.test("one_or_more_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("aba", true);
auto ctx = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input());
});
@ -351,7 +351,7 @@ void test_basic(testing & t) {
t.test("one_or_more_full_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });
auto ctx = common_peg_parse_context("single", false);
auto ctx = common_peg_parse_context("single");
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_full_match", true, result.success());
});
@ -360,7 +360,7 @@ void test_basic(testing & t) {
t.test("one_or_more_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });
auto ctx = common_peg_parse_context("success", false);
auto ctx = common_peg_parse_context("success");
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_no_match", true, result.fail());
});
@ -376,7 +376,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("1", false);
common_peg_parse_context ctx("1");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@ -390,7 +390,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[1]", false);
common_peg_parse_context ctx("[1]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@ -404,7 +404,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[2]]", false);
common_peg_parse_context ctx("[[2]]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@ -418,7 +418,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[[3]]]", false);
common_peg_parse_context ctx("[[[3]]]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@ -432,7 +432,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[", true);
common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
@ -446,7 +446,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[a]", false);
common_peg_parse_context ctx("[a]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_fail", true, result.fail());
@ -458,8 +458,8 @@ void test_basic(testing & t) {
return p.marker();
});
common_peg_parse_context ctx_square("[marker]", false);
common_peg_parse_context ctx_sharp("<marker>", false);
common_peg_parse_context ctx_square("[marker]");
common_peg_parse_context ctx_sharp("<marker>");
auto result_square = bracket_parser.parse(ctx_square);
auto result_sharp = bracket_parser.parse(ctx_sharp);

12
tests/peg-parser/test-json-parser.cpp
View File

@ -46,7 +46,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"({"name": "test", "value": )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@ -58,7 +58,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"([1, 2, 3, )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@ -70,7 +70,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"({"data": {"nested": )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@ -84,7 +84,7 @@ void test_json_parser(testing &t) {
t.test("success", [&](testing &t) {
std::string input = R"("name": "bob")";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@ -92,7 +92,7 @@ void test_json_parser(testing &t) {
t.test("partial", [&](testing &t) {
std::string input = R"("name": "bo)";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_true("need more input", result.need_more_input());
@ -100,7 +100,7 @@ void test_json_parser(testing &t) {
t.test("failed", [&](testing &t) {
std::string input = R"([])";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("fail", result.fail());

6
tests/peg-parser/test-python-dict-parser.cpp
View File

@ -85,7 +85,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test', 'value': ";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@ -97,7 +97,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@ -229,7 +229,7 @@ void test_python_dict_parser(testing &t) {
t.test("incomplete string", [&](testing &t) {
std::string input = "'hello";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_true("need_more_input", result.need_more_input());

25
tests/peg-parser/test-unicode.cpp
View File

@ -58,7 +58,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@ -101,7 +101,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@ -142,7 +142,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@ -187,7 +187,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@ -225,7 +225,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -259,7 +259,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -293,7 +293,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -330,7 +330,7 @@ void test_unicode(testing &t) {
return p.sequence({p.json_string_content(), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -367,7 +367,7 @@ void test_unicode(testing &t) {
return p.json_string_content();
});
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -390,9 +390,6 @@ void test_unicode(testing &t) {
// Invalid continuation byte
{std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
// Overlong encoding (security issue)
{std::string("\xC0\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL},
};
for (size_t i = 0; i < test_cases.size(); i++) {
@ -404,7 +401,7 @@ void test_unicode(testing &t) {
return p.json_string_content();
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@ -433,7 +430,7 @@ void test_unicode(testing &t) {
return p.sequence({p.json_string_content(), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);

8
tests/test-chat-auto-parser.cpp
View File

@ -1478,7 +1478,7 @@ static void test_standard_json_tools_openai(testing & t) {
R"({"id": "call_abc123", "function": {"name": "get_current_weather", "arguments": {"location": "NYC"}}})"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@ -1524,7 +1524,7 @@ static void test_standard_json_tools_cohere(testing & t) {
R"({"tool_call_id": 0, "tool_name": "get_current_weather", "parameters": {"location": "NYC", "unit": "celsius"}})"
"]<|END_ACTION|>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@ -1570,7 +1570,7 @@ static void test_standard_json_tools_function_key(testing & t) {
R"({"get_current_weather": {"id": "call-0001", "args": {"location": "NYC", "unit": "celsius"}}})"
"]</tool_calls>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@ -1845,7 +1845,7 @@ static void test_tagged_args_with_embedded_quotes(testing & t) {
"</function>\n"
"</seed:tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {

18
tests/test-chat-peg-parser.cpp
View File

@ -361,7 +361,7 @@ static void test_example_native(testing & t) {
t.log(line);
}
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@ -458,7 +458,7 @@ static void test_example_qwen3_coder(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@ -523,7 +523,7 @@ static void test_example_qwen3_non_coder(testing & t) {
"\"fahrenheit\"}}"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@ -556,7 +556,7 @@ static void test_example_qwen3_non_coder(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@ -617,7 +617,7 @@ void test_command7_parser_compare(testing & t) {
auto test_current = [&](const common_peg_arena & p, const std::string & input, bool is_partial,
bool print_results) {
common_peg_parse_context ctx(input, is_partial);
common_peg_parse_context ctx(input, is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = p.parse(ctx);
common_chat_msg msg;
@ -780,7 +780,7 @@ static void test_prefix_tool_names(testing & t) {
"</function>"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@ -814,7 +814,7 @@ static void test_prefix_tool_names(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@ -864,7 +864,7 @@ static void test_prefix_tool_names(testing & t) {
"</function>"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@ -931,7 +931,7 @@ static void test_tagged_peg_parser(testing & t) {
return p.tag("prefix", p.until(":")) + ":" + p.tag("value", p.rest()) + p.end();
});
auto result = parser.parse_and_extract("key:val", true);
auto result = parser.parse_and_extract("key:val", COMMON_PEG_PARSE_FLAG_LENIENT);
t.assert_true("not fail", !result.result.fail());
t.assert_equal("prefix tag", "key", result.tags.at("prefix"));
t.assert_equal("value tag", "val", result.tags.at("value"));