diff --git a/common/jinja/jinja-lexer.cpp b/common/jinja/jinja-lexer.cpp index a5ce7af9e1..541452f3fe 100644 --- a/common/jinja/jinja-lexer.cpp +++ b/common/jinja/jinja-lexer.cpp @@ -54,12 +54,13 @@ std::string lexer::preprocess(const std::string & template_str, const preprocess return result; } -std::vector lexer::tokenize(const std::string & input, const preprocess_options & options) { +lexer_result lexer::tokenize(const std::string & input, const preprocess_options & options) { std::vector tokens; std::string src = preprocess(input, options); JJ_DEBUG("preprocessed input: '%s'", src.c_str()); size_t pos = 0; + size_t start_pos = 0; size_t curly_bracket_depth = 0; using pred = std::function; @@ -101,6 +102,7 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o }; while (pos < src.size()) { + start_pos = pos; JJ_DEBUG("lexer main loop at pos %zu: '%s...'", pos, src.substr(pos, 10).c_str()); // First, consume all text that is outside of a Jinja statement or expression @@ -122,13 +124,14 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o } JJ_DEBUG("consumed text: '%s'", text.c_str()); if (!text.empty()) { - tokens.push_back({token::text, text}); + tokens.push_back({token::text, text, start_pos}); continue; } } // Possibly consume a comment if (src[pos] == '{' && next_pos_is( {'#'} )) { + start_pos = pos; pos += 2; // Skip the opening {# std::string comment; while (!(src[pos] == '#' && next_pos_is( {'}'} ))) { @@ -138,7 +141,7 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o comment += src[pos++]; } JJ_DEBUG("consumed comment: '%s'", comment.c_str()); - tokens.push_back({token::comment, comment}); + tokens.push_back({token::comment, comment, start_pos}); pos += 2; // Skip the closing #} continue; } @@ -152,6 +155,7 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o // Check for unary operators if (ch == '-' || ch == '+') { + start_pos = pos; token::type last_token_type = tokens.empty() ? token::undefined : tokens.back().t; if (last_token_type == token::text || last_token_type == token::undefined) { throw std::runtime_error(std::string("lexer: unexpected character: ") + ch); @@ -176,7 +180,7 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o std::string value = std::string(1, ch) + num; token::type t = num.empty() ? token::unary_operator : token::numeric_literal; JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str()); - tokens.push_back({t, value}); + tokens.push_back({t, value, start_pos}); continue; } } @@ -185,12 +189,13 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o // Try to match one of the tokens in the mapping table bool matched = false; for (const auto & [seq, typ] : ordered_mapping_table) { + start_pos = pos; // Inside an object literal, don't treat "}}" as expression-end if (seq == "}}" && curly_bracket_depth > 0) { continue; } if (pos + seq.size() <= src.size() && src.substr(pos, seq.size()) == seq) { - tokens.push_back({typ, seq}); + tokens.push_back({typ, seq, start_pos}); if (typ == token::open_expression) { curly_bracket_depth = 0; } else if (typ == token::open_curly_bracket) { @@ -207,36 +212,39 @@ std::vector lexer::tokenize(const std::string & input, const preprocess_o // Strings if (ch == '\'' || ch == '"') { + start_pos = pos; ++pos; // Skip opening quote std::string str = consume_while([ch](char c) { return c != ch; }); - tokens.push_back({token::string_literal, str}); + tokens.push_back({token::string_literal, str, start_pos}); ++pos; // Skip closing quote continue; } // Numbers if (is_integer(ch)) { + start_pos = pos; std::string num = consume_while(is_integer); if (pos < src.size() && src[pos] == '.' && pos + 1 < src.size() && is_integer(src[pos + 1])) { ++pos; // Consume '.' std::string frac = consume_while(is_integer); num += "." + frac; } - tokens.push_back({token::numeric_literal, num}); + tokens.push_back({token::numeric_literal, num, start_pos}); continue; } // Identifiers if (is_word(ch)) { + start_pos = pos; std::string word = consume_while(is_word); - tokens.push_back({token::identifier, word}); + tokens.push_back({token::identifier, word, start_pos}); continue; } throw std::runtime_error(std::string("lexer: unexpected character: ") + ch); } - return tokens; + return {std::move(tokens), std::move(src)}; } } // namespace jinja diff --git a/common/jinja/jinja-lexer.h b/common/jinja/jinja-lexer.h index 3ed173a4f0..f9bbe0a991 100644 --- a/common/jinja/jinja-lexer.h +++ b/common/jinja/jinja-lexer.h @@ -48,6 +48,7 @@ struct token { }; type t; std::string value; + size_t pos; }; static std::string type_to_string(token::type t) { @@ -82,6 +83,11 @@ static std::string type_to_string(token::type t) { } } +struct lexer_result { + std::vector tokens; + std::string preprocessed_source; +}; + struct lexer { const std::map escape_chars = { {'n', '\n'}, @@ -140,7 +146,7 @@ struct lexer { std::string preprocess(const std::string& template_str, const preprocess_options& options) const; - std::vector tokenize(const std::string & input, const preprocess_options & options); + lexer_result tokenize(const std::string & input, const preprocess_options & options); }; } // namespace jinja diff --git a/common/jinja/jinja-parser.cpp b/common/jinja/jinja-parser.cpp index c375d545ef..5f42b0bd89 100644 --- a/common/jinja/jinja-parser.cpp +++ b/common/jinja/jinja-parser.cpp @@ -8,6 +8,8 @@ #include #include +#define FILENAME "jinja-parser" + namespace jinja { // Helper to check type without asserting (useful for logic) @@ -19,9 +21,18 @@ static bool is_type(const statement_ptr & ptr) { class parser { const std::vector & tokens; size_t current = 0; + size_t prev_cur = 0; + + // for debugging; a token can be multiple chars in source + std::vector tok_pos_to_src_pos; public: - parser(const std::vector & t) : tokens(t) {} + parser(const std::vector & t) : tokens(t) { + tok_pos_to_src_pos.resize(tokens.size()); + for (size_t i = 0; i < tokens.size(); i++) { + tok_pos_to_src_pos[i] = tokens[i].pos; + } + } program parse() { statements body; @@ -31,10 +42,18 @@ public: return program(std::move(body)); } + template + std::unique_ptr mk_stmt(Args&&... args) { + auto ptr = std::make_unique(std::forward(args)...); + ptr->pos = tok_pos_to_src_pos[prev_cur]; + JJ_DEBUG("Created %s statement at src pos %zu", ptr->type().c_str(), ptr->pos); + return ptr; + } + private: const token & peek(size_t offset = 0) const { if (current + offset >= tokens.size()) { - static const token end_token{token::undefined, ""}; + static const token end_token{token::undefined, "", 0}; return end_token; } return tokens[current + offset]; @@ -74,6 +93,7 @@ private: } statement_ptr parse_any() { + prev_cur = current; switch (peek().t) { case token::comment: return mk_stmt(tokens[current++].value); @@ -90,6 +110,7 @@ private: statement_ptr parse_jinja_expression() { // Consume {{ }} tokens + prev_cur = current; expect(token::open_expression, "Expected {{"); auto result = parse_expression(); expect(token::close_expression, "Expected }}"); @@ -98,6 +119,7 @@ private: statement_ptr parse_jinja_statement() { // Consume {% token + prev_cur = current; expect(token::open_statement, "Expected {%"); if (peek().t != token::identifier) { @@ -194,6 +216,8 @@ private: auto left = parse_expression_sequence(); statement_ptr value = nullptr; statements body; + + prev_cur = current; if (is(token::equals)) { current++; @@ -218,6 +242,8 @@ private: statements body; statements alternate; + prev_cur = current; + // Keep parsing 'if' body until we reach the first {% elif %} or {% else %} or {% endif %} while (!is_statement({"elif", "else", "endif"})) { body.push_back(parse_any()); @@ -257,6 +283,7 @@ private: exprs.push_back(primary ? parse_primary_expression() : parse_expression()); bool is_tuple = is(token::comma); while (is(token::comma)) { + prev_cur = current; current++; // consume comma exprs.push_back(primary ? parse_primary_expression() : parse_expression()); if (!is(token::comma)) break; @@ -283,6 +310,7 @@ private: } if (is_statement({"else"})) { + prev_cur = current; current += 2; expect(token::close_statement, "Expected %}"); while (!is_statement({"endfor"})) { @@ -303,10 +331,12 @@ private: auto a = parse_logical_or_expression(); if (is_identifier("if")) { // Ternary expression + prev_cur = current; ++current; // consume 'if' auto test = parse_logical_or_expression(); if (is_identifier("else")) { // Ternary expression with else + prev_cur = current; ++current; // consume 'else' auto false_expr = parse_if_expression(); // recurse to support chained ternaries return mk_stmt(std::move(test), std::move(a), std::move(false_expr)); @@ -321,6 +351,7 @@ private: statement_ptr parse_logical_or_expression() { auto left = parse_logical_and_expression(); while (is_identifier("or")) { + prev_cur = current; token op = tokens[current++]; left = mk_stmt(op, std::move(left), parse_logical_and_expression()); } @@ -330,6 +361,7 @@ private: statement_ptr parse_logical_and_expression() { auto left = parse_logical_negation_expression(); while (is_identifier("and")) { + prev_cur = current; auto op = tokens[current++]; left = mk_stmt(op, std::move(left), parse_logical_negation_expression()); } @@ -339,6 +371,7 @@ private: statement_ptr parse_logical_negation_expression() { // Try parse unary operators if (is_identifier("not")) { + prev_cur = current; auto op = tokens[current]; ++current; // consume 'not' return mk_stmt(op, parse_logical_negation_expression()); @@ -352,8 +385,9 @@ private: auto left = parse_additive_expression(); while (true) { token op; + prev_cur = current; if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") { - op = {token::identifier, "not in"}; + op = {token::identifier, "not in", tokens[current].pos}; current += 2; } else if (is_identifier("in")) { op = tokens[current++]; @@ -368,6 +402,7 @@ private: statement_ptr parse_additive_expression() { auto left = parse_multiplicative_expression(); while (is(token::additive_binary_operator)) { + prev_cur = current; auto op = tokens[current++]; left = mk_stmt(op, std::move(left), parse_multiplicative_expression()); } @@ -377,6 +412,7 @@ private: statement_ptr parse_multiplicative_expression() { auto left = parse_test_expression(); while (is(token::multiplicative_binary_operator)) { + prev_cur = current; auto op = tokens[current++]; left = mk_stmt(op, std::move(left), parse_test_expression()); } @@ -386,6 +422,7 @@ private: statement_ptr parse_test_expression() { auto operand = parse_filter_expression(); while (is_identifier("is")) { + prev_cur = current; current++; bool negate = false; if (is_identifier("not")) { current++; negate = true; } @@ -398,6 +435,7 @@ private: statement_ptr parse_filter_expression() { auto operand = parse_call_member_expression(); while (is(token::pipe)) { + prev_cur = current; current++; auto filter = parse_primary_expression(); if (is(token::open_paren)) filter = parse_call_expression(std::move(filter)); @@ -428,6 +466,7 @@ private: statements args; while (!is(token::close_paren)) { statement_ptr arg; + prev_cur = current; // unpacking: *expr if (peek().t == token::multiplicative_binary_operator && peek().value == "*") { ++current; // consume * @@ -472,6 +511,7 @@ private: statements slices; bool is_slice = false; while (!is(token::close_square_bracket)) { + prev_cur = current; if (is(token::colon)) { // A case where a default is used // e.g., [:2] will be parsed as [undefined, 2] @@ -496,6 +536,7 @@ private: } statement_ptr parse_primary_expression() { + prev_cur = current; auto t = tokens[current++]; switch (t.t) { case token::numeric_literal: diff --git a/common/jinja/jinja-value.h b/common/jinja/jinja-value.h index 6c6f4a30d6..94c638eab2 100644 --- a/common/jinja/jinja-value.h +++ b/common/jinja/jinja-value.h @@ -164,6 +164,9 @@ struct value_string_t : public value_t { } return ss.str(); } + virtual bool as_bool() const override { + return val_str.length() > 0; + } virtual const func_builtins & get_builtins() const override; void mark_input() { val_str.mark_input(); diff --git a/common/jinja/jinja-vm-builtins.cpp b/common/jinja/jinja-vm-builtins.cpp index ed601eb9b1..5802253a3e 100644 --- a/common/jinja/jinja-vm-builtins.cpp +++ b/common/jinja/jinja-vm-builtins.cpp @@ -173,25 +173,6 @@ const func_builtins & value_float_t::get_builtins() const { return builtins; } - -// static std::string string_strip(const std::string & str, bool left, bool right) { -// size_t start = 0; -// size_t end = str.length(); -// if (left) { -// while (start < end && isspace(static_cast(str[start]))) { -// ++start; -// } -// } -// if (right) { -// while (end > start && isspace(static_cast(str[end - 1]))) { -// --end; -// } -// } -// return str.substr(start, end - start); -// } - - - static bool string_startswith(const std::string & str, const std::string & prefix) { if (str.length() < prefix.length()) return false; return str.compare(0, prefix.length(), prefix) == 0; diff --git a/common/jinja/jinja-vm.cpp b/common/jinja/jinja-vm.cpp index fea7c75f06..ca213b0462 100644 --- a/common/jinja/jinja-vm.cpp +++ b/common/jinja/jinja-vm.cpp @@ -8,8 +8,9 @@ #include #include -#define JJ_DEBUG(msg, ...) printf("jinja-vm:%3d : " msg "\n", __LINE__, __VA_ARGS__) -//#define JJ_DEBUG(msg, ...) // no-op +#define FILENAME "jinja-vm" + +bool g_jinja_debug = true; namespace jinja { @@ -22,7 +23,51 @@ static value_array exec_statements(const statements & stmts, context & ctx) { return result; } -value identifier::execute(context & ctx) { +static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) { + if (search.empty()) { + return; + } + std::string builder; + builder.reserve(s.length()); + size_t pos = 0; + size_t last_pos = 0; + while ((pos = s.find(search, last_pos)) != std::string::npos) { + builder.append(s, last_pos, pos - last_pos); + builder.append(replace); + last_pos = pos + search.length(); + } + builder.append(s, last_pos, std::string::npos); + s = std::move(builder); +} + +// execute with error handling +value statement::execute(context & ctx) { + try { + return execute_impl(ctx); + } catch (const std::exception & e) { + if (ctx.source.empty()) { + std::ostringstream oss; + oss << "\nError executing " << type() << " at position " << pos << ": " << e.what(); + throw raised_exception(oss.str()); + } else { + std::ostringstream oss; + constexpr int max_peak_chars = 40; + oss << "\n------------\n"; + oss << "While executing " << type() << " at position " << pos << " in source:\n"; + size_t start = (pos >= max_peak_chars) ? (pos - max_peak_chars) : 0; + size_t end = std::min(pos + max_peak_chars, ctx.source.length()); + std::string substr = ctx.source.substr(start, end - start); + string_replace_all(substr, "\n", "\\n"); + oss << "..." << substr << "...\n"; + std::string spaces(pos - start + 3, ' '); + oss << spaces << "^\n"; + oss << "Error: " << e.what(); + throw raised_exception(oss.str()); + } + } +} + +value identifier::execute_impl(context & ctx) { auto it = ctx.var.find(val); auto builtins = global_builtins(); if (it != ctx.var.end()) { @@ -37,7 +82,7 @@ value identifier::execute(context & ctx) { } } -value binary_expression::execute(context & ctx) { +value binary_expression::execute_impl(context & ctx) { value left_val = left->execute(ctx); JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right->type().c_str()); @@ -176,7 +221,7 @@ static value try_builtin_func(const std::string & name, const value & input, boo throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type()); } -value filter_expression::execute(context & ctx) { +value filter_expression::execute_impl(context & ctx) { value input = operand->execute(ctx); if (is_stmt(filter)) { @@ -203,7 +248,7 @@ value filter_expression::execute(context & ctx) { } } -value test_expression::execute(context & ctx) { +value test_expression::execute_impl(context & ctx) { // NOTE: "value is something" translates to function call "test_is_something(value)" const auto & builtins = global_builtins(); if (!is_stmt(test)) { @@ -222,7 +267,7 @@ value test_expression::execute(context & ctx) { return it->second(args); } -value unary_expression::execute(context & ctx) { +value unary_expression::execute_impl(context & ctx) { value operand_val = argument->execute(ctx); JJ_DEBUG("Executing unary expression with operator '%s'", op.value.c_str()); @@ -241,7 +286,7 @@ value unary_expression::execute(context & ctx) { throw std::runtime_error("Unknown unary operator '" + op.value + "'"); } -value if_statement::execute(context & ctx) { +value if_statement::execute_impl(context & ctx) { value test_val = test->execute(ctx); auto out = mk_val(); if (test_val->as_bool()) { @@ -258,7 +303,7 @@ value if_statement::execute(context & ctx) { return out; } -value for_statement::execute(context & ctx) { +value for_statement::execute_impl(context & ctx) { context scope(ctx); // new scope for loop variables statement_ptr iter_expr = std::move(iterable); @@ -377,7 +422,7 @@ value for_statement::execute(context & ctx) { return result; } -value set_statement::execute(context & ctx) { +value set_statement::execute_impl(context & ctx) { auto rhs = val ? val->execute(ctx) : exec_statements(body, ctx); if (is_stmt(assignee)) { @@ -427,7 +472,7 @@ value set_statement::execute(context & ctx) { return mk_val(); } -value macro_statement::execute(context & ctx) { +value macro_statement::execute_impl(context & ctx) { std::string name = cast_stmt(this->name)->val; const func_handler func = [this, &ctx, name](const func_args & args) -> value { JJ_DEBUG("Invoking macro '%s' with %zu arguments", name.c_str(), args.args.size()); @@ -454,7 +499,7 @@ value macro_statement::execute(context & ctx) { return mk_val(); } -value member_expression::execute(context & ctx) { +value member_expression::execute_impl(context & ctx) { value object = this->object->execute(ctx); value property; @@ -536,7 +581,7 @@ value member_expression::execute(context & ctx) { return val; } -value call_expression::execute(context & ctx) { +value call_expression::execute_impl(context & ctx) { // gather arguments func_args args; for (auto & arg_stmt : this->args) { @@ -587,7 +632,7 @@ bool value_compare(const value & a, const value & b) { return false; } -value keyword_argument_expression::execute(context & ctx) { +value keyword_argument_expression::execute_impl(context & ctx) { if (!is_stmt(key)) { throw std::runtime_error("Keyword argument key must be identifiers"); } diff --git a/common/jinja/jinja-vm.h b/common/jinja/jinja-vm.h index 165bfafd96..639fba9d03 100644 --- a/common/jinja/jinja-vm.h +++ b/common/jinja/jinja-vm.h @@ -9,6 +9,9 @@ #include #include +#define JJ_DEBUG(msg, ...) if (g_jinja_debug) printf("%s:%3d : " msg "\n", FILENAME, __LINE__, __VA_ARGS__) + +extern bool g_jinja_debug; namespace jinja { @@ -37,14 +40,11 @@ template const T * cast_stmt(const statement_ptr & ptr) { return dynamic_cast(ptr.get()); } -template -std::unique_ptr mk_stmt(Args&&... args) { - return std::make_unique(std::forward(args)...); -} // End Helpers struct context { std::map var; + std::string source; // for debugging context() { var["true"] = mk_val(true); @@ -65,9 +65,13 @@ struct context { * Base class for all nodes in the AST. */ struct statement { + size_t pos; // position in source, for debugging virtual ~statement() = default; virtual std::string type() const { return "Statement"; } - virtual value execute(context &) { throw std::runtime_error("cannot exec " + type()); } + // execute_impl must be overridden by derived classes + virtual value execute_impl(context &) { throw std::runtime_error("cannot exec " + type()); } + // execute is the public method to execute a statement with error handling + virtual value execute(context &); }; // Type Checking Utilities @@ -100,7 +104,7 @@ struct program : public statement { explicit program(statements && body) : body(std::move(body)) {} std::string type() const override { return "Program"; } - value execute(context &) override { + value execute_impl(context &) override { throw std::runtime_error("Cannot execute program directly, use jinja::vm instead"); } }; @@ -116,7 +120,7 @@ struct if_statement : public statement { } std::string type() const override { return "If"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct identifier; @@ -140,7 +144,7 @@ struct for_statement : public statement { } std::string type() const override { return "For"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct break_statement : public statement { @@ -152,7 +156,7 @@ struct break_statement : public statement { } }; - value execute(context &) override { + value execute_impl(context &) override { throw break_statement::exception(); } }; @@ -166,7 +170,7 @@ struct continue_statement : public statement { } }; - value execute(context &) override { + value execute_impl(context &) override { throw continue_statement::exception(); } }; @@ -183,7 +187,7 @@ struct set_statement : public statement { } std::string type() const override { return "Set"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct macro_statement : public statement { @@ -198,14 +202,14 @@ struct macro_statement : public statement { } std::string type() const override { return "Macro"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct comment_statement : public statement { std::string val; explicit comment_statement(const std::string & v) : val(v) {} std::string type() const override { return "Comment"; } - value execute(context &) override { + value execute_impl(context &) override { return mk_val(); } }; @@ -223,7 +227,7 @@ struct member_expression : public expression { chk_type(this->property); } std::string type() const override { return "MemberExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct call_expression : public expression { @@ -236,7 +240,7 @@ struct call_expression : public expression { for (const auto& arg : this->args) chk_type(arg); } std::string type() const override { return "CallExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; /** @@ -246,7 +250,7 @@ struct identifier : public expression { std::string val; explicit identifier(const std::string & val) : val(val) {} std::string type() const override { return "Identifier"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; // Literals @@ -255,7 +259,7 @@ struct integer_literal : public expression { int64_t val; explicit integer_literal(int64_t val) : val(val) {} std::string type() const override { return "IntegerLiteral"; } - value execute(context &) override { + value execute_impl(context &) override { return std::make_unique(val); } }; @@ -264,7 +268,7 @@ struct float_literal : public expression { double val; explicit float_literal(double val) : val(val) {} std::string type() const override { return "FloatLiteral"; } - value execute(context &) override { + value execute_impl(context &) override { return std::make_unique(val); } }; @@ -273,7 +277,7 @@ struct string_literal : public expression { std::string val; explicit string_literal(const std::string & val) : val(val) {} std::string type() const override { return "StringLiteral"; } - value execute(context &) override { + value execute_impl(context &) override { return std::make_unique(val); } }; @@ -324,7 +328,7 @@ struct binary_expression : public expression { chk_type(this->right); } std::string type() const override { return "BinaryExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; /** @@ -341,7 +345,7 @@ struct filter_expression : public expression { chk_type(this->filter); } std::string type() const override { return "FilterExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct filter_statement : public statement { @@ -388,7 +392,7 @@ struct test_expression : public expression { chk_type(this->test); } std::string type() const override { return "TestExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; /** @@ -403,7 +407,7 @@ struct unary_expression : public expression { chk_type(this->argument); } std::string type() const override { return "UnaryExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct slice_expression : public expression { @@ -418,7 +422,7 @@ struct slice_expression : public expression { chk_type(this->step_expr); } std::string type() const override { return "SliceExpression"; } - value execute(context &) override { + value execute_impl(context &) override { throw std::runtime_error("must be handled by MemberExpression"); } }; @@ -433,7 +437,7 @@ struct keyword_argument_expression : public expression { chk_type(this->val); } std::string type() const override { return "KeywordArgumentExpression"; } - value execute(context & ctx) override; + value execute_impl(context & ctx) override; }; struct spread_expression : public expression { diff --git a/tests/test-chat-jinja.cpp b/tests/test-chat-jinja.cpp index eff9831ff4..36cfde7c5f 100644 --- a/tests/test-chat-jinja.cpp +++ b/tests/test-chat-jinja.cpp @@ -16,9 +16,10 @@ int main(void) { //std::string contents = "{% if messages[0]['role'] != 'system' %}nice {{ messages[0]['content'] }}{% endif %}"; - //std::string contents = " {{ messages[0]['content'] }} "; + //std::string contents = " {{ messages[a]['content'] }} "; + //std::string contents = "{{ aaa[bbb] }}"; - std::ifstream infile("models/templates/Qwen-Qwen3-0.6B.jinja"); + std::ifstream infile("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja"); std::string contents((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); std::cout << "=== INPUT ===\n" << contents << "\n\n"; @@ -27,19 +28,20 @@ int main(void) { jinja::preprocess_options options; options.trim_blocks = true; options.lstrip_blocks = false; - auto tokens = lexer.tokenize(contents, options); - for (const auto & tok : tokens) { - std::cout << "token: type=" << static_cast(tok.t) << " text='" << tok.value << "'\n"; + auto lexer_res = lexer.tokenize(contents, options); + for (const auto & tok : lexer_res.tokens) { + std::cout << "token: type=" << static_cast(tok.t) << " text='" << tok.value << "' pos=" << tok.pos << "\n"; } std::cout << "\n=== AST ===\n"; - jinja::program ast = jinja::parse_from_tokens(tokens); + jinja::program ast = jinja::parse_from_tokens(lexer_res.tokens); for (const auto & stmt : ast.body) { std::cout << "stmt type: " << stmt->type() << "\n"; } std::cout << "\n=== RUN ===\n"; jinja::context ctx; + ctx.source = lexer_res.preprocessed_source; auto make_non_special_string = [](const std::string & s) { jinja::value_string str_val = jinja::mk_val(s);