diff --git a/common/jinja/jinja-vm-builtins.cpp b/common/jinja/jinja-vm-builtins.cpp index cf9de3636e..ecc2cfea52 100644 --- a/common/jinja/jinja-vm-builtins.cpp +++ b/common/jinja/jinja-vm-builtins.cpp @@ -429,6 +429,11 @@ const func_builtins & value_object_t::get_builtins() const { } return result; }}, + {{"dictsort"}, [](const func_args & args) -> value { + // no-op + args.ensure_vals(); + return args.args[0]; + }}, }; return builtins; } diff --git a/common/jinja/jinja-vm.cpp b/common/jinja/jinja-vm.cpp index 7aef38cfbd..276c79156c 100644 --- a/common/jinja/jinja-vm.cpp +++ b/common/jinja/jinja-vm.cpp @@ -82,6 +82,17 @@ value identifier::execute_impl(context & ctx) { } } +value object_literal::execute_impl(context & ctx) { + auto obj = mk_val(); + for (const auto & pair : val) { + std::string key = pair.first->execute(ctx)->as_string().str(); + value val = pair.second->execute(ctx); + JJ_DEBUG("Object literal: setting key '%s' of type %s", key.c_str(), val->type().c_str()); + obj->val_obj[key] = val; + } + return obj; +} + value binary_expression::execute_impl(context & ctx) { value left_val = left->execute(ctx); JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right->type().c_str()); @@ -208,7 +219,7 @@ value binary_expression::execute_impl(context & ctx) { throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type()); } -static value try_builtin_func(const std::string & name, const value & input, bool undef_on_missing = true) { +static value try_builtin_func(const std::string & name, const value & input, bool undef_on_missing = false) { auto builtins = input->get_builtins(); auto it = builtins.find(name); if (it != builtins.end()) { @@ -331,11 +342,16 @@ value for_statement::execute_impl(context & ctx) { std::vector items; if (is_val(iterable_val)) { + JJ_DEBUG("%s", "For loop over object keys"); auto & obj = iterable_val->as_object(); for (auto & p : obj) { - items.push_back(mk_val(p.first)); + auto tuple = mk_val(); + tuple->push_back(mk_val(p.first)); + tuple->push_back(p.second); + items.push_back(tuple); } } else { + JJ_DEBUG("%s", "For loop over array items"); auto & arr = iterable_val->as_array(); for (const auto & item : arr) { items.push_back(item); diff --git a/common/jinja/jinja-vm.h b/common/jinja/jinja-vm.h index 639fba9d03..647da3a72b 100644 --- a/common/jinja/jinja-vm.h +++ b/common/jinja/jinja-vm.h @@ -308,6 +308,7 @@ struct object_literal : public expression { } } std::string type() const override { return "ObjectLiteral"; } + value execute_impl(context & ctx) override; }; // Complex Expressions diff --git a/tests/test-chat-jinja.cpp b/tests/test-chat-jinja.cpp index 097c60a543..0bf15bed91 100644 --- a/tests/test-chat-jinja.cpp +++ b/tests/test-chat-jinja.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #undef NDEBUG #include @@ -11,6 +12,8 @@ #include "jinja/jinja-parser.h" #include "jinja/jinja-lexer.h" +void run(std::string contents); + int main(void) { //std::string contents = "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\\n' + message['content'] | trim + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\\n'}}{% endif %}"; @@ -19,8 +22,29 @@ int main(void) { //std::string contents = " {{ messages[a]['content'] }} "; //std::string contents = "{% if a is not defined %}hello{% endif %}"; - std::ifstream infile("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja"); std::string contents((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); + //std::ifstream infile("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja"); std::string contents((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); + // list all files in models/templates/ and run each + std::string dir_path = "models/templates/"; + for (const auto & entry : std::filesystem::directory_iterator(dir_path)) { + if (entry.is_regular_file()) { + std::cout << "\n\n=== RUNNING TEMPLATE FILE: " << entry.path().string() << " ===\n"; + std::ifstream infile(entry.path()); + std::string contents((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); + try { + run(contents); + } catch (const std::exception & e) { + std::cout << "Exception: " << e.what() << "\n"; + std::cout << "=== CURRENT TEMPLATE FILE: " << entry.path().string() << " ===\n"; + exit(1); + } + } + } + return 0; +} + + +void run(std::string contents) { std::cout << "=== INPUT ===\n" << contents << "\n\n"; jinja::lexer lexer; @@ -68,6 +92,4 @@ int main(void) { for (const auto & part : parts) { std::cout << (part.is_input ? "DATA" : "TMPL") << ": " << part.val << "\n"; } - - return 0; }