testing more templates

common/jinja/jinja-vm-builtins.cpp

@@ -429,6 +429,11 @@ const func_builtins & value_object_t::get_builtins() const {
             }
             return result;
         }},
+        {{"dictsort"}, [](const func_args & args) -> value {
+            // no-op
+            args.ensure_vals<value_object>();
+            return args.args[0];
+        }},
     };
     return builtins;
 }

common/jinja/jinja-vm.cpp

@@ -82,6 +82,17 @@ value identifier::execute_impl(context & ctx) {
     }
 }
 
+value object_literal::execute_impl(context & ctx) {
+    auto obj = mk_val<value_object>();
+    for (const auto & pair : val) {
+        std::string key = pair.first->execute(ctx)->as_string().str();
+        value val = pair.second->execute(ctx);
+        JJ_DEBUG("Object literal: setting key '%s' of type %s", key.c_str(), val->type().c_str());
+        obj->val_obj[key] = val;
+    }
+    return obj;
+}
+
 value binary_expression::execute_impl(context & ctx) {
     value left_val = left->execute(ctx);
     JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right->type().c_str());
@@ -208,7 +219,7 @@ value binary_expression::execute_impl(context & ctx) {
     throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
 }
 
-static value try_builtin_func(const std::string & name, const value & input, bool undef_on_missing = true) {
+static value try_builtin_func(const std::string & name, const value & input, bool undef_on_missing = false) {
     auto builtins = input->get_builtins();
     auto it = builtins.find(name);
     if (it != builtins.end()) {
@@ -331,11 +342,16 @@ value for_statement::execute_impl(context & ctx) {
 
     std::vector<value> items;
     if (is_val<value_object>(iterable_val)) {
+        JJ_DEBUG("%s", "For loop over object keys");
         auto & obj = iterable_val->as_object();
         for (auto & p : obj) {
-            items.push_back(mk_val<value_string>(p.first));
+            auto tuple = mk_val<value_array>();
+            tuple->push_back(mk_val<value_string>(p.first));
+            tuple->push_back(p.second);
+            items.push_back(tuple);
         }
     } else {
+        JJ_DEBUG("%s", "For loop over array items");
         auto & arr = iterable_val->as_array();
         for (const auto & item : arr) {
             items.push_back(item);

common/jinja/jinja-vm.h

@@ -308,6 +308,7 @@ struct object_literal : public expression {
         }
     }
     std::string type() const override { return "ObjectLiteral"; }
+    value execute_impl(context & ctx) override;
 };
 
 // Complex Expressions

tests/test-chat-jinja.cpp

@@ -4,6 +4,7 @@
 #include <regex>
 #include <iostream>
 #include <fstream>
+#include <filesystem>
 
 #undef NDEBUG
 #include <cassert>
@@ -11,6 +12,8 @@
 #include "jinja/jinja-parser.h"
 #include "jinja/jinja-lexer.h"
 
+void run(std::string contents);
+
 int main(void) {
     //std::string contents = "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\\n' + message['content'] | trim + '<end_of_turn>\\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\\n'}}{% endif %}";
 
@@ -19,8 +22,29 @@ int main(void) {
     //std::string contents = "<some_tokens> {{ messages[a]['content'] }} <another_token>";
     //std::string contents = "{% if a is not defined %}hello{% endif %}";
 
-    std::ifstream infile("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja"); std::string contents((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
+    //std::ifstream infile("models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja"); std::string contents((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
 
+    // list all files in models/templates/ and run each
+    std::string dir_path = "models/templates/";
+    for (const auto & entry : std::filesystem::directory_iterator(dir_path)) {
+        if (entry.is_regular_file()) {
+            std::cout << "\n\n=== RUNNING TEMPLATE FILE: " << entry.path().string() << " ===\n";
+            std::ifstream infile(entry.path());
+            std::string contents((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
+            try {
+                run(contents);
+            } catch (const std::exception & e) {
+                std::cout << "Exception: " << e.what() << "\n";
+                std::cout << "=== CURRENT TEMPLATE FILE: " << entry.path().string() << " ===\n";
+                exit(1);
+            }
+        }
+    }
+    return 0;
+}
+
+
+void run(std::string contents) {
     std::cout << "=== INPUT ===\n" << contents << "\n\n";
 
     jinja::lexer lexer;
@@ -68,6 +92,4 @@ int main(void) {
     for (const auto & part : parts) {
         std::cout << (part.is_input ? "DATA" : "TMPL") << ": " << part.val << "\n";
     }
-
-    return 0;
 }