parser ok

common/CMakeLists.txt

@@ -83,6 +83,9 @@ add_library(${TARGET} STATIC
     speculative.h
     unicode.cpp
     unicode.h
+    jinja/jinja-lexer.cpp
+    jinja/jinja-parser.cpp
+    jinja/jinja-vm.cpp
     )
 
 target_include_directories(${TARGET} PUBLIC . ../vendor)

common/jinja/jinja-lexer.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include <vector>
 #include <string>
 #include <map>
@@ -48,7 +50,7 @@ struct token {
     std::string value;
 };
 
-std::string type_to_string(token::type t) {
+static std::string type_to_string(token::type t) {
     switch (t) {
         case token::undefined: return "undefined";
         case token::text: return "text";

common/jinja/jinja-parser.cpp

@@ -1,5 +1,6 @@
 #include "jinja-lexer.h"
 #include "jinja-vm.h"
+#include "jinja-parser.h"
 
 #include <string>
 #include <vector>
@@ -22,12 +23,12 @@ class parser {
 public:
     parser(const std::vector<token> & t) : tokens(t) {}
 
-    statement_ptr parse() {
+    program parse() {
         statements body;
         while (current < tokens.size()) {
             body.push_back(parse_any());
         }
-        return std::make_unique<program>(std::move(body));
+        return program(std::move(body));
     }
 
 private:
@@ -320,7 +321,7 @@ private:
     statement_ptr parse_logical_or_expression() {
         auto left = parse_logical_and_expression();
         while (is_identifier("or")) {
-            auto op = tokens[current++];
+            token op = tokens[current++];
             left = std::make_unique<binary_expression>(op, std::move(left), parse_logical_and_expression());
         }
         return left;
@@ -538,7 +539,7 @@ private:
     }
 };
 
-statement_ptr parse(const std::vector<token>& tokens) {
+program parse_from_tokens(const std::vector<token> & tokens) {
     return parser(tokens).parse();
 }
 

common/jinja/jinja-parser.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "jinja-lexer.h"
+#include "jinja-vm.h"
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <stdexcept>
+#include <algorithm>
+
+namespace jinja {
+
+program parse_from_tokens(const std::vector<token> & tokens);
+
+} // namespace jinja

common/jinja/jinja-vm.h

@@ -1,3 +1,4 @@
+#pragma once
 #include "jinja-lexer.h"
 
 #include <string>
@@ -181,26 +182,21 @@ struct identifier : public expression {
 
 // Literals
 
-/**
+struct integer_literal : public expression { 
- * Abstract base class for all Literal expressions.
+    int64_t value;
- * Should not be instantiated directly.
+    explicit integer_literal(int64_t value) : value(value) {}
- */
-template <typename T>
-struct literal : public expression {
-    T value;
-    explicit literal(T && value) : value(std::move(value)) {}
-    std::string type() const override { return "Literal"; }
-};
-
-struct integer_literal : public literal<int64_t> { 
     std::string type() const override { return "IntegerLiteral"; }
 };
 
-struct float_literal : public literal<double> {
+struct float_literal : public expression {
+    double value;
+    explicit float_literal(double value) : value(value) {}
     std::string type() const override { return "FloatLiteral"; }
 };
 
-struct string_literal : public literal<std::string> {
+struct string_literal : public expression {
+    std::string value;
+    explicit string_literal(const std::string & value) : value(value) {}
     std::string type() const override { return "StringLiteral"; }
 };
 
@@ -240,11 +236,11 @@ struct object_literal : public expression {
  * of operations being determined by the operator.
  */
 struct binary_expression : public expression {
-    token::type op;
+    token op;
     statement_ptr left;
     statement_ptr right;
 
-    binary_expression(token::type op, statement_ptr && left, statement_ptr && right)
+    binary_expression(token op, statement_ptr && left, statement_ptr && right)
         : op(op), left(std::move(left)), right(std::move(right)) {
         chk_type<expression>(this->left);
         chk_type<expression>(this->right);

tests/test-chat-jinja.cpp

@@ -7,9 +7,7 @@
 #undef NDEBUG
 #include <cassert>
 
-#include "peg-parser.h"
+#include "jinja/jinja-parser.h"
-#include "json-schema-to-grammar.h"
-#include "jinja/jinja-compiler.h"
 #include "jinja/jinja-lexer.h"
 
 int main(void) {
@@ -26,30 +24,11 @@ int main(void) {
         std::cout << "token: type=" << static_cast<int>(tok.t) << " text='" << tok.value << "'\n";
     }
 
-    // jinja::compiler compiler;
+    jinja::program ast = jinja::parse_from_tokens(tokens);
-    // compiler.builder.set_root(compiler.root);
+    std::cout << "\n=== AST ===\n";
-    // auto parser = compiler.builder.build();
+    for (const auto & stmt : ast.body) {
-
+        std::cout << "stmt type: " << stmt->type() << "\n";
-    // auto grammar = build_grammar([&](const common_grammar_builder & builder0) {
+    }
-    //     parser.build_grammar(builder0);
-    // });
-    // printf("== GRAMMAR ==\n");
-    // printf("%s\n", grammar.c_str());
-
-    // // printf("== DUMP ==\n");
-    // // printf("%s\n", parser.dump(compiler.root.id()).c_str());
-
-    // printf("== PARSE ==\n");
-
-    // common_peg_parse_context ctx(contents);
-    // const auto result = parser.parse(ctx);
-    // if (!result.success()) {
-    //     throw std::runtime_error("failed to parse, type = " + std::to_string(result.type));
-    // }
-
-    // ctx.ast.visit(result, [&](const common_peg_ast_node & node) {
-    //     printf("node: rule='%s' text='%s'\n", node.rule.c_str(), std::string(node.text).c_str());
-    // });
 
     return 0;
 }