diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 3448c4f5be..e055447e0a 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -476,6 +476,74 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools( return force_tool_calls ? section : optional(section); } +// Python-style tool calls: name(arg1="value1", arg2=123) +// Used only by LFM2 for now, so we don't merge it into autoparser +common_peg_parser common_chat_peg_builder::python_style_tool_calls( + const nlohmann::json & tools, + bool parallel_tool_calls) { + if (!tools.is_array() || tools.empty()) { + return eps(); + } + + auto tool_choices = choice(); + + for (const auto & tool_def : tools) { + if (!tool_def.contains("function")) { + continue; + } + const auto & function = tool_def.at("function"); + std::string name = function.at("name"); + nlohmann::json params = function.contains("parameters") ? function.at("parameters") : nlohmann::json::object(); + + auto args = eps(); + if (params.contains("properties") && !params["properties"].empty()) { + auto arg_choice = choice(); + for (const auto & el : params["properties"].items()) { + const std::string & prop_name = el.key(); + const auto & prop_def = el.value(); + bool is_string_type = (prop_def.contains("type") && prop_def["type"] == "string"); + + auto arg_name_parser = literal(prop_name); + + common_peg_parser arg_value_parser = eps(); + auto string_value_parser = choice({ + literal("\"") + tool_arg_string_value(json_string_content()) + literal("\""), + literal("'") + tool_arg_string_value(json_string_content()) + literal("'") + }); + + if (is_string_type) { + arg_value_parser = string_value_parser; + } else { + arg_value_parser = tool_arg_value(python_value()); + } + + // Full argument: name="value" or name=value + auto arg_rule = tool_arg( + tool_arg_open(eps()) + + tool_arg_name(arg_name_parser) + + literal("=") + + arg_value_parser + + tool_arg_close(eps()) + ); + arg_choice |= arg_rule; + } + + args = arg_choice + zero_or_more("," + space() + arg_choice); + } + + auto tool_parser = tool(tool_open(tool_name(literal(name)) + literal("(")) + + space() + tool_args(args) + space() + tool_close(literal(")")) + ); + + tool_choices |= rule("tool-" + name, tool_parser); + } + + if (parallel_tool_calls) { + return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]"; + } + return "[" + space() + tool_choices + space() + "]"; +} + // Helper: Parse dot notation key into prefix and field name static std::pair parse_key_spec(const std::string & key) { auto dot_pos = key.find('.'); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index fe4c1b648f..5ea14be039 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -112,6 +112,11 @@ class common_chat_peg_builder : public common_peg_parser_builder { bool parallel_tool_calls, bool force_tool_calls); + // Helper for Python-style function call format: name(arg1="value1", arg2=123) + // Used by LFM2 and similar templates + common_peg_parser python_style_tool_calls(const nlohmann::json & tools, + bool parallel_tool_calls); + private: // Implementation helpers for standard_json_tools — one per JSON tool call layout mode common_peg_parser build_json_tools_function_is_key(const nlohmann::json & tools, diff --git a/common/chat.cpp b/common/chat.cpp index d12802bd76..29d2e5fd12 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1274,6 +1274,82 @@ static common_chat_params common_chat_params_init_kimi_k2(const common_chat_temp return data; } +// LFM2 format: +// - Reasoning: {reasoning} (optional, only if enable_thinking is true) +// - Content: text after reasoning (optional) +// - Tool calls: <|tool_call_start|>[function_name(arg1="value1", arg2="value2")]<|tool_call_end|> +// Tool calls can appear multiple times (parallel tool calls) +static common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, + const autoparser::templates_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "<|tool_list_start|>", + "<|tool_list_end|>", + "<|tool_call_start|>", + "<|tool_call_end|>", + "", + "", + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + + const std::string TOOL_CALL_START = "<|tool_call_start|>"; + const std::string TOOL_CALL_END = "<|tool_call_end|>"; + const std::string THINK_START = ""; + const std::string THINK_END = ""; + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning && inputs.enable_thinking) { + reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return reasoning + p.content(p.rest()) + end; + } + + auto tool_calls = p.rule("tool-calls", + p.trigger_rule("tool-call", p.literal(TOOL_CALL_START) + + p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls) + + p.literal(TOOL_CALL_END) + ) + ); + + auto content = p.content(p.until(TOOL_CALL_START)); + + return reasoning + content + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START } + }; + } + + return data; +} + namespace workaround { // if first message is system and template does not support it, merge it with next message @@ -1422,6 +1498,14 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ return common_chat_params_init_kimi_k2(tmpl, params); } + // LFM2 - uses <|tool_list_start|>/<|tool_list_end|> markers and <|tool_call_start|>[name(args)]<|tool_call_end|> format + // Detection: template has "<|tool_list_start|>" and "<|tool_list_end|>" markers + if (src.find("<|tool_list_start|>") != std::string::npos && + src.find("<|tool_list_end|>") != std::string::npos) { + LOG_DBG("Using specialized template: LFM2\n"); + return common_chat_params_init_lfm2(tmpl, params); + } + try { LOG_DBG("Using differential autoparser\n"); struct autoparser::autoparser autoparser; diff --git a/models/templates/LFM2-8B-A1B.jinja b/models/templates/LFM2-8B-A1B.jinja index 3738b3d145..fab22e952b 100644 --- a/models/templates/LFM2-8B-A1B.jinja +++ b/models/templates/LFM2-8B-A1B.jinja @@ -6,7 +6,7 @@ {%- set messages = messages[1:] -%} {%- endif -%} {%- if tools -%} - {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "You can use the following tools: <|tool_list_start|>[" -%} + {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%} {%- for tool in tools -%} {%- if tool is not string -%} {%- set tool = tool | tojson -%} @@ -17,7 +17,6 @@ {%- endif -%} {%- endfor -%} {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%} - {{- '**IMPORTANT**: The syntax for calling the tools is: <|tool_call_start|>JSON tool call goes here<|tool_call_end|>. Please only call tools in the specified manner.' -}} {%- endif -%} {%- if ns.system_prompt -%} {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}} @@ -30,18 +29,9 @@ {%- endif -%} {%- if message["role"] == "tool" -%} {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%} - {%- elif message["role"] == "assistant" -%} - {%- if message.tool_calls %} - {%- for tool_call in message.tool_calls %} - {%- if tool_call.function %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '\n<|tool_call_start|>\n{"name": "' + tool_call.name + '", "arguments": ' + (tool_call.arguments if tool_call.arguments is string else tool_call.arguments | tojson) + '}\n<|tool_call_end|>\n' }} - {%- endfor %} - {%- endif %} {%- endif -%} {{- content + "<|im_end|>\n" -}} {%- endfor -%} {%- if add_generation_prompt -%} {{- "<|im_start|>assistant\n" -}} -{%- endif -%} +{%- endif -%} \ No newline at end of file diff --git a/models/templates/llama-cpp-lfm2.jinja b/models/templates/llama-cpp-lfm2.jinja deleted file mode 100644 index b7921120bc..0000000000 --- a/models/templates/llama-cpp-lfm2.jinja +++ /dev/null @@ -1,37 +0,0 @@ -{{- bos_token -}} -{%- set system_prompt = "" -%} -{%- set ns = namespace(system_prompt="") -%} -{%- if messages[0]["role"] == "system" -%} - {%- set ns.system_prompt = messages[0]["content"] -%} - {%- set messages = messages[1:] -%} -{%- endif -%} -{%- if tools -%} - {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%} - {%- for tool in tools -%} - {%- if tool is not string -%} - {%- set tool = tool | tojson -%} - {%- endif -%} - {%- set ns.system_prompt = ns.system_prompt + tool -%} - {%- if not loop.last -%} - {%- set ns.system_prompt = ns.system_prompt + ", " -%} - {%- endif -%} - {%- endfor -%} - {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%} -{%- endif -%} -{%- if ns.system_prompt -%} - {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}} -{%- endif -%} -{%- for message in messages -%} - {{- "<|im_start|>" + message["role"] + "\n" -}} - {%- set content = message["content"] -%} - {%- if content is not string -%} - {%- set content = content | tojson -%} - {%- endif -%} - {%- if message["role"] == "tool" -%} - {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%} - {%- endif -%} - {{- content + "<|im_end|>\n" -}} -{%- endfor -%} -{%- if add_generation_prompt -%} - {{- "<|im_start|>assistant\n" -}} -{%- endif -%} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 7b44776713..2f83d7c0b1 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2387,6 +2387,78 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // LFM2-8B-A1B tests - uses <|tool_list_start|>/<|tool_list_end|> and <|tool_call_start|>[name(args)]<|tool_call_end|> + { + auto tst = peg_tester("models/templates/LFM2-8B-A1B.jinja", detailed_debug); + + // Basic content only + tst.test("Hello, world!\nWhat's up?").expect(message_assist).run(); + + // Single tool call without reasoning + tst.test("<|tool_call_start|>[special_function(arg1=1)]<|tool_call_end|>") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + // Tool call with string argument + tst.test("<|tool_call_start|>[get_time(city=\"XYZCITY\")]<|tool_call_end|>") + .tools({ get_time_tool }) + .expect(message_with_tool_calls("get_time", "{\"city\":\"XYZCITY\"}")) + .run(); + + // Tool call with reasoning (enable_thinking=true) + tst.test("I'm\nthinking<|tool_call_start|>[special_function(arg1=1)]<|tool_call_end|>") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect(message_assist_call_thoughts) + .run(); + + // Multiple tool calls (parallel) + tst.test("<|tool_call_start|>[special_function(arg1=1), special_function_with_opt(arg1=1, arg2=2)]<|tool_call_end|>") + .parallel_tool_calls(true) + .tools({ + special_function_tool, special_function_tool_with_optional_param + }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + + // Tool call with reasoning and content + tst.test("I need to call a function" + "Let me check the time.<|tool_call_start|>[get_time(city=\"Paris\")]<|tool_call_end|>") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ get_time_tool }) + .expect(message_with_reasoning_content_and_multiple_tool_calls( + "I need to call a function", "Let me check the time.", { { "get_time", "{\"city\":\"Paris\"}" } } + )) + .run(); + + // Python tool with multiline code in string + tst.test("<|tool_call_start|>[python(code=\"def hello():\\n print('hey')\")]<|tool_call_end|>") + .tools({ python_tool }) + .expect_tool_calls({ + { "python", R"#({"code": "def hello():\\n print('hey')"})#", "" } + }) + .run(); + + // Partial tool call (streaming) + tst.test("<|tool_call_start|>[special_function(arg1=") + .tools({ special_function_tool }) + .is_partial(true) + .expect(simple_assist_msg("", "", "special_function", "{\"arg1\": ")) + .run(); + + // Tool call with empty arguments + tst.test("<|tool_call_start|>[empty_args()]<|tool_call_end|>") + .tools({ empty_args_tool }) + .expect(simple_assist_msg("", "", "empty_args", "{}")) + .run(); + } + // Apertus-8B-Instruct tests - FUNC_NAME_AS_KEY format // Format: <|tools_prefix|>[{"function_name": {...arguments...}}]<|tools_suffix|> {