The site title

#pragma once #include "chat-diff-analyzer.h" #include #include #include std::string trim_whitespace(const std::string & str); std::string trim_leading_whitespace(const std::string & str); std::string trim_trailing_whitespace(const std::string & str); std::string trim_trailing_newlines(const std::string & str); // calculate a diff split (longest common prefix, longest common suffix excluding prefix, // mismatched part on the left, mismatched part on the right) between two strings // account for markers - align prefix and suffix endings so that they end on markers // * eg.: // calculate_diff_split("

", "

Something

") -> // { "prefix": "" (not: "<"), "suffix": "", "left": "

", "right": "

Something

" } // calculate_diff_split("Something", "") -> // { "prefix": "", "suffix": "", "left": "Something", "right": "" } diff_split calculate_diff_split(const std::string & left, const std::string & right); // Returns the prefix of `full` up until the first occurrence of the common prefix of `left` and `right` // Returns empty string if there's no common prefix // * eg.: // until_common_prefix("really want a FUNCTION call", "FUNCTION alpha", "FUNCTION beta") -> "really want a " // until_common_prefix("", "", "") -> "" // until_common_prefix("some text", "1234", "abcd") -> "" // until_common_prefix("one arg two args three args four", "argument alpha", "argument beta") -> "one "" std::string until_common_prefix(const std::string & full, const std::string & left, const std::string & right); // Returns the suffix of `full` after the last occurrence of the common suffix of `left` and `right` // Returns empty string if there's no common suffix // Mirror function of `until_common_prefix` // * eg.: // after_common_suffix("really want a FUNCTION call", "first FUNCTION", "second FUNCTION") -> " call" // after_common_suffix("one arg two-args three args four", "alpha-args", "beta-args") -> " three args four" std::string after_common_suffix(const std::string & full, const std::string & left, const std::string & right); // Segmentize text into markers and non-marker fragments // * eg.: // segmentize_markers("The site title

Here's some content

" -> // [ (MARKER, ""), (MARKER, ""), (MARKER, ""), (TEXT, "The site title"), (MARKER, ""), // (MARKER, ""), (MARKER, "

"), (TEXT, "Here's some "), (MARKER, ""), (TEXT, "content"), (MARKER, ""), // (MARKER, "

"), (MARKER, ""), (MARKER, "") // ] // segmentize_markers("<|tool_call|>[args]{ are here }[/args]<|tool_call_end|>") -> // [ (MARKER, "<|tool_call|>"), (MARKER, "[args]"), (TEXT, "{ are here }"), (MARKER, "[/args]"), (MARKER, "<|tool_call_end|>") ] std::vector segmentize_markers(const std::string & text); // Prune whitespace-only segments from a vector of segments // * eg.: // segmentize_markers("\n\n\n \n\n\n") -> // X = [ (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n \n"), // (MARKER, ""), (TEXT, "\n"), (MARKER, ""), (TEXT, "\n"), (MARKER, "") ] // prune_whitespace_segments(X) -> [ (MARKER, ""), (MARKER, ""), (MARKER, ""), (MARKER, ""), // (MARKER, ""), (MARKER, "") ] std::vector prune_whitespace_segments(const std::vector & segments); namespace autoparser { // Apply a template with the given parameters, returning the rendered string (empty on failure) std::string apply_template(const common_chat_template & tmpl, const template_params & params); // Factorized differential comparison function // Takes base params and a single modifier lambda to create variant B // Returns compare_variants_result containing diff and both outputs, or std::nullopt on failure std::optional compare_variants( const common_chat_template & tmpl, const template_params & params_A, const std::function & params_modifier); } // namespace autoparser