refactor

tools/server/README-dev.md

@@ -101,7 +101,12 @@ This endpoint is intended to be used internally by the Web UI and subject to cha
 
 **GET /tools**
 
-Get a list of tools, the tool definition is in OAI-compat format.
+Get a list of tools, each tool has these fields:
+- `tool` (string): the ID name of the tool, to be used in POST call. Example: `read_file`
+- `displayName` (string): the name to be displayed on UI. Example: `Read file`
+- `type` (string): always be `"builtin"` for now
+- `permissions` (object): a mapping string --> boolean that indicates the permission required by this tool. This is useful for the UI to ask the user before calling the tool. For now, the only permission supported is `"write"`
+- `definition` (object): the OAI-compat definition of this tool
 
 **POST /tools**
 
@@ -109,7 +114,41 @@ Invoke a tool call, request body is a JSON object with:
 - `tool` (string): the name of the tool
 - `params` (object): a mapping from argument name (string) to argument value
 
-Returns JSON object, the schema depends on the tool itself.
+Returns JSON object. There are two response formats:
+
+Format 1: Plain text. The text will be placed into a field called `plain_text_response`, example:
+
+```json
+{
+    "plain_text_response": "this is a text response"
+}
+```
+
+The client should extract this value and place it inside message content (note: content is no longer a JSON), example
+
+```json
+{
+    "role": "tool",
+    "content": "this is a text response"
+}
+```
+
+Format 2: Normal JSON response, example:
+
+```json
+{
+    "error": "cannot open this file"
+}
+```
+
+That requires `JSON.stringify` when formatted to message content:
+
+```json
+{
+    "role": "tool",
+    "content": "{\"error\":\"cannot open this file\"}"
+}
+```
 
 ### Notable Related PRs
 

tools/server/server-tools.cpp

@@ -139,11 +139,25 @@ static bool glob_match(const std::string & pattern, const std::string & str) {
 
 struct server_tool {
     std::string name;
+    std::string displayName;
     json        definition;
     bool        permission_write = false;
+
     virtual ~server_tool() = default;
-    virtual json to_json() = 0;
+    virtual json get_definition() = 0;
     virtual json invoke(json params) = 0;
+
+    json to_json() {
+        return {
+            {"displayName", displayName},
+            {"tool", name},
+            {"type", "builtin"},
+            {"permissions", json{
+                {"write", permission_write}
+            }},
+            {"definition", get_definition()},
+        };
+    }
 };
 
 //
@@ -153,9 +167,13 @@ struct server_tool {
 static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB
 
 struct server_tool_read_file : server_tool {
-    server_tool_read_file() { name = "read_file"; permission_write = false; }
+    server_tool_read_file() {
+        name = "read_file";
+        displayName = "Read file";
+        permission_write = false;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
@@ -221,7 +239,7 @@ struct server_tool_read_file : server_tool {
             result += out_line;
         }
 
-        return {{"content", result}};
+        return {{"plain_text_response", result}};
     }
 };
 
@@ -232,9 +250,13 @@ struct server_tool_read_file : server_tool {
 static constexpr size_t SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
 
 struct server_tool_file_glob_search : server_tool {
-    server_tool_file_glob_search() { name = "file_glob_search"; permission_write = false; }
+    server_tool_file_glob_search() {
+        name = "file_glob_search";
+        displayName = "File search";
+        permission_write = false;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
@@ -258,7 +280,8 @@ struct server_tool_file_glob_search : server_tool {
         std::string include = json_value(params, "include", std::string("**"));
         std::string exclude = json_value(params, "exclude", std::string(""));
 
-        json files = json::array();
+        std::ostringstream output_text;
+        size_t count = 0;
 
         std::error_code ec;
         for (const auto & entry : fs::recursive_directory_iterator(base,
@@ -272,13 +295,15 @@ struct server_tool_file_glob_search : server_tool {
             if (!glob_match(include, rel)) continue;
             if (!exclude.empty() && glob_match(exclude, rel)) continue;
 
-            files.push_back(entry.path().string());
+            output_text << entry.path().string() << "\n";
-            if (files.size() >= SERVER_TOOL_FILE_SEARCH_MAX_RESULTS) {
+            if (++count >= SERVER_TOOL_FILE_SEARCH_MAX_RESULTS) {
                 break;
             }
         }
 
-        return {{"files", files}, {"count", files.size()}};
+        output_text << "\n---\nTotal matches: " << count << "\n";
+
+        return {{"plain_text_response", output_text.str()}};
     }
 };
 
@@ -289,9 +314,13 @@ struct server_tool_file_glob_search : server_tool {
 static constexpr size_t SERVER_TOOL_GREP_SEARCH_MAX_RESULTS = 100;
 
 struct server_tool_grep_search : server_tool {
-    server_tool_grep_search() { name = "grep_search"; permission_write = false; }
+    server_tool_grep_search() {
+        name = "grep_search";
+        displayName = "Grep search";
+        permission_write = false;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
@@ -326,7 +355,7 @@ struct server_tool_grep_search : server_tool {
             return {{"error", std::string("invalid regex: ") + e.what()}};
         }
 
-        json matches = json::array();
+        std::ostringstream output_text;
         size_t total = 0;
 
         auto search_file = [&](const fs::path & fpath) {
@@ -337,11 +366,11 @@ struct server_tool_grep_search : server_tool {
             while (std::getline(f, line) && total < SERVER_TOOL_GREP_SEARCH_MAX_RESULTS) {
                 lineno++;
                 if (std::regex_search(line, pattern)) {
-                    json match = {{"file", fpath.string()}, {"content", line}};
+                    output_text << fpath.string() << ":";
                     if (show_lineno) {
-                        match["line"] = lineno;
+                        output_text << lineno << ":";
                     }
-                    matches.push_back(match);
+                    output_text << line << "\n";
                     total++;
                 }
             }
@@ -369,7 +398,9 @@ struct server_tool_grep_search : server_tool {
             return {{"error", "path does not exist: " + path}};
         }
 
-        return {{"matches", matches}, {"count", total}};
+        output_text << "\n\n---\nTotal matches: " << total << "\n";
+
+        return {{"plain_text_response", output_text.str()}};
     }
 };
 
@@ -381,9 +412,13 @@ static constexpr size_t SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_OUTPUT_SIZE = 16 * 10
 static constexpr int    SERVER_TOOL_EXEC_SHELL_COMMAND_MAX_TIMEOUT     = 60;        // seconds
 
 struct server_tool_exec_shell_command : server_tool {
-    server_tool_exec_shell_command() { name = "exec_shell_command"; permission_write = true; }
+    server_tool_exec_shell_command() {
+        name = "exec_shell_command";
+        displayName = "Execute shell command";
+        permission_write = true;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
@@ -418,11 +453,13 @@ struct server_tool_exec_shell_command : server_tool {
 
         auto res = run_process(args, max_output, timeout);
 
-        json out = {{"output", res.output}, {"exit_code", res.exit_code}};
+        std::string text_output = res.output;
+        text_output += string_format("\n[exit code: %d]", res.exit_code);
         if (res.timed_out) {
-            out["timed_out"] = true;
+            text_output += " [exit due to timed out]";
         }
-        return out;
+
+        return {{"plain_text_response", text_output}};
     }
 };
 
@@ -431,14 +468,18 @@ struct server_tool_exec_shell_command : server_tool {
 //
 
 struct server_tool_write_file : server_tool {
-    server_tool_write_file() { name = "write_file"; permission_write = true; }
+    server_tool_write_file() {
+        name = "write_file";
+        displayName = "Write file";
+        permission_write = true;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
                 {"name", name},
-                {"description", "Write content to a file, creating it (including parent directories) if it does not exist."},
+                {"description", "Write content to a file, creating it (including parent directories) if it does not exist. May use with edit_file for more complex edits."},
                 {"parameters", {
                     {"type", "object"},
                     {"properties", {
@@ -478,18 +519,188 @@ struct server_tool_write_file : server_tool {
 };
 
 //
-// edit_file: apply a unified diff via git apply
+// edit_file: edit file content via line-based changes
 //
 
 struct server_tool_edit_file : server_tool {
-    server_tool_edit_file() { name = "edit_file"; permission_write = true; }
+    server_tool_edit_file() {
+        name = "edit_file";
+        displayName = "Edit file";
+        permission_write = true;
+    }
 
-    json to_json() override {
+    json get_definition() override {
         return {
             {"type", "function"},
             {"function", {
                 {"name", name},
-                {"description", "Apply a unified diff to edit one or more files using git apply."},
+                {"description", 
+                    "Edit a file by applying a list of line-based changes. "
+                    "Each change targets a 1-based inclusive line range and has a mode: "
+                    "\"replace\" (replace lines with content), "
+                    "\"delete\" (remove lines, content must be empty string), "
+                    "\"append\" (insert content after lineEnd). "
+                    "Set lineStart to -1 to target the end of file (lineEnd is ignored in that case). "
+                    "Changes must not overlap. They are applied in reverse line order automatically."},
+                {"parameters", {
+                    {"type", "object"},
+                    {"properties", {
+                        {"path",    {{"type", "string"}, {"description", "Path to the file to edit"}}},
+                        {"changes", {
+                            {"type", "array"},
+                            {"description", "List of changes to apply"},
+                            {"items", {
+                                {"type", "object"},
+                                {"properties", {
+                                    {"mode",      {{"type", "string"},  {"description", "\"replace\", \"delete\", or \"append\""}}},
+                                    {"lineStart", {{"type", "integer"}, {"description", "First line of the range (1-based); use -1 for end of file"}}},
+                                    {"lineEnd",   {{"type", "integer"}, {"description", "Last line of the range (1-based, inclusive); ignored when lineStart is -1"}}},
+                                    {"content",   {{"type", "string"},  {"description", "Content to insert; must be empty string for delete mode"}}},
+                                }},
+                                {"required", json::array({"mode", "lineStart", "lineEnd", "content"})},
+                            }},
+                        }},
+                    }},
+                    {"required", json::array({"path", "changes"})},
+                }},
+            }},
+        };
+    }
+
+    json invoke(json params) override {
+        std::string path = params.at("path").get<std::string>();
+        const json & changes = params.at("changes");
+
+        if (!changes.is_array()) {
+            return {{"error", "\"changes\" must be an array"}};
+        }
+
+        // read file into lines
+        std::ifstream fin(path);
+        if (!fin) {
+            return {{"error", "failed to open file: " + path}};
+        }
+        std::vector<std::string> lines;
+        {
+            std::string line;
+            while (std::getline(fin, line)) {
+                lines.push_back(line);
+            }
+        }
+        fin.close();
+
+        // validate and collect changes, then sort descending by lineStart
+        struct change_entry {
+            std::string mode;
+            int         line_start; // 1-based
+            int         line_end;   // 1-based inclusive
+            std::string content;
+        };
+        std::vector<change_entry> entries;
+        entries.reserve(changes.size());
+
+        for (const auto & ch : changes) {
+            change_entry e;
+            e.mode       = ch.at("mode").get<std::string>();
+            e.line_start = ch.at("lineStart").get<int>();
+            e.line_end   = ch.at("lineEnd").get<int>();
+            e.content    = ch.at("content").get<std::string>();
+
+            if (e.mode != "replace" && e.mode != "delete" && e.mode != "append") {
+                return {{"error", "invalid mode \"" + e.mode + "\"; must be replace, delete, or append"}};
+            }
+            if (e.mode == "delete" && !e.content.empty()) {
+                return {{"error", "content must be empty string for delete mode"}};
+            }
+            int n = (int) lines.size();
+            if (e.line_start == -1) {
+                // -1 means end of file; lineEnd is ignored — normalize to point past last line
+                e.line_start = n + 1;
+                e.line_end   = n + 1;
+            } else {
+                if (e.line_start < 1 || e.line_end < e.line_start) {
+                    return {{"error", string_format("invalid line range [%d, %d]", e.line_start, e.line_end)}};
+                }
+                if (e.line_end > n) {
+                    return {{"error", string_format("lineEnd %d exceeds file length %d", e.line_end, n)}};
+                }
+            }
+            entries.push_back(std::move(e));
+        }
+
+        // sort descending so earlier-indexed changes don't shift later ones
+        std::sort(entries.begin(), entries.end(), [](const change_entry & a, const change_entry & b) {
+            return a.line_start > b.line_start;
+        });
+
+        // apply changes (0-based indices internally)
+        for (const auto & e : entries) {
+            int idx_start = e.line_start - 1; // 0-based
+            int idx_end   = e.line_end   - 1; // 0-based inclusive
+
+            // split content into lines (preserve trailing newline awareness)
+            std::vector<std::string> new_lines;
+            if (!e.content.empty()) {
+                std::istringstream ss(e.content);
+                std::string ln;
+                while (std::getline(ss, ln)) {
+                    new_lines.push_back(ln);
+                }
+                // if content ends with \n, getline consumed it — no extra empty line needed
+                // if content does NOT end with \n, last line is still captured correctly
+            }
+
+            if (e.mode == "replace") {
+                // erase [idx_start, idx_end] and insert new_lines
+                lines.erase(lines.begin() + idx_start, lines.begin() + idx_end + 1);
+                lines.insert(lines.begin() + idx_start, new_lines.begin(), new_lines.end());
+            } else if (e.mode == "delete") {
+                lines.erase(lines.begin() + idx_start, lines.begin() + idx_end + 1);
+            } else { // append
+                // idx_end + 1 may equal lines.size() when lineStart == -1 (end of file)
+                lines.insert(lines.begin() + idx_end + 1, new_lines.begin(), new_lines.end());
+            }
+        }
+
+        // write file back
+        std::ofstream fout(path, std::ios::binary);
+        if (!fout) {
+            return {{"error", "failed to open file for writing: " + path}};
+        }
+        for (size_t i = 0; i < lines.size(); i++) {
+            fout << lines[i];
+            if (i + 1 < lines.size()) {
+                fout << "\n";
+            }
+        }
+        if (!lines.empty()) {
+            fout << "\n";
+        }
+        if (!fout) {
+            return {{"error", "failed to write file: " + path}};
+        }
+
+        return {{"result", "file edited successfully"}, {"path", path}, {"lines", (int) lines.size()}};
+    }
+};
+
+//
+// apply_diff: apply a unified diff via git apply
+//
+
+struct server_tool_apply_diff : server_tool {
+    server_tool_apply_diff() {
+        name = "apply_diff";
+        displayName = "Apply diff";
+        permission_write = true;
+    }
+
+    json get_definition() override {
+        return {
+            {"type", "function"},
+            {"function", {
+                {"name", name},
+                {"description", "Apply a unified diff to edit one or more files using git apply. Use this instead of edit_file when the changes are complex."},
                 {"parameters", {
                     {"type", "object"},
                     {"properties", {
@@ -541,6 +752,7 @@ static std::vector<std::unique_ptr<server_tool>> build_tools() {
     tools.push_back(std::make_unique<server_tool_exec_shell_command>());
     tools.push_back(std::make_unique<server_tool_write_file>());
     tools.push_back(std::make_unique<server_tool_edit_file>());
+    tools.push_back(std::make_unique<server_tool_apply_diff>());
     return tools;
 }