happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

webui: enable streaming of tool call arguments

This commit is contained in:
Pascal 2026-01-10 18:48:46 +01:00
parent 74b119e81e
commit b7288a4dd7
4 changed files with 50 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tools/server/webui/src/lib/clients/openai-sse.ts
View File

@ -166,6 +166,9 @@ export class OpenAISseClient {
}
processToolCalls(delta?.tool_calls);
if (aggregatedToolCalls.length > 0) {
callbacks.onToolCallChunk?.(JSON.stringify(aggregatedToolCalls));
}
}
}

37
tools/server/webui/src/lib/components/app/chat/ChatMessages/AgenticContent.svelte
View File

@ -73,13 +73,7 @@
}
const toolName = match[1];
const toolArgsBase64 = match[2];
let toolArgs = '';
try {
toolArgs = decodeURIComponent(escape(atob(toolArgsBase64)));
} catch {
toolArgs = toolArgsBase64;
}
const toolArgs = match[2]; // Direct JSON
const toolResult = match[3].replace(/^\n+|\n+$/g, '');
sections.push({
@ -111,14 +105,8 @@
}
const toolName = pendingMatch[1];
const toolArgsBase64 = pendingMatch[2];
let toolArgs = '';
try {
toolArgs = decodeURIComponent(escape(atob(toolArgsBase64)));
} catch {
toolArgs = toolArgsBase64;
}
// Capture streaming result content (everything after args marker)
const toolArgs = pendingMatch[2]; // Direct JSON
// Capture streaming result content (everything after TOOL_ARGS_END marker)
const streamingResult = (pendingMatch[3] || '').replace(/^\n+|\n+$/g, '');
sections.push({
@ -137,24 +125,7 @@
}
}
const partialArgsBase64 = partialWithNameMatch[2] || '';
let partialArgs = '';
if (partialArgsBase64) {
try {
// Try to decode - may fail if incomplete base64
partialArgs = decodeURIComponent(escape(atob(partialArgsBase64)));
} catch {
// If decoding fails, try padding the base64
try {
const padded =
partialArgsBase64 + '=='.slice(0, (4 - (partialArgsBase64.length % 4)) % 4);
partialArgs = decodeURIComponent(escape(atob(padded)));
} catch {
// Show raw base64 if all decoding fails
partialArgs = '';
}
}
}
const partialArgs = partialWithNameMatch[2] || ''; // Direct JSON streaming
sections.push({
type: AgenticSectionType.TOOL_CALL_STREAMING,

9
tools/server/webui/src/lib/constants/agentic.ts
View File

@ -12,7 +12,8 @@ export const AGENTIC_TAGS = {
TOOL_CALL_START: '<<<AGENTIC_TOOL_CALL_START>>>',
TOOL_CALL_END: '<<<AGENTIC_TOOL_CALL_END>>>',
TOOL_NAME_PREFIX: '<<<TOOL_NAME:',
TOOL_ARGS_PREFIX: '<<<TOOL_ARGS_BASE64:',
TOOL_ARGS_START: '<<<TOOL_ARGS_START>>>',
TOOL_ARGS_END: '<<<TOOL_ARGS_END>>>',
TAG_SUFFIX: '>>>'
} as const;
@ -20,13 +21,13 @@ export const AGENTIC_TAGS = {
export const AGENTIC_REGEX = {
// Matches completed tool calls (with END marker)
COMPLETED_TOOL_CALL:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_BASE64:(.+?)>>>([\s\S]*?)<<<AGENTIC_TOOL_CALL_END>>>/g,
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*?)<<<TOOL_ARGS_END>>>([\s\S]*?)<<<AGENTIC_TOOL_CALL_END>>>/g,
// Matches pending tool call (has NAME and ARGS but no END)
PENDING_TOOL_CALL:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_BASE64:(.+?)>>>([\s\S]*)$/,
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*?)<<<TOOL_ARGS_END>>>([\s\S]*)$/,
// Matches partial tool call (has START and NAME, ARGS still streaming)
PARTIAL_WITH_NAME:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_BASE64:([\s\S]*)$/,
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*)$/,
// Matches early tool call (just START marker)
EARLY_MATCH: /<<<AGENTIC_TOOL_CALL_START>>>([\s\S]*)$/,
// Matches partial marker at end of content

62
tools/server/webui/src/lib/stores/agentic.svelte.ts
View File

@ -239,6 +239,42 @@ class AgenticStore {
// Prepare session state
const sessionMessages: AgenticMessage[] = toAgenticMessages(messages);
const allToolCalls: ApiChatCompletionToolCall[] = [];
// Wrapper to emit agentic tags progressively during streaming
const emittedToolCallStates = $state(
new Map<number, { emittedOnce: boolean; lastArgs: string }>()
);
const wrappedOnToolCallChunk = (serializedToolCalls: string) => {
const toolCalls: ApiChatCompletionToolCall[] = JSON.parse(serializedToolCalls);
for (let i = 0; i < toolCalls.length; i++) {
const toolCall = toolCalls[i];
const toolName = toolCall.function?.name ?? '';
const toolArgs = toolCall.function?.arguments ?? '';
const state = emittedToolCallStates.get(i) || { emittedOnce: false, lastArgs: '' };
if (!state.emittedOnce) {
// First emission: send full header + args
let output = `\n\n<<<AGENTIC_TOOL_CALL_START>>>`;
output += `\n<<<TOOL_NAME:${toolName}>>>`;
output += `\n<<<TOOL_ARGS_START>>>\n`;
output += toolArgs;
onChunk?.(output);
state.emittedOnce = true;
state.lastArgs = toolArgs;
} else if (toolArgs !== state.lastArgs) {
// Subsequent emissions: send only delta
const delta = toolArgs.slice(state.lastArgs.length);
onChunk?.(delta);
state.lastArgs = toolArgs;
}
emittedToolCallStates.set(i, state);
}
onToolCallChunk?.(serializedToolCalls);
};
let capturedTimings: ChatMessageTimings | undefined;
// Build base request from options (messages change per turn)
@ -278,6 +314,7 @@ class AgenticStore {
{
onChunk,
onReasoningChunk: shouldFilterReasoning ? undefined : onReasoningChunk,
onToolCallChunk: wrappedOnToolCallChunk,
onModel,
onFirstValidChunk: undefined,
onProcessingUpdate: (timings, progress) => {
@ -323,7 +360,6 @@ class AgenticStore {
});
}
this._totalToolCalls = allToolCalls.length;
onToolCallChunk?.(JSON.stringify(allToolCalls));
// Add assistant message with tool calls to session
sessionMessages.push({
@ -339,9 +375,6 @@ class AgenticStore {
return;
}
// Emit tool call start (shows "pending" state in UI)
this.emitToolCallStart(toolCall, onChunk);
const mcpCall: MCPToolCall = {
id: toolCall.id,
function: {
@ -404,26 +437,6 @@ class AgenticStore {
}));
}
/**
* Emit tool call start marker (shows "pending" state in UI).
*/
private emitToolCallStart(
toolCall: AgenticToolCallList[number],
emit?: (chunk: string) => void
): void {
if (!emit) return;
const toolName = toolCall.function.name;
const toolArgs = toolCall.function.arguments;
// Base64 encode args to avoid conflicts with markdown/HTML parsing
const toolArgsBase64 = btoa(unescape(encodeURIComponent(toolArgs)));
let output = `\n\n<<<AGENTIC_TOOL_CALL_START>>>`;
output += `\n<<<TOOL_NAME:${toolName}>>>`;
output += `\n<<<TOOL_ARGS_BASE64:${toolArgsBase64}>>>`;
emit(output);
}
/**
* Emit tool call result and end marker.
*/
@ -435,6 +448,7 @@ class AgenticStore {
if (!emit) return;
let output = '';
output += `\n<<<TOOL_ARGS_END>>>`;
if (this.isBase64Image(result)) {
output += `\n![tool-result](${result.trim()})`;
} else {