happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: persist base64 attachments from tool results

This commit is contained in:
Pascal 2026-01-15 23:21:27 +01:00
parent a377605f60
commit a3c2144c1d
3 changed files with 107 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
tools/server/webui/src/lib/clients/agentic.client.ts
View File

@ -39,11 +39,13 @@ import type {
} from '$lib/types/chat';
import type { MCPToolCall } from '$lib/types';
import type { DatabaseMessage, DatabaseMessageExtra, McpServerOverride } from '$lib/types/database';
import { AttachmentType } from '$lib/enums';
export interface AgenticFlowCallbacks {
onChunk?: (chunk: string) => void;
onReasoningChunk?: (chunk: string) => void;
onToolCallChunk?: (serializedToolCalls: string) => void;
onAttachments?: (extras: DatabaseMessageExtra[]) => void;
onModel?: (model: string) => void;
onComplete?: (
content: string,
@ -131,8 +133,16 @@ export class AgenticClient {
*/
async runAgenticFlow(params: AgenticFlowParams): Promise<AgenticFlowResult> {
const { messages, options = {}, callbacks, signal, perChatOverrides } = params;
const { onChunk, onReasoningChunk, onToolCallChunk, onModel, onComplete, onError, onTimings } =
callbacks;
const {
onChunk,
onReasoningChunk,
onToolCallChunk,
onAttachments,
onModel,
onComplete,
onError,
onTimings
} = callbacks;
// Get agentic configuration (considering per-chat MCP overrides)
const agenticConfig = getAgenticConfig(config(), perChatOverrides);
@ -188,6 +198,7 @@ export class AgenticClient {
onChunk,
onReasoningChunk,
onToolCallChunk,
onAttachments,
onModel,
onComplete,
onError,
@ -222,8 +233,15 @@ export class AgenticClient {
signal?: AbortSignal;
}): Promise<void> {
const { messages, options, tools, agenticConfig, callbacks, signal } = params;
const { onChunk, onReasoningChunk, onToolCallChunk, onModel, onComplete, onTimings } =
callbacks;
const {
onChunk,
onReasoningChunk,
onToolCallChunk,
onAttachments,
onModel,
onComplete,
onTimings
} = callbacks;
const sessionMessages: AgenticMessage[] = toAgenticMessages(messages);
const allToolCalls: ApiChatCompletionToolCall[] = [];
@ -509,10 +527,15 @@ export class AgenticClient {
return;
}
this.emitToolCallResult(result, maxToolPreviewLines, onChunk);
const { cleanedResult, attachments } = this.extractBase64Attachments(result);
if (attachments.length > 0) {
onAttachments?.(attachments);
}
// Add tool result to session (sanitize base64 images for context)
const contextValue = this.isBase64Image(result) ? '[Image displayed to user]' : result;
this.emitToolCallResult(cleanedResult, maxToolPreviewLines, onChunk);
// Add tool result to session (sanitize base64 payloads for context)
const contextValue = attachments.length > 0 ? cleanedResult : result;
sessionMessages.push({
role: 'tool',
tool_call_id: toolCall.id,
@ -597,14 +620,10 @@ export class AgenticClient {
let output = '';
output += `\n<<<TOOL_ARGS_END>>>`;
if (this.isBase64Image(result)) {
output += `\n![tool-result](${result.trim()})`;
} else {
// Don't wrap in code fences - result may already be markdown with its own code blocks
const lines = result.split('\n');
const trimmedLines = lines.length > maxLines ? lines.slice(-maxLines) : lines;
output += `\n${trimmedLines.join('\n')}`;
}
// Don't wrap in code fences - result may already be markdown with its own code blocks
const lines = result.split('\n');
const trimmedLines = lines.length > maxLines ? lines.slice(-maxLines) : lines;
output += `\n${trimmedLines.join('\n')}`;
output += `\n<<<AGENTIC_TOOL_CALL_END>>>\n`;
emit(output);
@ -618,15 +637,59 @@ export class AgenticClient {
*
*/
private isBase64Image(content: string): boolean {
const trimmed = content.trim();
if (!trimmed.startsWith('data:image/')) return false;
private extractBase64Attachments(result: string): {
cleanedResult: string;
attachments: DatabaseMessageExtra[];
} {
if (!result.trim()) {
return { cleanedResult: result, attachments: [] };
}
const match = trimmed.match(/^data:image\/(png|jpe?g|gif|webp);base64,([A-Za-z0-9+/]+=*)$/);
if (!match) return false;
const lines = result.split('\n');
const attachments: DatabaseMessageExtra[] = [];
let attachmentIndex = 0;
const base64Payload = match[2];
return base64Payload.length > 0 && base64Payload.length % 4 === 0;
const cleanedLines = lines.map((line) => {
const trimmedLine = line.trim();
const match = trimmedLine.match(/^data:([^;]+);base64,([A-Za-z0-9+/]+=*)$/);
if (!match) return line;
const mimeType = match[1].toLowerCase();
const base64Data = match[2];
if (!base64Data) return line;
attachmentIndex += 1;
const name = this.buildAttachmentName(mimeType, attachmentIndex);
if (mimeType.startsWith('image/')) {
attachments.push({
type: AttachmentType.IMAGE,
name,
base64Url: trimmedLine
});
return `[Attachment saved: ${name}]`;
}
return line;
});
return {
cleanedResult: cleanedLines.join('\n').trim(),
attachments
};
}
private buildAttachmentName(mimeType: string, index: number): string {
const extensionMap: Record<string, string> = {
'image/jpeg': 'jpg',
'image/jpg': 'jpg',
'image/png': 'png',
'image/gif': 'gif',
'image/webp': 'webp'
};
const extension = extensionMap[mimeType] ?? 'img';
const timestamp = Date.now();
return `mcp-attachment-${timestamp}-${index}.${extension}`;
}
clearError(): void {

19
tools/server/webui/src/lib/clients/chat.client.ts
View File

@ -55,6 +55,7 @@ export interface ChatStreamCallbacks {
onChunk?: (chunk: string) => void;
onReasoningChunk?: (chunk: string) => void;
onToolCallChunk?: (chunk: string) => void;
onAttachments?: (extras: DatabaseMessageExtra[]) => void;
onModel?: (model: string) => void;
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
onComplete?: (
@ -479,6 +480,9 @@ export class ChatClient {
let streamedToolCallContent = '';
let resolvedModel: string | null = null;
let modelPersisted = false;
let streamedExtras: DatabaseMessageExtra[] = assistantMessage.extra
? JSON.parse(JSON.stringify(assistantMessage.extra))
: [];
const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => {
if (!modelName) return;
@ -520,6 +524,15 @@ export class ChatClient {
const idx = conversationsStore.findMessageIndex(assistantMessage.id);
conversationsStore.updateMessageAtIndex(idx, { toolCalls: streamedToolCallContent });
},
onAttachments: (extras: DatabaseMessageExtra[]) => {
if (!extras.length) return;
streamedExtras = [...streamedExtras, ...extras];
const idx = conversationsStore.findMessageIndex(assistantMessage.id);
conversationsStore.updateMessageAtIndex(idx, { extra: streamedExtras });
DatabaseService.updateMessage(assistantMessage.id, { extra: streamedExtras }).catch(
console.error
);
},
onModel: (modelName: string) => recordModel(modelName),
onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => {
const tokensPerSecond =
@ -552,6 +565,9 @@ export class ChatClient {
toolCalls: toolCallContent || streamedToolCallContent,
timings
};
if (streamedExtras.length > 0) {
updateData.extra = streamedExtras;
}
if (resolvedModel && !modelPersisted) {
updateData.model = resolvedModel;
}
@ -562,6 +578,9 @@ export class ChatClient {
content: updateData.content as string,
toolCalls: updateData.toolCalls as string
};
if (streamedExtras.length > 0) {
uiUpdate.extra = streamedExtras;
}
if (timings) uiUpdate.timings = timings;
if (resolvedModel) uiUpdate.model = resolvedModel;

4
tools/server/webui/src/lib/types/settings.d.ts vendored
View File

@ -1,6 +1,7 @@
import type { SETTING_CONFIG_DEFAULT } from '$lib/constants/settings-config';
import type { ChatMessageTimings } from './chat';
import type { ChatMessagePromptProgress, ChatMessageTimings } from './chat';
import type { OpenAIToolDefinition } from './mcp';
import type { DatabaseMessageExtra } from './database';
export type SettingsConfigValue = string | number | boolean;
@ -53,6 +54,7 @@ export interface SettingsChatServiceOptions {
onChunk?: (chunk: string) => void;
onReasoningChunk?: (chunk: string) => void;
onToolCallChunk?: (chunk: string) => void;
onAttachments?: (extras: DatabaseMessageExtra[]) => void;
onModel?: (model: string) => void;
onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void;
onComplete?: (