From a3c2144c1def1ce4862c209b0e21196b602faa5f Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 15 Jan 2026 23:21:27 +0100 Subject: [PATCH] feat: persist base64 attachments from tool results --- .../webui/src/lib/clients/agentic.client.ts | 107 ++++++++++++++---- .../webui/src/lib/clients/chat.client.ts | 19 ++++ .../server/webui/src/lib/types/settings.d.ts | 4 +- 3 files changed, 107 insertions(+), 23 deletions(-) diff --git a/tools/server/webui/src/lib/clients/agentic.client.ts b/tools/server/webui/src/lib/clients/agentic.client.ts index 915d85805c..ecbea1e02d 100644 --- a/tools/server/webui/src/lib/clients/agentic.client.ts +++ b/tools/server/webui/src/lib/clients/agentic.client.ts @@ -39,11 +39,13 @@ import type { } from '$lib/types/chat'; import type { MCPToolCall } from '$lib/types'; import type { DatabaseMessage, DatabaseMessageExtra, McpServerOverride } from '$lib/types/database'; +import { AttachmentType } from '$lib/enums'; export interface AgenticFlowCallbacks { onChunk?: (chunk: string) => void; onReasoningChunk?: (chunk: string) => void; onToolCallChunk?: (serializedToolCalls: string) => void; + onAttachments?: (extras: DatabaseMessageExtra[]) => void; onModel?: (model: string) => void; onComplete?: ( content: string, @@ -131,8 +133,16 @@ export class AgenticClient { */ async runAgenticFlow(params: AgenticFlowParams): Promise { const { messages, options = {}, callbacks, signal, perChatOverrides } = params; - const { onChunk, onReasoningChunk, onToolCallChunk, onModel, onComplete, onError, onTimings } = - callbacks; + const { + onChunk, + onReasoningChunk, + onToolCallChunk, + onAttachments, + onModel, + onComplete, + onError, + onTimings + } = callbacks; // Get agentic configuration (considering per-chat MCP overrides) const agenticConfig = getAgenticConfig(config(), perChatOverrides); @@ -188,6 +198,7 @@ export class AgenticClient { onChunk, onReasoningChunk, onToolCallChunk, + onAttachments, onModel, onComplete, onError, @@ -222,8 +233,15 @@ export class AgenticClient { signal?: AbortSignal; }): Promise { const { messages, options, tools, agenticConfig, callbacks, signal } = params; - const { onChunk, onReasoningChunk, onToolCallChunk, onModel, onComplete, onTimings } = - callbacks; + const { + onChunk, + onReasoningChunk, + onToolCallChunk, + onAttachments, + onModel, + onComplete, + onTimings + } = callbacks; const sessionMessages: AgenticMessage[] = toAgenticMessages(messages); const allToolCalls: ApiChatCompletionToolCall[] = []; @@ -509,10 +527,15 @@ export class AgenticClient { return; } - this.emitToolCallResult(result, maxToolPreviewLines, onChunk); + const { cleanedResult, attachments } = this.extractBase64Attachments(result); + if (attachments.length > 0) { + onAttachments?.(attachments); + } - // Add tool result to session (sanitize base64 images for context) - const contextValue = this.isBase64Image(result) ? '[Image displayed to user]' : result; + this.emitToolCallResult(cleanedResult, maxToolPreviewLines, onChunk); + + // Add tool result to session (sanitize base64 payloads for context) + const contextValue = attachments.length > 0 ? cleanedResult : result; sessionMessages.push({ role: 'tool', tool_call_id: toolCall.id, @@ -597,14 +620,10 @@ export class AgenticClient { let output = ''; output += `\n<<>>`; - if (this.isBase64Image(result)) { - output += `\n![tool-result](${result.trim()})`; - } else { - // Don't wrap in code fences - result may already be markdown with its own code blocks - const lines = result.split('\n'); - const trimmedLines = lines.length > maxLines ? lines.slice(-maxLines) : lines; - output += `\n${trimmedLines.join('\n')}`; - } + // Don't wrap in code fences - result may already be markdown with its own code blocks + const lines = result.split('\n'); + const trimmedLines = lines.length > maxLines ? lines.slice(-maxLines) : lines; + output += `\n${trimmedLines.join('\n')}`; output += `\n<<>>\n`; emit(output); @@ -618,15 +637,59 @@ export class AgenticClient { * */ - private isBase64Image(content: string): boolean { - const trimmed = content.trim(); - if (!trimmed.startsWith('data:image/')) return false; + private extractBase64Attachments(result: string): { + cleanedResult: string; + attachments: DatabaseMessageExtra[]; + } { + if (!result.trim()) { + return { cleanedResult: result, attachments: [] }; + } - const match = trimmed.match(/^data:image\/(png|jpe?g|gif|webp);base64,([A-Za-z0-9+/]+=*)$/); - if (!match) return false; + const lines = result.split('\n'); + const attachments: DatabaseMessageExtra[] = []; + let attachmentIndex = 0; - const base64Payload = match[2]; - return base64Payload.length > 0 && base64Payload.length % 4 === 0; + const cleanedLines = lines.map((line) => { + const trimmedLine = line.trim(); + const match = trimmedLine.match(/^data:([^;]+);base64,([A-Za-z0-9+/]+=*)$/); + if (!match) return line; + + const mimeType = match[1].toLowerCase(); + const base64Data = match[2]; + if (!base64Data) return line; + + attachmentIndex += 1; + const name = this.buildAttachmentName(mimeType, attachmentIndex); + + if (mimeType.startsWith('image/')) { + attachments.push({ + type: AttachmentType.IMAGE, + name, + base64Url: trimmedLine + }); + return `[Attachment saved: ${name}]`; + } + return line; + }); + + return { + cleanedResult: cleanedLines.join('\n').trim(), + attachments + }; + } + + private buildAttachmentName(mimeType: string, index: number): string { + const extensionMap: Record = { + 'image/jpeg': 'jpg', + 'image/jpg': 'jpg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp' + }; + + const extension = extensionMap[mimeType] ?? 'img'; + const timestamp = Date.now(); + return `mcp-attachment-${timestamp}-${index}.${extension}`; } clearError(): void { diff --git a/tools/server/webui/src/lib/clients/chat.client.ts b/tools/server/webui/src/lib/clients/chat.client.ts index 0608974610..566ff9fa89 100644 --- a/tools/server/webui/src/lib/clients/chat.client.ts +++ b/tools/server/webui/src/lib/clients/chat.client.ts @@ -55,6 +55,7 @@ export interface ChatStreamCallbacks { onChunk?: (chunk: string) => void; onReasoningChunk?: (chunk: string) => void; onToolCallChunk?: (chunk: string) => void; + onAttachments?: (extras: DatabaseMessageExtra[]) => void; onModel?: (model: string) => void; onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; onComplete?: ( @@ -479,6 +480,9 @@ export class ChatClient { let streamedToolCallContent = ''; let resolvedModel: string | null = null; let modelPersisted = false; + let streamedExtras: DatabaseMessageExtra[] = assistantMessage.extra + ? JSON.parse(JSON.stringify(assistantMessage.extra)) + : []; const recordModel = (modelName: string | null | undefined, persistImmediately = true): void => { if (!modelName) return; @@ -520,6 +524,15 @@ export class ChatClient { const idx = conversationsStore.findMessageIndex(assistantMessage.id); conversationsStore.updateMessageAtIndex(idx, { toolCalls: streamedToolCallContent }); }, + onAttachments: (extras: DatabaseMessageExtra[]) => { + if (!extras.length) return; + streamedExtras = [...streamedExtras, ...extras]; + const idx = conversationsStore.findMessageIndex(assistantMessage.id); + conversationsStore.updateMessageAtIndex(idx, { extra: streamedExtras }); + DatabaseService.updateMessage(assistantMessage.id, { extra: streamedExtras }).catch( + console.error + ); + }, onModel: (modelName: string) => recordModel(modelName), onTimings: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => { const tokensPerSecond = @@ -552,6 +565,9 @@ export class ChatClient { toolCalls: toolCallContent || streamedToolCallContent, timings }; + if (streamedExtras.length > 0) { + updateData.extra = streamedExtras; + } if (resolvedModel && !modelPersisted) { updateData.model = resolvedModel; } @@ -562,6 +578,9 @@ export class ChatClient { content: updateData.content as string, toolCalls: updateData.toolCalls as string }; + if (streamedExtras.length > 0) { + uiUpdate.extra = streamedExtras; + } if (timings) uiUpdate.timings = timings; if (resolvedModel) uiUpdate.model = resolvedModel; diff --git a/tools/server/webui/src/lib/types/settings.d.ts b/tools/server/webui/src/lib/types/settings.d.ts index f3e8923d2c..82c037c82c 100644 --- a/tools/server/webui/src/lib/types/settings.d.ts +++ b/tools/server/webui/src/lib/types/settings.d.ts @@ -1,6 +1,7 @@ import type { SETTING_CONFIG_DEFAULT } from '$lib/constants/settings-config'; -import type { ChatMessageTimings } from './chat'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from './chat'; import type { OpenAIToolDefinition } from './mcp'; +import type { DatabaseMessageExtra } from './database'; export type SettingsConfigValue = string | number | boolean; @@ -53,6 +54,7 @@ export interface SettingsChatServiceOptions { onChunk?: (chunk: string) => void; onReasoningChunk?: (chunk: string) => void; onToolCallChunk?: (chunk: string) => void; + onAttachments?: (extras: DatabaseMessageExtra[]) => void; onModel?: (model: string) => void; onTimings?: (timings?: ChatMessageTimings, promptProgress?: ChatMessagePromptProgress) => void; onComplete?: (