From db37b712b23cf040e62425504b9679dbbf812163 Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 16 Jan 2026 10:47:44 +0100 Subject: [PATCH] feat: resolve MCP attachment images via rehype plugin LLM can reference tool-generated images using markdown links like, plugin resolves attachment names to base64 from message.extra when present, regular HTTP/data URLs pass through unchanged (no regression) - rehypeResolveAttachmentImages plugin in markdown pipeline - Pass message prop to MarkdownContent and AgenticContent - Force processor reactivity on message.extra changes - Filter assistant images from API context (display-only) --- .../chat/ChatMessages/AgenticContent.svelte | 6 ++-- .../ChatMessages/ChatMessageAssistant.svelte | 4 +-- .../app/misc/MarkdownContent.svelte | 8 ++++- .../lib/markdown/resolve-attachment-images.ts | 32 +++++++++++++++++++ .../webui/src/lib/services/chat.service.ts | 12 ++++--- 5 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 tools/server/webui/src/lib/markdown/resolve-attachment-images.ts diff --git a/tools/server/webui/src/lib/components/app/chat/ChatMessages/AgenticContent.svelte b/tools/server/webui/src/lib/components/app/chat/ChatMessages/AgenticContent.svelte index 673f3b1069..41b9d0fc95 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatMessages/AgenticContent.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatMessages/AgenticContent.svelte @@ -17,8 +17,10 @@ import { AgenticSectionType } from '$lib/enums'; import { AGENTIC_TAGS, AGENTIC_REGEX } from '$lib/constants/agentic'; import { formatJsonPretty } from '$lib/utils/formatters'; + import type { DatabaseMessage } from '$lib/types/database'; interface Props { + message?: DatabaseMessage; content: string; isStreaming?: boolean; } @@ -31,7 +33,7 @@ toolResult?: string; } - let { content, isStreaming = false }: Props = $props(); + let { content, message, isStreaming = false }: Props = $props(); const sections = $derived(parseAgenticContent(content)); @@ -184,7 +186,7 @@ {#each sections as section, index (index)} {#if section.type === AgenticSectionType.TEXT}
- +
{:else if section.type === AgenticSectionType.TOOL_CALL_STREAMING} {@const streamingIcon = isStreaming ? Loader2 : AlertTriangle} diff --git a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte index 312e832d8a..ff7782624c 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte @@ -183,9 +183,9 @@ {#if showRawOutput} 
{messageContent || ''}
{:else if isAgenticContent} - + {:else} - + {/if} {:else}
diff --git a/tools/server/webui/src/lib/components/app/misc/MarkdownContent.svelte b/tools/server/webui/src/lib/components/app/misc/MarkdownContent.svelte index fb98f461b7..2c46330b52 100644 --- a/tools/server/webui/src/lib/components/app/misc/MarkdownContent.svelte +++ b/tools/server/webui/src/lib/components/app/misc/MarkdownContent.svelte @@ -14,6 +14,7 @@ import { rehypeRestoreTableHtml } from '$lib/markdown/table-html-restorer'; import { rehypeEnhanceLinks } from '$lib/markdown/enhance-links'; import { rehypeEnhanceCodeBlocks } from '$lib/markdown/enhance-code-blocks'; + import { rehypeResolveAttachmentImages } from '$lib/markdown/resolve-attachment-images'; import { remarkLiteralHtml } from '$lib/markdown/literal-html'; import { copyCodeToClipboard, preprocessLaTeX } from '$lib/utils'; import '$styles/katex-custom.scss'; @@ -21,9 +22,11 @@ import githubLightCss from 'highlight.js/styles/github.css?inline'; import { mode } from 'mode-watcher'; import CodePreviewDialog from './CodePreviewDialog.svelte'; + import type { DatabaseMessage } from '$lib/types/database'; import { getImageErrorFallbackHtml } from '$lib/utils/image-error-fallback'; interface Props { + message?: DatabaseMessage; content: string; class?: string; disableMath?: boolean; @@ -34,7 +37,7 @@ html: string; } - let { content, class: className = '', disableMath = false }: Props = $props(); + let { content, message, class: className = '', disableMath = false }: Props = $props(); let containerRef = $state(); let renderedBlocks = $state([]); @@ -49,6 +52,8 @@ const themeStyleId = `highlight-theme-${(window.idxThemeStyle = (window.idxThemeStyle ?? 0) + 1)}`; let processor = $derived(() => { + // Force reactivity on message changes + void message; // eslint-disable-next-line @typescript-eslint/no-explicit-any let proc: any = remark().use(remarkGfm); // GitHub Flavored Markdown @@ -70,6 +75,7 @@ .use(rehypeRestoreTableHtml) // Restore limited HTML (e.g.,
,
    ) inside Markdown tables .use(rehypeEnhanceLinks) // Add target="_blank" to links .use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions + .use(rehypeResolveAttachmentImages, { message }) .use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string }); diff --git a/tools/server/webui/src/lib/markdown/resolve-attachment-images.ts b/tools/server/webui/src/lib/markdown/resolve-attachment-images.ts new file mode 100644 index 0000000000..0e5845e4f6 --- /dev/null +++ b/tools/server/webui/src/lib/markdown/resolve-attachment-images.ts @@ -0,0 +1,32 @@ +import type { Root as HastRoot } from 'hast'; +import { visit } from 'unist-util-visit'; +import type { DatabaseMessage, DatabaseMessageExtraImageFile } from '$lib/types/database'; + +/** + * Rehype plugin to resolve attachment image sources. + * Converts attachment names (e.g., "mcp-attachment-xxx.png") to base64 data URLs. + */ +export function rehypeResolveAttachmentImages(options: { message?: DatabaseMessage }) { + return (tree: HastRoot) => { + visit(tree, 'element', (node) => { + if (node.tagName === 'img' && node.properties?.src) { + const src = String(node.properties.src); + + // Skip data URLs and external URLs + if (src.startsWith('data:') || src.startsWith('http')) { + return; + } + + // Find matching attachment + const attachment = options.message?.extra?.find( + (a): a is DatabaseMessageExtraImageFile => a.type === 'IMAGE' && a.name === src + ); + + // Replace with base64 URL if found + if (attachment?.base64Url) { + node.properties.src = attachment.base64Url; + } + } + }); + }; +} diff --git a/tools/server/webui/src/lib/services/chat.service.ts b/tools/server/webui/src/lib/services/chat.service.ts index 4036443fd2..cf435d47b0 100644 --- a/tools/server/webui/src/lib/services/chat.service.ts +++ b/tools/server/webui/src/lib/services/chat.service.ts @@ -619,10 +619,14 @@ export class ChatService { }); } - const imageFiles = message.extra.filter( - (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => - extra.type === AttachmentType.IMAGE - ); + // Only include images for user messages (assistant images are for display only) + const imageFiles = + message.role === 'user' + ? message.extra.filter( + (extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile => + extra.type === AttachmentType.IMAGE + ) + : []; for (const image of imageFiles) { contentParts.push({