feat: resolve MCP attachment images via rehype plugin

LLM can reference tool-generated images using markdown links like,
plugin resolves attachment names to base64 from message.extra when present,
regular HTTP/data URLs pass through unchanged (no regression)

- rehypeResolveAttachmentImages plugin in markdown pipeline
- Pass message prop to MarkdownContent and AgenticContent
- Force processor reactivity on message.extra changes
- Filter assistant images from API context (display-only)
This commit is contained in:
Pascal 2026-01-16 10:47:44 +01:00
parent a3c2144c1d
commit db37b712b2
5 changed files with 53 additions and 9 deletions

View File

@ -17,8 +17,10 @@
import { AgenticSectionType } from '$lib/enums';
import { AGENTIC_TAGS, AGENTIC_REGEX } from '$lib/constants/agentic';
import { formatJsonPretty } from '$lib/utils/formatters';
import type { DatabaseMessage } from '$lib/types/database';
interface Props {
message?: DatabaseMessage;
content: string;
isStreaming?: boolean;
}
@ -31,7 +33,7 @@
toolResult?: string;
}
let { content, isStreaming = false }: Props = $props();
let { content, message, isStreaming = false }: Props = $props();
const sections = $derived(parseAgenticContent(content));
@ -184,7 +186,7 @@
{#each sections as section, index (index)}
{#if section.type === AgenticSectionType.TEXT}
<div class="agentic-text">
<MarkdownContent content={section.content} />
<MarkdownContent content={section.content} {message} />
</div>
{:else if section.type === AgenticSectionType.TOOL_CALL_STREAMING}
{@const streamingIcon = isStreaming ? Loader2 : AlertTriangle}

View File

@ -183,9 +183,9 @@
{#if showRawOutput}
<pre class="raw-output">{messageContent || ''}</pre>
{:else if isAgenticContent}
<AgenticContent content={messageContent || ''} isStreaming={isChatStreaming()} />
<AgenticContent content={messageContent || ''} isStreaming={isChatStreaming()} {message} />
{:else}
<MarkdownContent content={messageContent || ''} />
<MarkdownContent content={messageContent || ''} {message} />
{/if}
{:else}
<div class="text-sm whitespace-pre-wrap">

View File

@ -14,6 +14,7 @@
import { rehypeRestoreTableHtml } from '$lib/markdown/table-html-restorer';
import { rehypeEnhanceLinks } from '$lib/markdown/enhance-links';
import { rehypeEnhanceCodeBlocks } from '$lib/markdown/enhance-code-blocks';
import { rehypeResolveAttachmentImages } from '$lib/markdown/resolve-attachment-images';
import { remarkLiteralHtml } from '$lib/markdown/literal-html';
import { copyCodeToClipboard, preprocessLaTeX } from '$lib/utils';
import '$styles/katex-custom.scss';
@ -21,9 +22,11 @@
import githubLightCss from 'highlight.js/styles/github.css?inline';
import { mode } from 'mode-watcher';
import CodePreviewDialog from './CodePreviewDialog.svelte';
import type { DatabaseMessage } from '$lib/types/database';
import { getImageErrorFallbackHtml } from '$lib/utils/image-error-fallback';
interface Props {
message?: DatabaseMessage;
content: string;
class?: string;
disableMath?: boolean;
@ -34,7 +37,7 @@
html: string;
}
let { content, class: className = '', disableMath = false }: Props = $props();
let { content, message, class: className = '', disableMath = false }: Props = $props();
let containerRef = $state<HTMLDivElement>();
let renderedBlocks = $state<MarkdownBlock[]>([]);
@ -49,6 +52,8 @@
const themeStyleId = `highlight-theme-${(window.idxThemeStyle = (window.idxThemeStyle ?? 0) + 1)}`;
let processor = $derived(() => {
// Force reactivity on message changes
void message;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let proc: any = remark().use(remarkGfm); // GitHub Flavored Markdown
@ -70,6 +75,7 @@
.use(rehypeRestoreTableHtml) // Restore limited HTML (e.g., <br>, <ul>) inside Markdown tables
.use(rehypeEnhanceLinks) // Add target="_blank" to links
.use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions
.use(rehypeResolveAttachmentImages, { message })
.use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string
});

View File

@ -0,0 +1,32 @@
import type { Root as HastRoot } from 'hast';
import { visit } from 'unist-util-visit';
import type { DatabaseMessage, DatabaseMessageExtraImageFile } from '$lib/types/database';
/**
* Rehype plugin to resolve attachment image sources.
* Converts attachment names (e.g., "mcp-attachment-xxx.png") to base64 data URLs.
*/
export function rehypeResolveAttachmentImages(options: { message?: DatabaseMessage }) {
return (tree: HastRoot) => {
visit(tree, 'element', (node) => {
if (node.tagName === 'img' && node.properties?.src) {
const src = String(node.properties.src);
// Skip data URLs and external URLs
if (src.startsWith('data:') || src.startsWith('http')) {
return;
}
// Find matching attachment
const attachment = options.message?.extra?.find(
(a): a is DatabaseMessageExtraImageFile => a.type === 'IMAGE' && a.name === src
);
// Replace with base64 URL if found
if (attachment?.base64Url) {
node.properties.src = attachment.base64Url;
}
}
});
};
}

View File

@ -619,10 +619,14 @@ export class ChatService {
});
}
const imageFiles = message.extra.filter(
(extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile =>
extra.type === AttachmentType.IMAGE
);
// Only include images for user messages (assistant images are for display only)
const imageFiles =
message.role === 'user'
? message.extra.filter(
(extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile =>
extra.type === AttachmentType.IMAGE
)
: [];
for (const image of imageFiles) {
contentParts.push({