feat: resolve MCP attachment images via rehype plugin
LLM can reference tool-generated images using markdown links like, plugin resolves attachment names to base64 from message.extra when present, regular HTTP/data URLs pass through unchanged (no regression) - rehypeResolveAttachmentImages plugin in markdown pipeline - Pass message prop to MarkdownContent and AgenticContent - Force processor reactivity on message.extra changes - Filter assistant images from API context (display-only)
This commit is contained in:
parent
a3c2144c1d
commit
db37b712b2
|
|
@ -17,8 +17,10 @@
|
|||
import { AgenticSectionType } from '$lib/enums';
|
||||
import { AGENTIC_TAGS, AGENTIC_REGEX } from '$lib/constants/agentic';
|
||||
import { formatJsonPretty } from '$lib/utils/formatters';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
|
||||
interface Props {
|
||||
message?: DatabaseMessage;
|
||||
content: string;
|
||||
isStreaming?: boolean;
|
||||
}
|
||||
|
|
@ -31,7 +33,7 @@
|
|||
toolResult?: string;
|
||||
}
|
||||
|
||||
let { content, isStreaming = false }: Props = $props();
|
||||
let { content, message, isStreaming = false }: Props = $props();
|
||||
|
||||
const sections = $derived(parseAgenticContent(content));
|
||||
|
||||
|
|
@ -184,7 +186,7 @@
|
|||
{#each sections as section, index (index)}
|
||||
{#if section.type === AgenticSectionType.TEXT}
|
||||
<div class="agentic-text">
|
||||
<MarkdownContent content={section.content} />
|
||||
<MarkdownContent content={section.content} {message} />
|
||||
</div>
|
||||
{:else if section.type === AgenticSectionType.TOOL_CALL_STREAMING}
|
||||
{@const streamingIcon = isStreaming ? Loader2 : AlertTriangle}
|
||||
|
|
|
|||
|
|
@ -183,9 +183,9 @@
|
|||
{#if showRawOutput}
|
||||
<pre class="raw-output">{messageContent || ''}</pre>
|
||||
{:else if isAgenticContent}
|
||||
<AgenticContent content={messageContent || ''} isStreaming={isChatStreaming()} />
|
||||
<AgenticContent content={messageContent || ''} isStreaming={isChatStreaming()} {message} />
|
||||
{:else}
|
||||
<MarkdownContent content={messageContent || ''} />
|
||||
<MarkdownContent content={messageContent || ''} {message} />
|
||||
{/if}
|
||||
{:else}
|
||||
<div class="text-sm whitespace-pre-wrap">
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
import { rehypeRestoreTableHtml } from '$lib/markdown/table-html-restorer';
|
||||
import { rehypeEnhanceLinks } from '$lib/markdown/enhance-links';
|
||||
import { rehypeEnhanceCodeBlocks } from '$lib/markdown/enhance-code-blocks';
|
||||
import { rehypeResolveAttachmentImages } from '$lib/markdown/resolve-attachment-images';
|
||||
import { remarkLiteralHtml } from '$lib/markdown/literal-html';
|
||||
import { copyCodeToClipboard, preprocessLaTeX } from '$lib/utils';
|
||||
import '$styles/katex-custom.scss';
|
||||
|
|
@ -21,9 +22,11 @@
|
|||
import githubLightCss from 'highlight.js/styles/github.css?inline';
|
||||
import { mode } from 'mode-watcher';
|
||||
import CodePreviewDialog from './CodePreviewDialog.svelte';
|
||||
import type { DatabaseMessage } from '$lib/types/database';
|
||||
import { getImageErrorFallbackHtml } from '$lib/utils/image-error-fallback';
|
||||
|
||||
interface Props {
|
||||
message?: DatabaseMessage;
|
||||
content: string;
|
||||
class?: string;
|
||||
disableMath?: boolean;
|
||||
|
|
@ -34,7 +37,7 @@
|
|||
html: string;
|
||||
}
|
||||
|
||||
let { content, class: className = '', disableMath = false }: Props = $props();
|
||||
let { content, message, class: className = '', disableMath = false }: Props = $props();
|
||||
|
||||
let containerRef = $state<HTMLDivElement>();
|
||||
let renderedBlocks = $state<MarkdownBlock[]>([]);
|
||||
|
|
@ -49,6 +52,8 @@
|
|||
const themeStyleId = `highlight-theme-${(window.idxThemeStyle = (window.idxThemeStyle ?? 0) + 1)}`;
|
||||
|
||||
let processor = $derived(() => {
|
||||
// Force reactivity on message changes
|
||||
void message;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let proc: any = remark().use(remarkGfm); // GitHub Flavored Markdown
|
||||
|
||||
|
|
@ -70,6 +75,7 @@
|
|||
.use(rehypeRestoreTableHtml) // Restore limited HTML (e.g., <br>, <ul>) inside Markdown tables
|
||||
.use(rehypeEnhanceLinks) // Add target="_blank" to links
|
||||
.use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions
|
||||
.use(rehypeResolveAttachmentImages, { message })
|
||||
.use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
import type { Root as HastRoot } from 'hast';
|
||||
import { visit } from 'unist-util-visit';
|
||||
import type { DatabaseMessage, DatabaseMessageExtraImageFile } from '$lib/types/database';
|
||||
|
||||
/**
|
||||
* Rehype plugin to resolve attachment image sources.
|
||||
* Converts attachment names (e.g., "mcp-attachment-xxx.png") to base64 data URLs.
|
||||
*/
|
||||
export function rehypeResolveAttachmentImages(options: { message?: DatabaseMessage }) {
|
||||
return (tree: HastRoot) => {
|
||||
visit(tree, 'element', (node) => {
|
||||
if (node.tagName === 'img' && node.properties?.src) {
|
||||
const src = String(node.properties.src);
|
||||
|
||||
// Skip data URLs and external URLs
|
||||
if (src.startsWith('data:') || src.startsWith('http')) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Find matching attachment
|
||||
const attachment = options.message?.extra?.find(
|
||||
(a): a is DatabaseMessageExtraImageFile => a.type === 'IMAGE' && a.name === src
|
||||
);
|
||||
|
||||
// Replace with base64 URL if found
|
||||
if (attachment?.base64Url) {
|
||||
node.properties.src = attachment.base64Url;
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
|
@ -619,10 +619,14 @@ export class ChatService {
|
|||
});
|
||||
}
|
||||
|
||||
const imageFiles = message.extra.filter(
|
||||
(extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile =>
|
||||
extra.type === AttachmentType.IMAGE
|
||||
);
|
||||
// Only include images for user messages (assistant images are for display only)
|
||||
const imageFiles =
|
||||
message.role === 'user'
|
||||
? message.extra.filter(
|
||||
(extra: DatabaseMessageExtra): extra is DatabaseMessageExtraImageFile =>
|
||||
extra.type === AttachmentType.IMAGE
|
||||
)
|
||||
: [];
|
||||
|
||||
for (const image of imageFiles) {
|
||||
contentParts.push({
|
||||
|
|
|
|||
Loading…
Reference in New Issue