feat: Condition available models based on modality + better model loading strategy & UX

tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte

@@ -13,7 +13,8 @@
 	import { modelsStore, modelOptions, selectedModelId } from '$lib/stores/models.svelte';
 	import { isRouterMode } from '$lib/stores/server.svelte';
 	import { chatStore } from '$lib/stores/chat.svelte';
-	import { activeMessages } from '$lib/stores/conversations.svelte';
+	import { activeMessages, usedModalities } from '$lib/stores/conversations.svelte';
+	import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
 	import type { ChatUploadedFile } from '$lib/types/chat';
 
 	interface Props {
@@ -157,6 +158,15 @@
 	export function openModelSelector() {
 		selectorModelRef?.open();
 	}
+
+	const { handleModelChange } = useModelChangeValidation({
+		getRequiredModalities: () => usedModalities(),
+		onValidationFailure: async (previousModelId) => {
+			if (previousModelId) {
+				await modelsStore.selectModelById(previousModelId);
+			}
+		}
+	});
 </script>
 
 <div class="flex w-full items-center gap-3 {className}" style="container-type: inline-size">
@@ -173,6 +183,7 @@
 		currentModel={conversationModel}
 		forceForegroundText={true}
 		useGlobalSelection={true}
+		onModelChange={handleModelChange}
 	/>
 
 	{#if isLoading}

tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte

@@ -9,6 +9,7 @@
 		SelectorModel
 	} from '$lib/components/app';
 	import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
+	import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
 	import { isLoading } from '$lib/stores/chat.svelte';
 	import autoResizeTextarea from '$lib/utils/autoresize-textarea';
 	import { fade } from 'svelte/transition';
@@ -18,8 +19,8 @@
 	import { INPUT_CLASSES } from '$lib/constants/input-classes';
 	import Label from '$lib/components/ui/label/label.svelte';
 	import { config } from '$lib/stores/settings.svelte';
+	import { conversationsStore } from '$lib/stores/conversations.svelte';
 	import { isRouterMode } from '$lib/stores/server.svelte';
-	import { modelsStore } from '$lib/stores/models.svelte';
 	import { copyToClipboard } from '$lib/utils/copy';
 	import type { ApiChatCompletionToolCall } from '$lib/types/api';
 
@@ -93,7 +94,6 @@
 	let currentConfig = $derived(config());
 	let isRouter = $derived(isRouterMode());
 	let displayedModel = $derived((): string | null => {
-		// Only show model from streaming data, no fallbacks to server props
 		if (message.model) {
 			return message.model;
 		}
@@ -101,16 +101,10 @@
 		return null;
 	});
 
-	async function handleModelChange(modelId: string, modelName: string) {
+	const { handleModelChange } = useModelChangeValidation({
-		try {
+		getRequiredModalities: () => conversationsStore.getModalitiesUpToMessage(message.id),
-			await modelsStore.selectModelById(modelId);
+		onSuccess: (modelName) => onRegenerate(modelName)
-
+	});
-			// Pass the selected model name for regeneration
-			onRegenerate(modelName);
-		} catch (error) {
-			console.error('Failed to change model:', error);
-		}
-	}
 
 	function handleCopyModel() {
 		const model = displayedModel();
@@ -258,6 +252,7 @@
 						currentModel={displayedModel()}
 						onModelChange={handleModelChange}
 						disabled={isLoading()}
+						upToMessageId={message.id}
 					/>
 				{:else}
 					<BadgeModelName model={displayedModel() || undefined} onclick={handleCopyModel} />

tools/server/webui/src/lib/components/app/misc/SelectorModel.svelte

@@ -1,6 +1,7 @@
 <script lang="ts">
 	import { onMount, tick } from 'svelte';
-	import { ChevronDown, Loader2, Package, Power } from '@lucide/svelte';
+	import { ChevronDown, EyeOff, Loader2, MicOff, Package, Power } from '@lucide/svelte';
+	import * as Tooltip from '$lib/components/ui/tooltip';
 	import { cn } from '$lib/components/ui/utils';
 	import { portalToBody } from '$lib/utils/portal-to-body';
 	import {
@@ -11,6 +12,7 @@
 		selectedModelId,
 		routerModels
 	} from '$lib/stores/models.svelte';
+	import { usedModalities, conversationsStore } from '$lib/stores/conversations.svelte';
 	import { ServerModelStatus } from '$lib/enums';
 	import { isRouterMode, serverStore } from '$lib/stores/server.svelte';
 	import { DialogModelInformation } from '$lib/components/app';
@@ -19,11 +21,18 @@
 	interface Props {
 		class?: string;
 		currentModel?: string | null;
-		onModelChange?: (modelId: string, modelName: string) => void;
+		/** Callback when model changes. Return false to keep menu open (e.g., for validation failures) */
+		onModelChange?: (modelId: string, modelName: string) => Promise<boolean> | boolean | void;
 		disabled?: boolean;
 		forceForegroundText?: boolean;
 		/** When true, user's global selection takes priority over currentModel (for form selector) */
 		useGlobalSelection?: boolean;
+		/**
+		 * When provided, only consider modalities from messages BEFORE this message.
+		 * Used for regeneration - allows selecting models that don't support modalities
+		 * used in later messages.
+		 */
+		upToMessageId?: string;
 	}
 
 	let {
@@ -32,7 +41,8 @@
 		onModelChange,
 		disabled = false,
 		forceForegroundText = false,
-		useGlobalSelection = false
+		useGlobalSelection = false,
+		upToMessageId
 	}: Props = $props();
 
 	let options = $derived(modelOptions());
@@ -45,12 +55,47 @@
 	// Reactive router models state - needed for proper reactivity of status checks
 	let currentRouterModels = $derived(routerModels());
 
-	// Helper to get model status from server - establishes reactive dependency
+	let requiredModalities = $derived(
+		upToMessageId ? conversationsStore.getModalitiesUpToMessage(upToMessageId) : usedModalities()
+	);
+
 	function getModelStatus(modelId: string): ServerModelStatus | null {
-		const model = currentRouterModels.find((m) => m.name === modelId);
+		const model = currentRouterModels.find((m) => m.id === modelId);
 		return (model?.status?.value as ServerModelStatus) ?? null;
 	}
 
+	/**
+	 * Checks if a model supports all modalities used in the conversation.
+	 * Returns true if the model can be selected, false if it should be disabled.
+	 */
+	function isModelCompatible(option: ModelOption): boolean {
+		const modelModalities = option.modalities;
+
+		if (!modelModalities) return true;
+
+		if (requiredModalities.vision && !modelModalities.vision) return false;
+		if (requiredModalities.audio && !modelModalities.audio) return false;
+
+		return true;
+	}
+
+	/**
+	 * Gets missing modalities for a model.
+	 * Returns object with vision/audio booleans indicating what's missing.
+	 */
+	function getMissingModalities(option: ModelOption): { vision: boolean; audio: boolean } | null {
+		const modelModalities = option.modalities;
+		if (!modelModalities) return null;
+
+		const missing = {
+			vision: requiredModalities.vision && !modelModalities.vision,
+			audio: requiredModalities.audio && !modelModalities.audio
+		};
+
+		if (!missing.vision && !missing.audio) return null;
+		return missing;
+	}
+
 	let isHighlightedCurrentModelActive = $derived(
 		!isRouter || !currentModel
 			? false
@@ -251,23 +296,32 @@
 		const option = options.find((opt) => opt.id === modelId);
 		if (!option) return;
 
-		closeMenu();
+		let shouldCloseMenu = true;
 
 		if (onModelChange) {
 			// If callback provided, use it (for regenerate functionality)
-			onModelChange(option.id, option.model);
+			const result = await onModelChange(option.id, option.model);
+
+			// If callback returns false, keep menu open (validation failed)
+			if (result === false) {
+				shouldCloseMenu = false;
+			}
 		} else {
 			// Update global selection
 			await modelsStore.selectModelById(option.id);
+
+			// Load the model if not already loaded (router mode)
+			if (isRouter && getModelStatus(option.model) !== ServerModelStatus.LOADED) {
+				try {
+					await modelsStore.loadModel(option.model);
+				} catch (error) {
+					console.error('Failed to load model:', error);
+				}
+			}
 		}
 
-		// Load the model if not already loaded (router mode)
+		if (shouldCloseMenu) {
-		if (isRouter && getModelStatus(option.model) !== ServerModelStatus.LOADED) {
+			closeMenu();
-			try {
-				await modelsStore.loadModel(option.model);
-			} catch (error) {
-				console.error('Failed to load model:', error);
-			}
 		}
 	}
 
@@ -405,20 +459,28 @@
 							{@const isLoaded = status === ServerModelStatus.LOADED}
 							{@const isLoading = status === ServerModelStatus.LOADING}
 							{@const isSelected = currentModel === option.model || activeId === option.id}
+							{@const isCompatible = isModelCompatible(option)}
+							{@const missingModalities = getMissingModalities(option)}
 							<div
 								class={cn(
-									'group flex w-full cursor-pointer items-center gap-2 px-3 py-2 text-left text-sm transition hover:bg-muted focus:bg-muted focus:outline-none',
+									'group flex w-full items-center gap-2 px-3 py-2 text-left text-sm transition focus:outline-none',
+									isCompatible
+										? 'cursor-pointer hover:bg-muted focus:bg-muted'
+										: 'cursor-not-allowed opacity-50',
 									isSelected
 										? 'bg-accent text-accent-foreground'
-										: 'hover:bg-accent hover:text-accent-foreground',
+										: isCompatible
+											? 'hover:bg-accent hover:text-accent-foreground'
+											: '',
 									isLoaded ? 'text-popover-foreground' : 'text-muted-foreground'
 								)}
 								role="option"
 								aria-selected={isSelected}
-								tabindex="0"
+								aria-disabled={!isCompatible}
-								onclick={() => handleSelect(option.id)}
+								tabindex={isCompatible ? 0 : -1}
+								onclick={() => isCompatible && handleSelect(option.id)}
 								onkeydown={(e) => {
-									if (e.key === 'Enter' || e.key === ' ') {
+									if (isCompatible && (e.key === 'Enter' || e.key === ' ')) {
 										e.preventDefault();
 										handleSelect(option.id);
 									}
@@ -426,29 +488,65 @@
 							>
 								<span class="min-w-0 flex-1 truncate">{option.model}</span>
 
-								{#if isLoading}
+								{#if missingModalities}
-									<Loader2 class="h-4 w-4 shrink-0 animate-spin text-muted-foreground" />
+									<span class="flex shrink-0 items-center gap-1 text-muted-foreground/70">
-								{:else if isLoaded}
+										{#if missingModalities.vision}
-									<!-- Green dot, on hover show red unload button -->
+											<Tooltip.Root>
-									<button
+												<Tooltip.Trigger>
-										type="button"
+													<EyeOff class="h-3.5 w-3.5" />
-										class="relative ml-2 flex h-4 w-4 shrink-0 items-center justify-center"
+												</Tooltip.Trigger>
-										onclick={(e) => {
+												<Tooltip.Content class="z-[9999]">
-											e.stopPropagation();
+													<p>No vision support</p>
-											modelsStore.unloadModel(option.model);
+												</Tooltip.Content>
-										}}
+											</Tooltip.Root>
-										title="Unload model"
+										{/if}
-									>
+										{#if missingModalities.audio}
-										<span
+											<Tooltip.Root>
-											class="mr-2 h-2 w-2 rounded-full bg-green-500 transition-opacity group-hover:opacity-0"
+												<Tooltip.Trigger>
-										></span>
+													<MicOff class="h-3.5 w-3.5" />
+												</Tooltip.Trigger>
+												<Tooltip.Content class="z-[9999]">
+													<p>No audio support</p>
+												</Tooltip.Content>
+											</Tooltip.Root>
+										{/if}
+									</span>
+								{/if}
 
-										<Power
+								{#if isLoading}
-											class="absolute mr-2 h-4 w-4 text-red-500 opacity-0 transition-opacity group-hover:opacity-100 hover:text-red-600"
+									<Tooltip.Root>
-										/>
+										<Tooltip.Trigger>
-									</button>
+											<Loader2 class="h-4 w-4 shrink-0 animate-spin text-muted-foreground" />
+										</Tooltip.Trigger>
+										<Tooltip.Content class="z-[9999]">
+											<p>Loading model...</p>
+										</Tooltip.Content>
+									</Tooltip.Root>
+								{:else if isLoaded}
+									<Tooltip.Root>
+										<Tooltip.Trigger>
+											<button
+												type="button"
+												class="relative ml-2 flex h-4 w-4 shrink-0 items-center justify-center"
+												onclick={(e) => {
+													e.stopPropagation();
+													modelsStore.unloadModel(option.model);
+												}}
+											>
+												<span
+													class="mr-2 h-2 w-2 rounded-full bg-green-500 transition-opacity group-hover:opacity-0"
+												></span>
+												<Power
+													class="absolute mr-2 h-4 w-4 text-red-500 opacity-0 transition-opacity group-hover:opacity-100 hover:text-red-600"
+												/>
+											</button>
+										</Tooltip.Trigger>
+										<Tooltip.Content class="z-[9999]">
+											<p>Unload model</p>
+										</Tooltip.Content>
+									</Tooltip.Root>
 								{:else}
-									<span class="mx-2 h-2 w-2 shrink-0 rounded-full bg-muted-foreground/50"></span>
+									<span class="mx-2 h-2 w-2 rounded-full bg-muted-foreground/50"></span>
 								{/if}
 							</div>
 						{/each}

tools/server/webui/src/lib/hooks/use-model-change-validation.svelte.ts

@@ -0,0 +1,119 @@
+import { modelsStore } from '$lib/stores/models.svelte';
+import { isRouterMode } from '$lib/stores/server.svelte';
+import { toast } from 'svelte-sonner';
+import type { ModelModalities } from '$lib/types/models';
+
+interface UseModelChangeValidationOptions {
+	/**
+	 * Function to get required modalities for validation.
+	 * For ChatForm: () => usedModalities() - all messages
+	 * For ChatMessageAssistant: () => getModalitiesUpToMessage(messageId) - messages before
+	 */
+	getRequiredModalities: () => ModelModalities;
+
+	/**
+	 * Optional callback to execute after successful validation.
+	 * For ChatForm: undefined - just select model
+	 * For ChatMessageAssistant: (modelName) => onRegenerate(modelName)
+	 */
+	onSuccess?: (modelName: string) => void;
+
+	/**
+	 * Optional callback for rollback on validation failure.
+	 * For ChatForm: (previousId) => selectModelById(previousId)
+	 * For ChatMessageAssistant: undefined - no rollback needed
+	 */
+	onValidationFailure?: (previousModelId: string | null) => Promise<void>;
+}
+
+export function useModelChangeValidation(options: UseModelChangeValidationOptions) {
+	const { getRequiredModalities, onSuccess, onValidationFailure } = options;
+
+	let previousSelectedModelId: string | null = null;
+	const isRouter = $derived(isRouterMode());
+
+	async function handleModelChange(modelId: string, modelName: string): Promise<boolean> {
+		try {
+			// Store previous selection for potential rollback
+			if (onValidationFailure) {
+				previousSelectedModelId = modelsStore.selectedModelId;
+			}
+
+			// Load model if not already loaded (router mode only)
+			let hasLoadedModel = false;
+			const isModelLoadedBefore = modelsStore.isModelLoaded(modelName);
+
+			if (isRouter && !isModelLoadedBefore) {
+				try {
+					await modelsStore.loadModel(modelName);
+					hasLoadedModel = true;
+				} catch {
+					toast.error(`Failed to load model "${modelName}"`);
+					return false;
+				}
+			}
+
+			// Fetch model props to validate modalities
+			const props = await modelsStore.fetchModelProps(modelName);
+
+			if (props?.modalities) {
+				const requiredModalities = getRequiredModalities();
+
+				// Check if model supports required modalities
+				const missingModalities: string[] = [];
+				if (requiredModalities.vision && !props.modalities.vision) {
+					missingModalities.push('vision');
+				}
+				if (requiredModalities.audio && !props.modalities.audio) {
+					missingModalities.push('audio');
+				}
+
+				if (missingModalities.length > 0) {
+					toast.error(
+						`Model "${modelName}" doesn't support required modalities: ${missingModalities.join(', ')}. Please select a different model.`
+					);
+
+					// Unload the model if we just loaded it
+					if (isRouter && hasLoadedModel) {
+						try {
+							await modelsStore.unloadModel(modelName);
+						} catch (error) {
+							console.error('Failed to unload incompatible model:', error);
+						}
+					}
+
+					// Execute rollback callback if provided
+					if (onValidationFailure && previousSelectedModelId) {
+						await onValidationFailure(previousSelectedModelId);
+					}
+
+					return false;
+				}
+			}
+
+			// Select the model (validation passed)
+			await modelsStore.selectModelById(modelId);
+
+			// Execute success callback if provided
+			if (onSuccess) {
+				onSuccess(modelName);
+			}
+
+			return true;
+		} catch (error) {
+			console.error('Failed to change model:', error);
+			toast.error('Failed to validate model capabilities');
+
+			// Execute rollback callback on error if provided
+			if (onValidationFailure && previousSelectedModelId) {
+				await onValidationFailure(previousSelectedModelId);
+			}
+
+			return false;
+		}
+	}
+
+	return {
+		handleModelChange
+	};
+}

tools/server/webui/src/lib/stores/conversations.svelte.ts

@@ -4,11 +4,13 @@ import { toast } from 'svelte-sonner';
 import { DatabaseService } from '$lib/services/database';
 import { config } from '$lib/stores/settings.svelte';
 import { filterByLeafNodeId, findLeafNode } from '$lib/utils/branching';
+import { AttachmentType } from '$lib/enums';
 import type {
 	DatabaseConversation,
 	DatabaseMessage,
 	ExportedConversations
 } from '$lib/types/database';
+import type { ModelModalities } from '$lib/types/models';
 
 /**
  * conversationsStore - Persistent conversation data and lifecycle management
@@ -62,6 +64,55 @@ class ConversationsStore {
 	/** Whether the store has been initialized */
 	isInitialized = $state(false);
 
+	/**
+	 * Modalities used in the active conversation.
+	 * Computed from attachments in activeMessages.
+	 * Used to filter available models - models must support all used modalities.
+	 */
+	usedModalities: ModelModalities = $derived.by(() => {
+		return this.calculateModalitiesFromMessages(this.activeMessages);
+	});
+
+	/**
+	 * Calculate modalities from a list of messages.
+	 * Helper method used by both usedModalities and getModalitiesUpToMessage.
+	 */
+	private calculateModalitiesFromMessages(messages: DatabaseMessage[]): ModelModalities {
+		const modalities: ModelModalities = { vision: false, audio: false };
+
+		for (const message of messages) {
+			if (!message.extra) continue;
+
+			for (const extra of message.extra) {
+				if (extra.type === AttachmentType.IMAGE || extra.type === AttachmentType.PDF) {
+					modalities.vision = true;
+				}
+				if (extra.type === AttachmentType.AUDIO) {
+					modalities.audio = true;
+				}
+			}
+
+			if (modalities.vision && modalities.audio) break;
+		}
+
+		return modalities;
+	}
+
+	/**
+	 * Get modalities used in messages BEFORE the specified message.
+	 * Used for regeneration - only consider context that was available when generating this message.
+	 */
+	getModalitiesUpToMessage(messageId: string): ModelModalities {
+		const messageIndex = this.activeMessages.findIndex((m) => m.id === messageId);
+
+		if (messageIndex === -1) {
+			return this.usedModalities;
+		}
+
+		const messagesBefore = this.activeMessages.slice(0, messageIndex);
+		return this.calculateModalitiesFromMessages(messagesBefore);
+	}
+
 	/** Callback for title update confirmation dialog */
 	titleUpdateConfirmationCallback?: (currentTitle: string, newTitle: string) => Promise<boolean>;
 
@@ -537,3 +588,4 @@ export const conversations = () => conversationsStore.conversations;
 export const activeConversation = () => conversationsStore.activeConversation;
 export const activeMessages = () => conversationsStore.activeMessages;
 export const isConversationsInitialized = () => conversationsStore.isInitialized;
+export const usedModalities = () => conversationsStore.usedModalities;

tools/server/webui/src/lib/stores/models.svelte.ts

@@ -68,7 +68,7 @@ class ModelsStore {
 	get loadedModelIds(): string[] {
 		return this.routerModels
 			.filter((m) => m.status.value === ServerModelStatus.LOADED)
-			.map((m) => m.name);
+			.map((m) => m.id);
 	}
 
 	get loadingModelIds(): string[] {
@@ -137,7 +137,7 @@ class ModelsStore {
 	// ─────────────────────────────────────────────────────────────────────────────
 
 	isModelLoaded(modelId: string): boolean {
-		const model = this.routerModels.find((m) => m.name === modelId);
+		const model = this.routerModels.find((m) => m.id === modelId);
 		return model?.status.value === ServerModelStatus.LOADED || false;
 	}
 
@@ -146,7 +146,7 @@ class ModelsStore {
 	}
 
 	getModelStatus(modelId: string): ServerModelStatus | null {
-		const model = this.routerModels.find((m) => m.name === modelId);
+		const model = this.routerModels.find((m) => m.id === modelId);
 		return model?.status.value ?? null;
 	}
 

tools/server/webui/src/lib/types/api.d.ts

@@ -54,8 +54,8 @@ export interface ApiModelStatus {
 export interface ApiModelDataEntry {
 	/** Model identifier (e.g., "ggml-org/Qwen2.5-Omni-7B-GGUF:latest") */
 	id: string;
-	/** Model name (usually same as id) */
+	/** Model name (optional, usually same as id - not always returned by API) */
-	name: string;
+	name?: string;
 	/** Object type, always "model" */
 	object: string;
 	/** Owner, usually "llamacpp" */