happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: remove multimodal validation from model selector 

Remove all frontend validation logic that prevented users from selecting
models based on multimodal capabilities. This refactoring removes
restrictive UI code while maintaining full functionality

- Vision models can describe images as text
- That text remains useful for non-vision models
- Chaining vision -> non-vision is a valid workflow
- Users know their use case better than the UI
- Users can return to vision models when needed
This commit is contained in:
Pascal 2026-01-18 10:40:30 +01:00
parent 5c28b7a2ee
commit d8af98f1ed
7 changed files with 24 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View File

@ -15,8 +15,7 @@
import { modelsStore, modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { activeMessages, usedModalities } from '$lib/stores/conversations.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
interface Props {
canSend?: boolean;
@ -157,15 +156,6 @@
selectorModelRef?.open();
}
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities: () => usedModalities(),
onValidationFailure: async (previousModelId) => {
if (previousModelId) {
await modelsStore.selectModelById(previousModelId);
}
}
});
let showMcpDialog = $state(false);
</script>
@ -189,7 +179,6 @@
currentModel={conversationModel}
forceForegroundText={true}
useGlobalSelection={true}
onModelChange={handleModelChange}
/>
</div>

14
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte
View File

@ -8,7 +8,6 @@
ModelsSelector
} from '$lib/components/app';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
import { agenticStreamingToolCall } from '$lib/stores/agentic.svelte';
import { autoResizeTextarea, copyToClipboard } from '$lib/utils';
@ -20,7 +19,6 @@
import { MessageRole } from '$lib/enums';
import Label from '$lib/components/ui/label/label.svelte';
import { config } from '$lib/stores/settings.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { AGENTIC_TAGS, REASONING_TAGS } from '$lib/constants/agentic';
@ -97,11 +95,6 @@
let displayedModel = $derived(message.model ?? null);
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities: () => conversationsStore.getModalitiesUpToMessage(message.id),
onSuccess: (modelName) => onRegenerate(modelName)
});
function handleCopyModel() {
void copyToClipboard(displayedModel ?? '');
}
@ -190,12 +183,7 @@
{#if displayedModel}
<div class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground">
{#if isRouter}
<ModelsSelector
currentModel={displayedModel}
onModelChange={handleModelChange}
disabled={isLoading()}
upToMessageId={message.id}
/>
<ModelsSelector currentModel={displayedModel} disabled={isLoading()} />
{:else}
<ModelBadge model={displayedModel || undefined} onclick={handleCopyModel} />
{/if}

73
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageEditForm.svelte
View File

@ -5,22 +5,13 @@
import { ChatAttachmentsList, DialogConfirmation, ModelsSelector } from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/css-classes';
import { SETTING_CONFIG_DEFAULT } from '$lib/constants/settings-config';
import { AttachmentType, FileTypeCategory, MimeTypeText } from '$lib/enums';
import { MimeTypeText } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { modelsStore } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import {
autoResizeTextarea,
getFileTypeCategory,
getFileTypeCategoryByExtension,
parseClipboardContent
} from '$lib/utils';
import { autoResizeTextarea, parseClipboardContent } from '$lib/utils';
interface Props {
messageId: string;
editedContent: string;
editedExtras?: DatabaseMessageExtra[];
editedUploadedFiles?: ChatUploadedFile[];
@ -38,7 +29,6 @@
}
let {
messageId,
editedContent,
editedExtras = [],
editedUploadedFiles = [],
@ -87,59 +77,6 @@
let canSubmit = $derived(editedContent.trim().length > 0 || hasAttachments);
function getEditedAttachmentsModalities(): ModelModalities {
const modalities: ModelModalities = { vision: false, audio: false };
for (const extra of editedExtras) {
if (extra.type === AttachmentType.IMAGE) {
modalities.vision = true;
}
if (
extra.type === AttachmentType.PDF &&
'processedAsImages' in extra &&
extra.processedAsImages
) {
modalities.vision = true;
}
if (extra.type === AttachmentType.AUDIO) {
modalities.audio = true;
}
}
for (const file of editedUploadedFiles) {
const category = getFileTypeCategory(file.type) || getFileTypeCategoryByExtension(file.name);
if (category === FileTypeCategory.IMAGE) {
modalities.vision = true;
}
if (category === FileTypeCategory.AUDIO) {
modalities.audio = true;
}
}
return modalities;
}
function getRequiredModalities(): ModelModalities {
const beforeModalities = conversationsStore.getModalitiesUpToMessage(messageId);
const editedModalities = getEditedAttachmentsModalities();
return {
vision: beforeModalities.vision || editedModalities.vision,
audio: beforeModalities.audio || editedModalities.audio
};
}
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities,
onValidationFailure: async (previousModelId) => {
if (previousModelId) {
await modelsStore.selectModelById(previousModelId);
}
}
});
function handleFileInputChange(event: Event) {
const input = event.target as HTMLInputElement;
if (!input.files || input.files.length === 0) return;
@ -336,11 +273,7 @@
<div class="flex-1"></div>
{#if isRouter}
<ModelsSelector
forceForegroundText={true}
useGlobalSelection={true}
onModelChange={handleModelChange}
/>
<ModelsSelector forceForegroundText={true} useGlobalSelection={true} />
{/if}
<Button

1
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageUser.svelte
View File

@ -100,7 +100,6 @@
{#if isEditing}
<ChatMessageEditForm
bind:textareaElement
messageId={message.id}
{editedContent}
{editedExtras}
{editedUploadedFiles}

153
tools/server/webui/src/lib/components/app/models/ModelsSelector.svelte
View File

@ -1,6 +1,6 @@
<script lang="ts">
import { onMount } from 'svelte';
import { ChevronDown, EyeOff, Loader2, MicOff, Package, Power } from '@lucide/svelte';
import { ChevronDown, Loader2, Package, Power } from '@lucide/svelte';
import * as Tooltip from '$lib/components/ui/tooltip';
import { cn } from '$lib/components/ui/utils';
import {
@ -10,10 +10,8 @@
modelsUpdating,
selectedModelId,
routerModels,
propsCacheVersion,
singleModelName
} from '$lib/stores/models.svelte';
import { usedModalities, conversationsStore } from '$lib/stores/conversations.svelte';
import { ServerModelStatus } from '$lib/enums';
import { isRouterMode } from '$lib/stores/server.svelte';
import {
@ -32,12 +30,6 @@
forceForegroundText?: boolean;
/** When true, user's global selection takes priority over currentModel (for form selector) */
useGlobalSelection?: boolean;
/**
* When provided, only consider modalities from messages BEFORE this message.
* Used for regeneration - allows selecting models that don't support modalities
* used in later messages.
*/
upToMessageId?: string;
}
let {
@ -46,8 +38,7 @@
onModelChange,
disabled = false,
forceForegroundText = false,
useGlobalSelection = false,
upToMessageId
useGlobalSelection = false
}: Props = $props();
let options = $derived(modelOptions());
@ -60,74 +51,11 @@
// Reactive router models state - needed for proper reactivity of status checks
let currentRouterModels = $derived(routerModels());
let requiredModalities = $derived(
upToMessageId ? conversationsStore.getModalitiesUpToMessage(upToMessageId) : usedModalities()
);
function getModelStatus(modelId: string): ServerModelStatus | null {
const model = currentRouterModels.find((m) => m.id === modelId);
return (model?.status?.value as ServerModelStatus) ?? null;
}
/**
* Checks if a model supports all modalities used in the conversation.
* Returns true if the model can be selected, false if it should be disabled.
*/
function isModelCompatible(option: ModelOption): boolean {
void propsCacheVersion();
const modelModalities = modelsStore.getModelModalities(option.model);
if (!modelModalities) {
const status = getModelStatus(option.model);
if (status === ServerModelStatus.LOADED) {
if (requiredModalities.vision || requiredModalities.audio) return false;
}
return true;
}
if (requiredModalities.vision && !modelModalities.vision) return false;
if (requiredModalities.audio && !modelModalities.audio) return false;
return true;
}
/**
* Gets missing modalities for a model.
* Returns object with vision/audio booleans indicating what's missing.
*/
function getMissingModalities(option: ModelOption): { vision: boolean; audio: boolean } | null {
void propsCacheVersion();
const modelModalities = modelsStore.getModelModalities(option.model);
if (!modelModalities) {
const status = getModelStatus(option.model);
if (status === ServerModelStatus.LOADED) {
const missing = {
vision: requiredModalities.vision,
audio: requiredModalities.audio
};
if (missing.vision || missing.audio) return missing;
}
return null;
}
const missing = {
vision: requiredModalities.vision && !modelModalities.vision,
audio: requiredModalities.audio && !modelModalities.audio
};
if (!missing.vision && !missing.audio) return null;
return missing;
}
let isHighlightedCurrentModelActive = $derived(
!isRouter || !currentModel
? false
@ -159,13 +87,6 @@
})()
);
// Get indices of compatible options for keyboard navigation
let compatibleIndices = $derived(
filteredOptions
.map((option, index) => (isModelCompatible(option) ? index : -1))
.filter((i) => i !== -1)
);
// Reset highlighted index when search term changes
$effect(() => {
void searchTerm;
@ -214,34 +135,30 @@
if (event.key === 'ArrowDown') {
event.preventDefault();
if (compatibleIndices.length === 0) return;
if (filteredOptions.length === 0) return;
const currentPos = compatibleIndices.indexOf(highlightedIndex);
if (currentPos === -1 || currentPos === compatibleIndices.length - 1) {
highlightedIndex = compatibleIndices[0];
if (highlightedIndex === -1 || highlightedIndex === filteredOptions.length - 1) {
highlightedIndex = 0;
} else {
highlightedIndex = compatibleIndices[currentPos + 1];
highlightedIndex += 1;
}
} else if (event.key === 'ArrowUp') {
event.preventDefault();
if (compatibleIndices.length === 0) return;
if (filteredOptions.length === 0) return;
const currentPos = compatibleIndices.indexOf(highlightedIndex);
if (currentPos === -1 || currentPos === 0) {
highlightedIndex = compatibleIndices[compatibleIndices.length - 1];
if (highlightedIndex === -1 || highlightedIndex === 0) {
highlightedIndex = filteredOptions.length - 1;
} else {
highlightedIndex = compatibleIndices[currentPos - 1];
highlightedIndex -= 1;
}
} else if (event.key === 'Enter') {
event.preventDefault();
if (highlightedIndex >= 0 && highlightedIndex < filteredOptions.length) {
const option = filteredOptions[highlightedIndex];
if (isModelCompatible(option)) {
handleSelect(option.id);
}
} else if (compatibleIndices.length > 0) {
// No selection - highlight first compatible option
highlightedIndex = compatibleIndices[0];
handleSelect(option.id);
} else if (filteredOptions.length > 0) {
// No selection - highlight first option
highlightedIndex = 0;
}
}
}
@ -412,31 +329,24 @@
{@const isLoaded = status === ServerModelStatus.LOADED}
{@const isLoading = status === ServerModelStatus.LOADING}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isCompatible = isModelCompatible(option)}
{@const isHighlighted = index === highlightedIndex}
{@const missingModalities = getMissingModalities(option)}
<div
class={cn(
'group flex w-full items-center gap-2 rounded-sm p-2 text-left text-sm transition focus:outline-none',
isCompatible
? 'cursor-pointer hover:bg-muted focus:bg-muted'
: 'cursor-not-allowed opacity-50',
'cursor-pointer hover:bg-muted focus:bg-muted',
isSelected || isHighlighted
? 'bg-accent text-accent-foreground'
: isCompatible
? 'hover:bg-accent hover:text-accent-foreground'
: '',
: 'hover:bg-accent hover:text-accent-foreground',
isLoaded ? 'text-popover-foreground' : 'text-muted-foreground'
)}
role="option"
aria-selected={isSelected || isHighlighted}
aria-disabled={!isCompatible}
tabindex={isCompatible ? 0 : -1}
onclick={() => isCompatible && handleSelect(option.id)}
tabindex="0"
onclick={() => handleSelect(option.id)}
onmouseenter={() => (highlightedIndex = index)}
onkeydown={(e) => {
if (isCompatible && (e.key === 'Enter' || e.key === ' ')) {
if (e.key === 'Enter' || e.key === ' ') {
e.preventDefault();
handleSelect(option.id);
}
@ -444,31 +354,6 @@
>
<TruncatedText text={option.model} class="min-w-0 flex-1 text-left" />
{#if missingModalities}
<span class="flex shrink-0 items-center gap-1 text-muted-foreground/70">
{#if missingModalities.vision}
<Tooltip.Root>
<Tooltip.Trigger>
<EyeOff class="h-3.5 w-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>No vision support</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
{#if missingModalities.audio}
<Tooltip.Root>
<Tooltip.Trigger>
<MicOff class="h-3.5 w-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>No audio support</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
</span>
{/if}
{#if isLoading}
<Tooltip.Root>
<Tooltip.Trigger>

118
tools/server/webui/src/lib/hooks/use-model-change-validation.svelte.ts
View File

@ -1,118 +0,0 @@
import { modelsStore } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { toast } from 'svelte-sonner';
interface UseModelChangeValidationOptions {
/**
* Function to get required modalities for validation.
* For ChatForm: () => usedModalities() - all messages
* For ChatMessageAssistant: () => getModalitiesUpToMessage(messageId) - messages before
*/
getRequiredModalities: () => ModelModalities;
/**
* Optional callback to execute after successful validation.
* For ChatForm: undefined - just select model
* For ChatMessageAssistant: (modelName) => onRegenerate(modelName)
*/
onSuccess?: (modelName: string) => void;
/**
* Optional callback for rollback on validation failure.
* For ChatForm: (previousId) => selectModelById(previousId)
* For ChatMessageAssistant: undefined - no rollback needed
*/
onValidationFailure?: (previousModelId: string | null) => Promise<void>;
}
export function useModelChangeValidation(options: UseModelChangeValidationOptions) {
const { getRequiredModalities, onSuccess, onValidationFailure } = options;
let previousSelectedModelId: string | null = null;
const isRouter = $derived(isRouterMode());
async function handleModelChange(modelId: string, modelName: string): Promise<boolean> {
try {
// Store previous selection for potential rollback
if (onValidationFailure) {
previousSelectedModelId = modelsStore.selectedModelId;
}
// Load model if not already loaded (router mode only)
let hasLoadedModel = false;
const isModelLoadedBefore = modelsStore.isModelLoaded(modelName);
if (isRouter && !isModelLoadedBefore) {
try {
await modelsStore.loadModel(modelName);
hasLoadedModel = true;
} catch {
toast.error(`Failed to load model "${modelName}"`);
return false;
}
}
// Fetch model props to validate modalities
const props = await modelsStore.fetchModelProps(modelName);
if (props?.modalities) {
const requiredModalities = getRequiredModalities();
// Check if model supports required modalities
const missingModalities: string[] = [];
if (requiredModalities.vision && !props.modalities.vision) {
missingModalities.push('vision');
}
if (requiredModalities.audio && !props.modalities.audio) {
missingModalities.push('audio');
}
if (missingModalities.length > 0) {
toast.error(
`Model "${modelName}" doesn't support required modalities: ${missingModalities.join(', ')}. Please select a different model.`
);
// Unload the model if we just loaded it
if (isRouter && hasLoadedModel) {
try {
await modelsStore.unloadModel(modelName);
} catch (error) {
console.error('Failed to unload incompatible model:', error);
}
}
// Execute rollback callback if provided
if (onValidationFailure && previousSelectedModelId) {
await onValidationFailure(previousSelectedModelId);
}
return false;
}
}
// Select the model (validation passed)
await modelsStore.selectModelById(modelId);
// Execute success callback if provided
if (onSuccess) {
onSuccess(modelName);
}
return true;
} catch (error) {
console.error('Failed to change model:', error);
toast.error('Failed to validate model capabilities');
// Execute rollback callback on error if provided
if (onValidationFailure && previousSelectedModelId) {
await onValidationFailure(previousSelectedModelId);
}
return false;
}
}
return {
handleModelChange
};
}

61
tools/server/webui/src/lib/stores/conversations.svelte.ts
View File

@ -20,7 +20,6 @@
*/
import { browser } from '$app/environment';
import { AttachmentType } from '$lib/enums';
import type { McpServerOverride } from '$lib/types/database';
class ConversationsStore {
@ -42,14 +41,6 @@ class ConversationsStore {
/** Callback for title update confirmation dialog */
titleUpdateConfirmationCallback?: (currentTitle: string, newTitle: string) => Promise<boolean>;
/**
* Modalities used in the active conversation.
* Computed from attachments in activeMessages.
*/
usedModalities: ModelModalities = $derived.by(() => {
return this.calculateModalitiesFromMessages(this.activeMessages);
});
/** Reference to the client (lazy loaded to avoid circular dependency) */
private _client: typeof import('$lib/clients/conversations.client').conversationsClient | null =
null;
@ -91,57 +82,6 @@ class ConversationsStore {
await conversationsClient.initialize();
}
/**
* Calculate modalities from a list of messages.
*/
private calculateModalitiesFromMessages(messages: DatabaseMessage[]): ModelModalities {
const modalities: ModelModalities = { vision: false, audio: false };
for (const message of messages) {
// Ignore assistant messages (MCP tool results)
if (message.role !== 'user') continue;
if (!message.extra) continue;
for (const extra of message.extra) {
if (extra.type === AttachmentType.IMAGE) {
modalities.vision = true;
}
// PDF only requires vision if processed as images
if (extra.type === AttachmentType.PDF) {
const pdfExtra = extra as DatabaseMessageExtraPdfFile;
if (pdfExtra.processedAsImages) {
modalities.vision = true;
}
}
if (extra.type === AttachmentType.AUDIO) {
modalities.audio = true;
}
}
if (modalities.vision && modalities.audio) break;
}
return modalities;
}
/**
* Get modalities used in messages BEFORE the specified message.
*/
getModalitiesUpToMessage(messageId: string): ModelModalities {
const messageIndex = this.activeMessages.findIndex((m) => m.id === messageId);
if (messageIndex === -1) {
return this.usedModalities;
}
const messagesBefore = this.activeMessages.slice(0, messageIndex);
return this.calculateModalitiesFromMessages(messagesBefore);
}
/**
* Adds a message to the active messages array
*/
@ -336,4 +276,3 @@ export const conversations = () => conversationsStore.conversations;
export const activeConversation = () => conversationsStore.activeConversation;
export const activeMessages = () => conversationsStore.activeMessages;
export const isConversationsInitialized = () => conversationsStore.isInitialized;
export const usedModalities = () => conversationsStore.usedModalities;