From 13e7988459a6282a9bf7d20db4e6946fea8cbe03 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 26 Nov 2025 17:51:25 +0100 Subject: [PATCH] refactor: Model modality handling --- .../app/chat/ChatForm/ChatForm.svelte | 29 ++-- .../ChatFormActions/ChatFormActions.svelte | 33 ++-- .../app/chat/ChatScreen/ChatScreen.svelte | 33 ++-- .../webui/src/lib/stores/models.svelte.ts | 161 +++++++++++++++++- .../webui/src/lib/stores/server.svelte.ts | 75 +++----- 5 files changed, 211 insertions(+), 120 deletions(-) diff --git a/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index bed1c546ae..44a37ced0e 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -9,14 +9,15 @@ } from '$lib/components/app'; import { INPUT_CLASSES } from '$lib/constants/input-classes'; import { config } from '$lib/stores/settings.svelte'; - import { modelOptions, selectedModelId } from '$lib/stores/models.svelte'; import { + modelOptions, + selectedModelId, isRouterMode, - supportsAudio, - supportsVision, fetchModelProps, - getModelProps - } from '$lib/stores/server.svelte'; + getModelProps, + modelSupportsVision, + modelSupportsAudio + } from '$lib/stores/models.svelte'; import { getConversationModel } from '$lib/stores/chat.svelte'; import { activeMessages } from '$lib/stores/conversations.svelte'; import { @@ -117,28 +118,20 @@ } }); - // Derive modalities from model props (ROUTER) or server props (MODEL) + // Derive modalities from active model (works for both MODEL and ROUTER mode) let hasAudioModality = $derived.by(() => { - if (!isRouter) return supportsAudio(); - if (activeModelId) { - void modelPropsVersion; - const props = getModelProps(activeModelId); - if (props) return props.modalities?.audio ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsAudio(activeModelId); } - return false; }); let hasVisionModality = $derived.by(() => { - if (!isRouter) return supportsVision(); - if (activeModelId) { - void modelPropsVersion; - const props = getModelProps(activeModelId); - if (props) return props.modalities?.vision ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsVision(activeModelId); } - return false; }); diff --git a/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index a5170763f0..4dae0166b5 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -9,15 +9,17 @@ } from '$lib/components/app'; import { FileTypeCategory } from '$lib/enums'; import { getFileTypeCategory } from '$lib/utils/file-type'; + import { config } from '$lib/stores/settings.svelte'; import { - supportsAudio, - supportsVision, + modelOptions, + selectedModelId, + selectModelByName, isRouterMode, fetchModelProps, - getModelProps - } from '$lib/stores/server.svelte'; - import { config } from '$lib/stores/settings.svelte'; - import { modelOptions, selectedModelId, selectModelByName } from '$lib/stores/models.svelte'; + getModelProps, + modelSupportsVision, + modelSupportsAudio + } from '$lib/stores/models.svelte'; import { getConversationModel } from '$lib/stores/chat.svelte'; import { activeMessages } from '$lib/stores/conversations.svelte'; import type { ChatUploadedFile } from '$lib/types/chat'; @@ -101,29 +103,20 @@ } }); + // Derive modalities from active model (works for both MODEL and ROUTER mode) let hasAudioModality = $derived.by(() => { - if (!isRouter) return supportsAudio(); - if (activeModelId) { - void modelPropsVersion; - - const props = getModelProps(activeModelId); - if (props) return props.modalities?.audio ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsAudio(activeModelId); } - return false; }); let hasVisionModality = $derived.by(() => { - if (!isRouter) return supportsVision(); - if (activeModelId) { - void modelPropsVersion; - - const props = getModelProps(activeModelId); - if (props) return props.modalities?.vision ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsVision(activeModelId); } - return false; }); diff --git a/tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index ade6eb99f9..b4fd0112a0 100644 --- a/tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -29,17 +29,16 @@ deleteConversation } from '$lib/stores/conversations.svelte'; import { config } from '$lib/stores/settings.svelte'; + import { serverLoading, serverError, serverStore } from '$lib/stores/server.svelte'; import { - supportsVision, - supportsAudio, - serverLoading, - serverError, - serverStore, + modelOptions, + selectedModelId, isRouterMode, fetchModelProps, - getModelProps - } from '$lib/stores/server.svelte'; - import { modelOptions, selectedModelId } from '$lib/stores/models.svelte'; + getModelProps, + modelSupportsVision, + modelSupportsAudio + } from '$lib/stores/models.svelte'; import { getConversationModel } from '$lib/stores/chat.svelte'; import { parseFilesToMessageExtras } from '$lib/utils/convert-files-to-extra'; import { isFileTypeSupported } from '$lib/utils/file-type'; @@ -133,28 +132,20 @@ } }); - // Derive modalities from model props (ROUTER) or server props (MODEL) + // Derive modalities from active model (works for both MODEL and ROUTER mode) let hasAudioModality = $derived.by(() => { - if (!isRouter) return supportsAudio(); - if (activeModelId) { - void modelPropsVersion; - const props = getModelProps(activeModelId); - if (props) return props.modalities?.audio ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsAudio(activeModelId); } - return false; }); let hasVisionModality = $derived.by(() => { - if (!isRouter) return supportsVision(); - if (activeModelId) { - void modelPropsVersion; - const props = getModelProps(activeModelId); - if (props) return props.modalities?.vision ?? false; + void modelPropsVersion; // Trigger reactivity on props fetch + return modelSupportsVision(activeModelId); } - return false; }); diff --git a/tools/server/webui/src/lib/stores/models.svelte.ts b/tools/server/webui/src/lib/stores/models.svelte.ts index 8983934ac8..2422b44a4f 100644 --- a/tools/server/webui/src/lib/stores/models.svelte.ts +++ b/tools/server/webui/src/lib/stores/models.svelte.ts @@ -1,7 +1,7 @@ import { SvelteSet } from 'svelte/reactivity'; import { ModelsService } from '$lib/services/models'; -import { ServerModelStatus } from '$lib/enums'; -import { serverStore } from '$lib/stores/server.svelte'; +import { PropsService } from '$lib/services/props'; +import { ServerModelStatus, ServerRole } from '$lib/enums'; import type { ModelOption, ModelModalities } from '$lib/types/models'; import type { ApiModelDataEntry } from '$lib/types/api'; @@ -16,11 +16,16 @@ import type { ApiModelDataEntry } from '$lib/types/api'; * - Automatic unloading of unused models * * **Architecture & Relationships:** - * - **ModelsService**: Stateless service for API communication + * - **ModelsService**: Stateless service for model API communication + * - **PropsService**: Stateless service for props/modalities fetching * - **ModelsStore** (this class): Reactive store for model state - * - **ServerStore**: Provides server mode detection * - **ConversationsStore**: Tracks which conversations use which models * + * **API Inconsistency Workaround:** + * In MODEL mode, `/props` returns modalities for the single model. + * In ROUTER mode, `/props` has no modalities - must use `/props?model=` per model. + * This store normalizes this behavior so consumers don't need to know the server mode. + * * **Key Features:** * - **MODEL mode**: Single model, always loaded * - **ROUTER mode**: Multi-model with load/unload capability @@ -43,6 +48,20 @@ class ModelsStore { private modelUsage = $state>>(new Map()); private modelLoadingStates = $state>(new Map()); + /** + * Server role detection - determines API behavior + * In ROUTER mode, modalities come from /props?model= + * In MODEL mode, modalities come from /props (single model) + */ + serverRole = $state(null); + + /** + * Model-specific props cache + * Key: modelId, Value: props data including modalities + */ + private modelPropsCache = $state>(new Map()); + private modelPropsFetching = $state>(new Set()); + // ───────────────────────────────────────────────────────────────────────────── // Computed Getters // ───────────────────────────────────────────────────────────────────────────── @@ -64,6 +83,69 @@ class ModelsStore { .map(([id]) => id); } + get isRouterMode(): boolean { + return this.serverRole === ServerRole.ROUTER; + } + + get isModelMode(): boolean { + return this.serverRole === ServerRole.MODEL; + } + + // ───────────────────────────────────────────────────────────────────────────── + // Methods - Model Modalities + // ───────────────────────────────────────────────────────────────────────────── + + /** + * Get modalities for a specific model + * Returns cached modalities from model props + */ + getModelModalities(modelId: string): ModelModalities | null { + // First check if modalities are stored in the model option + const model = this.models.find((m) => m.model === modelId || m.id === modelId); + if (model?.modalities) { + return model.modalities; + } + + // Fall back to props cache + const props = this.modelPropsCache.get(modelId); + if (props?.modalities) { + return { + vision: props.modalities.vision ?? false, + audio: props.modalities.audio ?? false + }; + } + + return null; + } + + /** + * Check if a model supports vision modality + */ + modelSupportsVision(modelId: string): boolean { + return this.getModelModalities(modelId)?.vision ?? false; + } + + /** + * Check if a model supports audio modality + */ + modelSupportsAudio(modelId: string): boolean { + return this.getModelModalities(modelId)?.audio ?? false; + } + + /** + * Get props for a specific model (from cache) + */ + getModelProps(modelId: string): ApiLlamaCppServerProps | null { + return this.modelPropsCache.get(modelId) ?? null; + } + + /** + * Check if props are being fetched for a model + */ + isModelPropsFetching(modelId: string): boolean { + return this.modelPropsFetching.has(modelId); + } + // ───────────────────────────────────────────────────────────────────────────── // Methods - Model Status // ───────────────────────────────────────────────────────────────────────────── @@ -96,7 +178,8 @@ class ModelsStore { // ───────────────────────────────────────────────────────────────────────────── /** - * Fetch list of models from server + * Fetch list of models from server and detect server role + * Also fetches modalities for MODEL mode (single model) */ async fetch(force = false): Promise { if (this.loading) return; @@ -106,6 +189,11 @@ class ModelsStore { this.error = null; try { + // Fetch server props to detect role and get modalities for MODEL mode + const serverProps = await PropsService.fetch(); + this.serverRole = + serverProps.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL; + const response = await ModelsService.list(); const models: ModelOption[] = response.data.map((item, index) => { @@ -127,6 +215,22 @@ class ModelsStore { }); this.models = models; + + // In MODEL mode, populate modalities from /props (single model) + // WORKAROUND: In MODEL mode, /props returns modalities for the single model, + // but /v1/models doesn't include modalities. We bridge this gap here. + if (this.isModelMode && this.models.length > 0 && serverProps.modalities) { + const modalities: ModelModalities = { + vision: serverProps.modalities.vision ?? false, + audio: serverProps.modalities.audio ?? false + }; + // Cache props for the single model + this.modelPropsCache.set(this.models[0].model, serverProps); + // Update model with modalities + this.models = this.models.map((model, index) => + index === 0 ? { ...model, modalities } : model + ); + } } catch (error) { this.models = []; this.error = error instanceof Error ? error.message : 'Failed to load models'; @@ -151,16 +255,45 @@ class ModelsStore { } } + /** + * Fetch props for a specific model from /props endpoint + * Uses caching to avoid redundant requests + * + * @param modelId - Model identifier to fetch props for + * @returns Props data or null if fetch failed + */ + async fetchModelProps(modelId: string): Promise { + // Return cached props if available + const cached = this.modelPropsCache.get(modelId); + if (cached) return cached; + + // Avoid duplicate fetches + if (this.modelPropsFetching.has(modelId)) return null; + + this.modelPropsFetching.add(modelId); + + try { + const props = await PropsService.fetchForModel(modelId); + this.modelPropsCache.set(modelId, props); + return props; + } catch (error) { + console.warn(`Failed to fetch props for model ${modelId}:`, error); + return null; + } finally { + this.modelPropsFetching.delete(modelId); + } + } + /** * Fetch modalities for all loaded models from /props endpoint - * This updates the modalities field in _models array + * This updates the modalities field in models array */ async fetchModalitiesForLoadedModels(): Promise { const loadedModelIds = this.loadedModelIds; if (loadedModelIds.length === 0) return; // Fetch props for each loaded model in parallel - const propsPromises = loadedModelIds.map((modelId) => serverStore.fetchModelProps(modelId)); + const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId)); try { const results = await Promise.all(propsPromises); @@ -191,7 +324,7 @@ class ModelsStore { */ async updateModelModalities(modelId: string): Promise { try { - const props = await serverStore.fetchModelProps(modelId); + const props = await this.fetchModelProps(modelId); if (!props?.modalities) return; const modalities: ModelModalities = { @@ -448,8 +581,11 @@ class ModelsStore { this.error = null; this.selectedModelId = null; this.selectedModelName = null; + this.serverRole = null; this.modelUsage.clear(); this.modelLoadingStates.clear(); + this.modelPropsCache.clear(); + this.modelPropsFetching.clear(); } } @@ -469,6 +605,8 @@ export const selectedModelName = () => modelsStore.selectedModelName; export const selectedModelOption = () => modelsStore.selectedModel; export const loadedModelIds = () => modelsStore.loadedModelIds; export const loadingModelIds = () => modelsStore.loadingModelIds; +export const isRouterMode = () => modelsStore.isRouterMode; +export const isModelMode = () => modelsStore.isModelMode; // ───────────────────────────────────────────────────────────────────────────── // Actions @@ -491,3 +629,10 @@ export const clearModelSelection = modelsStore.clearSelection.bind(modelsStore); export const findModelByName = modelsStore.findModelByName.bind(modelsStore); export const findModelById = modelsStore.findModelById.bind(modelsStore); export const hasModel = modelsStore.hasModel.bind(modelsStore); + +// Model modalities +export const getModelModalities = modelsStore.getModelModalities.bind(modelsStore); +export const modelSupportsVision = modelsStore.modelSupportsVision.bind(modelsStore); +export const modelSupportsAudio = modelsStore.modelSupportsAudio.bind(modelsStore); +export const fetchModelProps = modelsStore.fetchModelProps.bind(modelsStore); +export const getModelProps = modelsStore.getModelProps.bind(modelsStore); diff --git a/tools/server/webui/src/lib/stores/server.svelte.ts b/tools/server/webui/src/lib/stores/server.svelte.ts index 3fe0a887b9..c4d1fb8736 100644 --- a/tools/server/webui/src/lib/stores/server.svelte.ts +++ b/tools/server/webui/src/lib/stores/server.svelte.ts @@ -2,21 +2,26 @@ import { PropsService } from '$lib/services/props'; import { ServerRole, ModelModality } from '$lib/enums'; /** - * ServerStore - Server state, capabilities, and mode detection + * ServerStore - Server connection state, configuration, and role detection * * This store manages the server connection state and properties fetched from `/props`. - * It provides reactive state for server configuration, capabilities, and role detection. + * It provides reactive state for server configuration and role detection. * * **Architecture & Relationships:** * - **PropsService**: Stateless service for fetching `/props` data * - **ServerStore** (this class): Reactive store for server state - * - **ModelsStore**: Uses server role for model management strategy + * - **ModelsStore**: Independent store for model management (uses PropsService directly) * * **Key Features:** * - **Server State**: Connection status, loading, error handling * - **Role Detection**: MODEL (single model) vs ROUTER (multi-model) - * - **Capability Detection**: Vision and audio modality support - * - **Props Cache**: Per-model props caching for ROUTER mode + * - **Default Params**: Server-wide generation defaults + * + * **Note on Modalities:** + * Model-specific modalities (vision, audio) are now managed by ModelsStore. + * Use `modelsStore.getModelModalities(modelId)` for per-model modality info. + * The `supportsVision`/`supportsAudio` getters here are deprecated and only + * apply to MODEL mode (single model). */ class ServerStore { props = $state(null); @@ -25,10 +30,6 @@ class ServerStore { role = $state(null); private fetchPromise: Promise | null = null; - // Model-specific props cache (ROUTER mode) - private modelPropsCache = $state>(new Map()); - private modelPropsFetching = $state>(new Set()); - // ───────────────────────────────────────────────────────────────────────────── // Computed Getters // ───────────────────────────────────────────────────────────────────────────── @@ -45,6 +46,10 @@ class ServerStore { return this.props.model_path.split(/(\\|\/)/).pop() || null; } + /** + * @deprecated Use modelsStore.getModelModalities(modelId) for per-model modalities. + * This only works in MODEL mode (single model). + */ get supportedModalities(): ModelModality[] { const modalities: ModelModality[] = []; if (this.props?.modalities?.audio) modalities.push(ModelModality.AUDIO); @@ -52,10 +57,18 @@ class ServerStore { return modalities; } + /** + * @deprecated Use modelsStore.modelSupportsVision(modelId) for per-model check. + * This only works in MODEL mode (single model). + */ get supportsVision(): boolean { return this.props?.modalities?.vision ?? false; } + /** + * @deprecated Use modelsStore.modelSupportsAudio(modelId) for per-model check. + * This only works in MODEL mode (single model). + */ get supportsAudio(): boolean { return this.props?.modalities?.audio ?? false; } @@ -102,18 +115,9 @@ class ServerStore { this.loading = true; this.error = null; - const previousBuildInfo = this.props?.build_info; - const fetchPromise = (async () => { try { const props = await PropsService.fetch(); - - // Clear model-specific props cache if server was restarted - if (previousBuildInfo && previousBuildInfo !== props.build_info) { - this.modelPropsCache.clear(); - console.info('Cleared model props cache due to server restart'); - } - this.props = props; this.error = null; this.detectRole(props); @@ -130,38 +134,6 @@ class ServerStore { await fetchPromise; } - // ───────────────────────────────────────────────────────────────────────────── - // Fetch Model-Specific Properties (ROUTER mode) - // ───────────────────────────────────────────────────────────────────────────── - - getModelProps(modelId: string): ApiLlamaCppServerProps | null { - return this.modelPropsCache.get(modelId) ?? null; - } - - isModelPropsFetching(modelId: string): boolean { - return this.modelPropsFetching.has(modelId); - } - - async fetchModelProps(modelId: string): Promise { - const cached = this.modelPropsCache.get(modelId); - if (cached) return cached; - - if (this.modelPropsFetching.has(modelId)) return null; - - this.modelPropsFetching.add(modelId); - - try { - const props = await PropsService.fetchForModel(modelId); - this.modelPropsCache.set(modelId, props); - return props; - } catch (error) { - console.warn(`Failed to fetch props for model ${modelId}:`, error); - return null; - } finally { - this.modelPropsFetching.delete(modelId); - } - } - // ───────────────────────────────────────────────────────────────────────────── // Error Handling // ───────────────────────────────────────────────────────────────────────────── @@ -202,7 +174,6 @@ class ServerStore { this.loading = false; this.role = null; this.fetchPromise = null; - this.modelPropsCache.clear(); } } @@ -228,5 +199,3 @@ export const isModelMode = () => serverStore.isModelMode; // Actions export const fetchServerProps = serverStore.fetch.bind(serverStore); -export const fetchModelProps = serverStore.fetchModelProps.bind(serverStore); -export const getModelProps = serverStore.getModelProps.bind(serverStore);