happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: Model modality handling

This commit is contained in:
Aleksander Grygier 2025-11-26 17:51:25 +01:00
parent d6ee3d133a
commit 13e7988459
5 changed files with 211 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
View File

@ -9,14 +9,15 @@
} from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { config } from '$lib/stores/settings.svelte';
import { modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import {
modelOptions,
selectedModelId,
isRouterMode,
supportsAudio,
supportsVision,
fetchModelProps,
getModelProps
} from '$lib/stores/server.svelte';
getModelProps,
modelSupportsVision,
modelSupportsAudio
} from '$lib/stores/models.svelte';
import { getConversationModel } from '$lib/stores/chat.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
import {
@ -117,28 +118,20 @@
}
});
// Derive modalities from model props (ROUTER) or server props (MODEL)
// Derive modalities from active model (works for both MODEL and ROUTER mode)
let hasAudioModality = $derived.by(() => {
if (!isRouter) return supportsAudio();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.audio ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsAudio(activeModelId);
}
return false;
});
let hasVisionModality = $derived.by(() => {
if (!isRouter) return supportsVision();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.vision ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsVision(activeModelId);
}
return false;
});

33
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View File

@ -9,15 +9,17 @@
} from '$lib/components/app';
import { FileTypeCategory } from '$lib/enums';
import { getFileTypeCategory } from '$lib/utils/file-type';
import { config } from '$lib/stores/settings.svelte';
import {
supportsAudio,
supportsVision,
modelOptions,
selectedModelId,
selectModelByName,
isRouterMode,
fetchModelProps,
getModelProps
} from '$lib/stores/server.svelte';
import { config } from '$lib/stores/settings.svelte';
import { modelOptions, selectedModelId, selectModelByName } from '$lib/stores/models.svelte';
getModelProps,
modelSupportsVision,
modelSupportsAudio
} from '$lib/stores/models.svelte';
import { getConversationModel } from '$lib/stores/chat.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
import type { ChatUploadedFile } from '$lib/types/chat';
@ -101,29 +103,20 @@
}
});
// Derive modalities from active model (works for both MODEL and ROUTER mode)
let hasAudioModality = $derived.by(() => {
if (!isRouter) return supportsAudio();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.audio ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsAudio(activeModelId);
}
return false;
});
let hasVisionModality = $derived.by(() => {
if (!isRouter) return supportsVision();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.vision ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsVision(activeModelId);
}
return false;
});

33
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
View File

@ -29,17 +29,16 @@
deleteConversation
} from '$lib/stores/conversations.svelte';
import { config } from '$lib/stores/settings.svelte';
import { serverLoading, serverError, serverStore } from '$lib/stores/server.svelte';
import {
supportsVision,
supportsAudio,
serverLoading,
serverError,
serverStore,
modelOptions,
selectedModelId,
isRouterMode,
fetchModelProps,
getModelProps
} from '$lib/stores/server.svelte';
import { modelOptions, selectedModelId } from '$lib/stores/models.svelte';
getModelProps,
modelSupportsVision,
modelSupportsAudio
} from '$lib/stores/models.svelte';
import { getConversationModel } from '$lib/stores/chat.svelte';
import { parseFilesToMessageExtras } from '$lib/utils/convert-files-to-extra';
import { isFileTypeSupported } from '$lib/utils/file-type';
@ -133,28 +132,20 @@
}
});
// Derive modalities from model props (ROUTER) or server props (MODEL)
// Derive modalities from active model (works for both MODEL and ROUTER mode)
let hasAudioModality = $derived.by(() => {
if (!isRouter) return supportsAudio();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.audio ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsAudio(activeModelId);
}
return false;
});
let hasVisionModality = $derived.by(() => {
if (!isRouter) return supportsVision();
if (activeModelId) {
void modelPropsVersion;
const props = getModelProps(activeModelId);
if (props) return props.modalities?.vision ?? false;
void modelPropsVersion; // Trigger reactivity on props fetch
return modelSupportsVision(activeModelId);
}
return false;
});

161
tools/server/webui/src/lib/stores/models.svelte.ts
View File

@ -1,7 +1,7 @@
import { SvelteSet } from 'svelte/reactivity';
import { ModelsService } from '$lib/services/models';
import { ServerModelStatus } from '$lib/enums';
import { serverStore } from '$lib/stores/server.svelte';
import { PropsService } from '$lib/services/props';
import { ServerModelStatus, ServerRole } from '$lib/enums';
import type { ModelOption, ModelModalities } from '$lib/types/models';
import type { ApiModelDataEntry } from '$lib/types/api';
@ -16,11 +16,16 @@ import type { ApiModelDataEntry } from '$lib/types/api';
* - Automatic unloading of unused models
*
* **Architecture & Relationships:**
* - **ModelsService**: Stateless service for API communication
* - **ModelsService**: Stateless service for model API communication
* - **PropsService**: Stateless service for props/modalities fetching
* - **ModelsStore** (this class): Reactive store for model state
* - **ServerStore**: Provides server mode detection
* - **ConversationsStore**: Tracks which conversations use which models
*
* **API Inconsistency Workaround:**
* In MODEL mode, `/props` returns modalities for the single model.
* In ROUTER mode, `/props` has no modalities - must use `/props?model=<id>` per model.
* This store normalizes this behavior so consumers don't need to know the server mode.
*
* **Key Features:**
* - **MODEL mode**: Single model, always loaded
* - **ROUTER mode**: Multi-model with load/unload capability
@ -43,6 +48,20 @@ class ModelsStore {
private modelUsage = $state<Map<string, SvelteSet<string>>>(new Map());
private modelLoadingStates = $state<Map<string, boolean>>(new Map());
/**
* Server role detection - determines API behavior
* In ROUTER mode, modalities come from /props?model=<id>
* In MODEL mode, modalities come from /props (single model)
*/
serverRole = $state<ServerRole | null>(null);
/**
* Model-specific props cache
* Key: modelId, Value: props data including modalities
*/
private modelPropsCache = $state<Map<string, ApiLlamaCppServerProps>>(new Map());
private modelPropsFetching = $state<Set<string>>(new Set());
// ─────────────────────────────────────────────────────────────────────────────
// Computed Getters
// ─────────────────────────────────────────────────────────────────────────────
@ -64,6 +83,69 @@ class ModelsStore {
.map(([id]) => id);
}
get isRouterMode(): boolean {
return this.serverRole === ServerRole.ROUTER;
}
get isModelMode(): boolean {
return this.serverRole === ServerRole.MODEL;
}
// ─────────────────────────────────────────────────────────────────────────────
// Methods - Model Modalities
// ─────────────────────────────────────────────────────────────────────────────
/**
* Get modalities for a specific model
* Returns cached modalities from model props
*/
getModelModalities(modelId: string): ModelModalities | null {
// First check if modalities are stored in the model option
const model = this.models.find((m) => m.model === modelId || m.id === modelId);
if (model?.modalities) {
return model.modalities;
}
// Fall back to props cache
const props = this.modelPropsCache.get(modelId);
if (props?.modalities) {
return {
vision: props.modalities.vision ?? false,
audio: props.modalities.audio ?? false
};
}
return null;
}
/**
* Check if a model supports vision modality
*/
modelSupportsVision(modelId: string): boolean {
return this.getModelModalities(modelId)?.vision ?? false;
}
/**
* Check if a model supports audio modality
*/
modelSupportsAudio(modelId: string): boolean {
return this.getModelModalities(modelId)?.audio ?? false;
}
/**
* Get props for a specific model (from cache)
*/
getModelProps(modelId: string): ApiLlamaCppServerProps | null {
return this.modelPropsCache.get(modelId) ?? null;
}
/**
* Check if props are being fetched for a model
*/
isModelPropsFetching(modelId: string): boolean {
return this.modelPropsFetching.has(modelId);
}
// ─────────────────────────────────────────────────────────────────────────────
// Methods - Model Status
// ─────────────────────────────────────────────────────────────────────────────
@ -96,7 +178,8 @@ class ModelsStore {
// ─────────────────────────────────────────────────────────────────────────────
/**
* Fetch list of models from server
* Fetch list of models from server and detect server role
* Also fetches modalities for MODEL mode (single model)
*/
async fetch(force = false): Promise<void> {
if (this.loading) return;
@ -106,6 +189,11 @@ class ModelsStore {
this.error = null;
try {
// Fetch server props to detect role and get modalities for MODEL mode
const serverProps = await PropsService.fetch();
this.serverRole =
serverProps.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL;
const response = await ModelsService.list();
const models: ModelOption[] = response.data.map((item, index) => {
@ -127,6 +215,22 @@ class ModelsStore {
});
this.models = models;
// In MODEL mode, populate modalities from /props (single model)
// WORKAROUND: In MODEL mode, /props returns modalities for the single model,
// but /v1/models doesn't include modalities. We bridge this gap here.
if (this.isModelMode && this.models.length > 0 && serverProps.modalities) {
const modalities: ModelModalities = {
vision: serverProps.modalities.vision ?? false,
audio: serverProps.modalities.audio ?? false
};
// Cache props for the single model
this.modelPropsCache.set(this.models[0].model, serverProps);
// Update model with modalities
this.models = this.models.map((model, index) =>
index === 0 ? { ...model, modalities } : model
);
}
} catch (error) {
this.models = [];
this.error = error instanceof Error ? error.message : 'Failed to load models';
@ -151,16 +255,45 @@ class ModelsStore {
}
}
/**
* Fetch props for a specific model from /props endpoint
* Uses caching to avoid redundant requests
*
* @param modelId - Model identifier to fetch props for
* @returns Props data or null if fetch failed
*/
async fetchModelProps(modelId: string): Promise<ApiLlamaCppServerProps | null> {
// Return cached props if available
const cached = this.modelPropsCache.get(modelId);
if (cached) return cached;
// Avoid duplicate fetches
if (this.modelPropsFetching.has(modelId)) return null;
this.modelPropsFetching.add(modelId);
try {
const props = await PropsService.fetchForModel(modelId);
this.modelPropsCache.set(modelId, props);
return props;
} catch (error) {
console.warn(`Failed to fetch props for model ${modelId}:`, error);
return null;
} finally {
this.modelPropsFetching.delete(modelId);
}
}
/**
* Fetch modalities for all loaded models from /props endpoint
* This updates the modalities field in _models array
* This updates the modalities field in models array
*/
async fetchModalitiesForLoadedModels(): Promise<void> {
const loadedModelIds = this.loadedModelIds;
if (loadedModelIds.length === 0) return;
// Fetch props for each loaded model in parallel
const propsPromises = loadedModelIds.map((modelId) => serverStore.fetchModelProps(modelId));
const propsPromises = loadedModelIds.map((modelId) => this.fetchModelProps(modelId));
try {
const results = await Promise.all(propsPromises);
@ -191,7 +324,7 @@ class ModelsStore {
*/
async updateModelModalities(modelId: string): Promise<void> {
try {
const props = await serverStore.fetchModelProps(modelId);
const props = await this.fetchModelProps(modelId);
if (!props?.modalities) return;
const modalities: ModelModalities = {
@ -448,8 +581,11 @@ class ModelsStore {
this.error = null;
this.selectedModelId = null;
this.selectedModelName = null;
this.serverRole = null;
this.modelUsage.clear();
this.modelLoadingStates.clear();
this.modelPropsCache.clear();
this.modelPropsFetching.clear();
}
}
@ -469,6 +605,8 @@ export const selectedModelName = () => modelsStore.selectedModelName;
export const selectedModelOption = () => modelsStore.selectedModel;
export const loadedModelIds = () => modelsStore.loadedModelIds;
export const loadingModelIds = () => modelsStore.loadingModelIds;
export const isRouterMode = () => modelsStore.isRouterMode;
export const isModelMode = () => modelsStore.isModelMode;
// ─────────────────────────────────────────────────────────────────────────────
// Actions
@ -491,3 +629,10 @@ export const clearModelSelection = modelsStore.clearSelection.bind(modelsStore);
export const findModelByName = modelsStore.findModelByName.bind(modelsStore);
export const findModelById = modelsStore.findModelById.bind(modelsStore);
export const hasModel = modelsStore.hasModel.bind(modelsStore);
// Model modalities
export const getModelModalities = modelsStore.getModelModalities.bind(modelsStore);
export const modelSupportsVision = modelsStore.modelSupportsVision.bind(modelsStore);
export const modelSupportsAudio = modelsStore.modelSupportsAudio.bind(modelsStore);
export const fetchModelProps = modelsStore.fetchModelProps.bind(modelsStore);
export const getModelProps = modelsStore.getModelProps.bind(modelsStore);

75
tools/server/webui/src/lib/stores/server.svelte.ts
View File

@ -2,21 +2,26 @@ import { PropsService } from '$lib/services/props';
import { ServerRole, ModelModality } from '$lib/enums';
/**
* ServerStore - Server state, capabilities, and mode detection
* ServerStore - Server connection state, configuration, and role detection
*
* This store manages the server connection state and properties fetched from `/props`.
* It provides reactive state for server configuration, capabilities, and role detection.
* It provides reactive state for server configuration and role detection.
*
* **Architecture & Relationships:**
* - **PropsService**: Stateless service for fetching `/props` data
* - **ServerStore** (this class): Reactive store for server state
* - **ModelsStore**: Uses server role for model management strategy
* - **ModelsStore**: Independent store for model management (uses PropsService directly)
*
* **Key Features:**
* - **Server State**: Connection status, loading, error handling
* - **Role Detection**: MODEL (single model) vs ROUTER (multi-model)
* - **Capability Detection**: Vision and audio modality support
* - **Props Cache**: Per-model props caching for ROUTER mode
* - **Default Params**: Server-wide generation defaults
*
* **Note on Modalities:**
* Model-specific modalities (vision, audio) are now managed by ModelsStore.
* Use `modelsStore.getModelModalities(modelId)` for per-model modality info.
* The `supportsVision`/`supportsAudio` getters here are deprecated and only
* apply to MODEL mode (single model).
*/
class ServerStore {
props = $state<ApiLlamaCppServerProps | null>(null);
@ -25,10 +30,6 @@ class ServerStore {
role = $state<ServerRole | null>(null);
private fetchPromise: Promise<void> | null = null;
// Model-specific props cache (ROUTER mode)
private modelPropsCache = $state<Map<string, ApiLlamaCppServerProps>>(new Map());
private modelPropsFetching = $state<Set<string>>(new Set());
// ─────────────────────────────────────────────────────────────────────────────
// Computed Getters
// ─────────────────────────────────────────────────────────────────────────────
@ -45,6 +46,10 @@ class ServerStore {
return this.props.model_path.split(/(\\|\/)/).pop() || null;
}
/**
* @deprecated Use modelsStore.getModelModalities(modelId) for per-model modalities.
* This only works in MODEL mode (single model).
*/
get supportedModalities(): ModelModality[] {
const modalities: ModelModality[] = [];
if (this.props?.modalities?.audio) modalities.push(ModelModality.AUDIO);
@ -52,10 +57,18 @@ class ServerStore {
return modalities;
}
/**
* @deprecated Use modelsStore.modelSupportsVision(modelId) for per-model check.
* This only works in MODEL mode (single model).
*/
get supportsVision(): boolean {
return this.props?.modalities?.vision ?? false;
}
/**
* @deprecated Use modelsStore.modelSupportsAudio(modelId) for per-model check.
* This only works in MODEL mode (single model).
*/
get supportsAudio(): boolean {
return this.props?.modalities?.audio ?? false;
}
@ -102,18 +115,9 @@ class ServerStore {
this.loading = true;
this.error = null;
const previousBuildInfo = this.props?.build_info;
const fetchPromise = (async () => {
try {
const props = await PropsService.fetch();
// Clear model-specific props cache if server was restarted
if (previousBuildInfo && previousBuildInfo !== props.build_info) {
this.modelPropsCache.clear();
console.info('Cleared model props cache due to server restart');
}
this.props = props;
this.error = null;
this.detectRole(props);
@ -130,38 +134,6 @@ class ServerStore {
await fetchPromise;
}
// ─────────────────────────────────────────────────────────────────────────────
// Fetch Model-Specific Properties (ROUTER mode)
// ─────────────────────────────────────────────────────────────────────────────
getModelProps(modelId: string): ApiLlamaCppServerProps | null {
return this.modelPropsCache.get(modelId) ?? null;
}
isModelPropsFetching(modelId: string): boolean {
return this.modelPropsFetching.has(modelId);
}
async fetchModelProps(modelId: string): Promise<ApiLlamaCppServerProps | null> {
const cached = this.modelPropsCache.get(modelId);
if (cached) return cached;
if (this.modelPropsFetching.has(modelId)) return null;
this.modelPropsFetching.add(modelId);
try {
const props = await PropsService.fetchForModel(modelId);
this.modelPropsCache.set(modelId, props);
return props;
} catch (error) {
console.warn(`Failed to fetch props for model ${modelId}:`, error);
return null;
} finally {
this.modelPropsFetching.delete(modelId);
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Error Handling
// ─────────────────────────────────────────────────────────────────────────────
@ -202,7 +174,6 @@ class ServerStore {
this.loading = false;
this.role = null;
this.fetchPromise = null;
this.modelPropsCache.clear();
}
}
@ -228,5 +199,3 @@ export const isModelMode = () => serverStore.isModelMode;
// Actions
export const fetchServerProps = serverStore.fetch.bind(serverStore);
export const fetchModelProps = serverStore.fetchModelProps.bind(serverStore);
export const getModelProps = serverStore.getModelProps.bind(serverStore);