happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: Use TTL cache for model properties in ModelsStore

This commit is contained in:
Aleksander Grygier 2026-01-27 13:10:15 +01:00
parent 2e2cb3d210
commit 2281ac50c6
1 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/server/webui/src/lib/stores/models.svelte.ts
View File

@ -2,6 +2,8 @@ import { SvelteSet } from 'svelte/reactivity';
import { ServerModelStatus, ModelModality } from '$lib/enums';
import { ModelsService, PropsService } from '$lib/services';
import { serverStore } from '$lib/stores/server.svelte';
import { TTLCache } from '$lib/utils';
import { MODEL_PROPS_CACHE_TTL_MS, MODEL_PROPS_CACHE_MAX_ENTRIES } from '$lib/constants/cache';
/**
* modelsStore - Reactive store for model management in both MODEL and ROUTER modes
@ -51,10 +53,14 @@ class ModelsStore {
private modelLoadingStates = $state<Map<string, boolean>>(new Map());
/**
* Model-specific props cache
* Model-specific props cache with TTL
* Key: modelId, Value: props data including modalities
* TTL: 10 minutes - props don't change frequently
*/
private modelPropsCache = $state<Map<string, ApiLlamaCppServerProps>>(new Map());
private modelPropsCache = new TTLCache<string, ApiLlamaCppServerProps>({
ttlMs: MODEL_PROPS_CACHE_TTL_MS,
maxEntries: MODEL_PROPS_CACHE_MAX_ENTRIES
});
private modelPropsFetching = $state<Set<string>>(new Set());
/**
@ -164,14 +170,14 @@ class ModelsStore {
* Get props for a specific model (from cache)
*/
getModelProps(modelId: string): ApiLlamaCppServerProps | null {
return this.modelPropsCache.get(modelId) ?? null;
return this.modelPropsCache.get(modelId);
}
/**
* Get context size (n_ctx) for a specific model from cached props
*/
getModelContextSize(modelId: string): number | null {
const props = this.modelPropsCache.get(modelId);
const props = this.getModelProps(modelId);
return props?.default_generation_settings?.n_ctx ?? null;
}
@ -603,6 +609,14 @@ class ModelsStore {
this.modelPropsCache.clear();
this.modelPropsFetching.clear();
}
/**
* Prune expired entries from caches.
* Call periodically for proactive memory cleanup.
*/
pruneExpiredCache(): number {
return this.modelPropsCache.prune();
}
}
export const modelsStore = new ModelsStore();