happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: Processing state reactivity

This commit is contained in:
Aleksander Grygier 2025-11-27 11:11:45 +01:00
parent 2a5922b1f6
commit 6b95118abc
3 changed files with 101 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreenProcessingInfo.svelte
View File

@ -4,9 +4,10 @@
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte'; import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { import {
isLoading, isLoading,
isChatStreaming,
clearProcessingState, clearProcessingState,
updateProcessingStateFromTimings, setActiveProcessingConversation,
setActiveProcessingConversation restoreProcessingStateFromMessages
} from '$lib/stores/chat.svelte'; } from '$lib/stores/chat.svelte';
import { activeMessages, activeConversation } from '$lib/stores/conversations.svelte'; import { activeMessages, activeConversation } from '$lib/stores/conversations.svelte';
import { config } from '$lib/stores/settings.svelte'; import { config } from '$lib/stores/settings.svelte';
@ -14,34 +15,39 @@
const processingState = useProcessingState(); const processingState = useProcessingState();
let isCurrentConversationLoading = $derived(isLoading()); let isCurrentConversationLoading = $derived(isLoading());
let isStreaming = $derived(isChatStreaming());
let hasProcessingData = $derived(processingState.processingState !== null);
let processingDetails = $derived(processingState.getProcessingDetails()); let processingDetails = $derived(processingState.getProcessingDetails());
let showSlotsInfo = $derived(isCurrentConversationLoading || config().keepStatsVisible);
// Sync active processing conversation with currently viewed conversation let showProcessingInfo = $derived(
isCurrentConversationLoading || isStreaming || config().keepStatsVisible || hasProcessingData
);
$effect(() => { $effect(() => {
const conversation = activeConversation(); const conversation = activeConversation();
// Use untrack to prevent creating reactive dependencies on state updates
untrack(() => setActiveProcessingConversation(conversation?.id ?? null)); untrack(() => setActiveProcessingConversation(conversation?.id ?? null));
}); });
// Track loading state reactively by checking if conversation ID is in loading conversations array
$effect(() => { $effect(() => {
const keepStatsVisible = config().keepStatsVisible; const keepStatsVisible = config().keepStatsVisible;
const shouldMonitor = keepStatsVisible || isCurrentConversationLoading || isStreaming;
if (keepStatsVisible || isCurrentConversationLoading) { if (shouldMonitor) {
untrack(() => processingState.startMonitoring()); processingState.startMonitoring();
} }
if (!isCurrentConversationLoading && !keepStatsVisible) { if (!isCurrentConversationLoading && !isStreaming && !keepStatsVisible) {
setTimeout(() => { const timeout = setTimeout(() => {
if (!config().keepStatsVisible) { if (!config().keepStatsVisible && !isChatStreaming()) {
processingState.stopMonitoring(); processingState.stopMonitoring();
} }
}, PROCESSING_INFO_TIMEOUT); }, PROCESSING_INFO_TIMEOUT);
return () => clearTimeout(timeout);
} }
}); });
// Update processing state from stored timings
$effect(() => { $effect(() => {
const conversation = activeConversation(); const conversation = activeConversation();
const messages = activeMessages() as DatabaseMessage[]; const messages = activeMessages() as DatabaseMessage[];
@ -49,47 +55,18 @@
if (keepStatsVisible && conversation) { if (keepStatsVisible && conversation) {
if (messages.length === 0) { if (messages.length === 0) {
// Use untrack to prevent creating reactive dependencies on state updates
untrack(() => clearProcessingState(conversation.id)); untrack(() => clearProcessingState(conversation.id));
return; return;
} }
// Search backwards through messages to find most recent assistant message with timing data if (!isCurrentConversationLoading && !isStreaming) {
// Using reverse iteration for performance - avoids array copy and stops at first match untrack(() => restoreProcessingStateFromMessages(messages, conversation.id));
let foundTimingData = false;
for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i];
if (message.role === 'assistant' && message.timings) {
foundTimingData = true;
// Use untrack to prevent creating reactive dependencies on state updates
untrack(() =>
updateProcessingStateFromTimings(
{
prompt_n: message.timings!.prompt_n || 0,
predicted_n: message.timings!.predicted_n || 0,
predicted_per_second:
message.timings!.predicted_n && message.timings!.predicted_ms
? (message.timings!.predicted_n / message.timings!.predicted_ms) * 1000
: 0,
cache_n: message.timings!.cache_n || 0
},
conversation.id
)
);
break;
}
}
if (!foundTimingData) {
untrack(() => clearProcessingState(conversation.id));
} }
} }
}); });
</script> </script>
<div class="chat-processing-info-container pointer-events-none" class:visible={showSlotsInfo}> <div class="chat-processing-info-container pointer-events-none" class:visible={showProcessingInfo}>
<div class="chat-processing-info-content"> <div class="chat-processing-info-content">
{#each processingDetails as detail (detail)} {#each processingDetails as detail (detail)}
<span class="chat-processing-info-detail pointer-events-auto">{detail}</span> <span class="chat-processing-info-detail pointer-events-auto">{detail}</span>

95
tools/server/webui/src/lib/hooks/use-processing-state.svelte.ts
View File

@ -1,8 +1,4 @@
import { import { activeProcessingState } from '$lib/stores/chat.svelte';
subscribeToProcessingState,
getCurrentProcessingState,
isChatStreaming
} from '$lib/stores/chat.svelte';
import { config } from '$lib/stores/settings.svelte'; import { config } from '$lib/stores/settings.svelte';
export interface UseProcessingStateReturn { export interface UseProcessingStateReturn {
@ -10,7 +6,7 @@ export interface UseProcessingStateReturn {
getProcessingDetails(): string[]; getProcessingDetails(): string[];
getProcessingMessage(): string; getProcessingMessage(): string;
shouldShowDetails(): boolean; shouldShowDetails(): boolean;
startMonitoring(): Promise<void>; startMonitoring(): void;
stopMonitoring(): void; stopMonitoring(): void;
} }
@ -18,93 +14,71 @@ export interface UseProcessingStateReturn {
* useProcessingState - Reactive processing state hook * useProcessingState - Reactive processing state hook
* *
* This hook provides reactive access to the processing state of the server. * This hook provides reactive access to the processing state of the server.
* It subscribes to timing data updates from ChatStore and provides * It directly reads from ChatStore's reactive state and provides
* formatted processing details for UI display. * formatted processing details for UI display.
* *
* **Features:** * **Features:**
* - Real-time processing state monitoring * - Real-time processing state via direct reactive state binding
* - Context and output token tracking * - Context and output token tracking
* - Tokens per second calculation * - Tokens per second calculation
* - Graceful degradation when slots endpoint unavailable * - Automatic updates when streaming data arrives
* - Automatic cleanup on component unmount * - Supports multiple concurrent conversations
* *
* @returns Hook interface with processing state and control methods * @returns Hook interface with processing state and control methods
*/ */
export function useProcessingState(): UseProcessingStateReturn { export function useProcessingState(): UseProcessingStateReturn {
let isMonitoring = $state(false); let isMonitoring = $state(false);
let processingState = $state<ApiProcessingState | null>(null);
let lastKnownState = $state<ApiProcessingState | null>(null); let lastKnownState = $state<ApiProcessingState | null>(null);
let unsubscribe: (() => void) | null = null;
async function startMonitoring(): Promise<void> { // Derive processing state reactively from ChatStore's direct state
if (isMonitoring) return; const processingState = $derived.by(() => {
if (!isMonitoring) {
isMonitoring = true; return lastKnownState;
unsubscribe = subscribeToProcessingState((state) => {
processingState = state;
if (state) {
lastKnownState = state;
} else {
lastKnownState = null;
}
});
try {
const currentState = await getCurrentProcessingState();
if (currentState) {
processingState = currentState;
lastKnownState = currentState;
}
// Check if streaming is active for UI purposes
if (isChatStreaming()) {
// Streaming is active, state will be updated via subscription
}
} catch (error) {
console.warn('Failed to start processing state monitoring:', error);
// Continue without monitoring - graceful degradation
} }
// Read directly from the reactive state export
return activeProcessingState();
});
// Track last known state for keepStatsVisible functionality
$effect(() => {
if (processingState && isMonitoring) {
lastKnownState = processingState;
}
});
function startMonitoring(): void {
if (isMonitoring) return;
isMonitoring = true;
} }
function stopMonitoring(): void { function stopMonitoring(): void {
if (!isMonitoring) return; if (!isMonitoring) return;
isMonitoring = false; isMonitoring = false;
// Only clear processing state if keepStatsVisible is disabled // Only clear last known state if keepStatsVisible is disabled
// This preserves the last known state for display when stats should remain visible
const currentConfig = config(); const currentConfig = config();
if (!currentConfig.keepStatsVisible) { if (!currentConfig.keepStatsVisible) {
processingState = null; lastKnownState = null;
} else if (lastKnownState) {
// Keep the last known state visible when keepStatsVisible is enabled
processingState = lastKnownState;
}
if (unsubscribe) {
unsubscribe();
unsubscribe = null;
} }
} }
function getProcessingMessage(): string { function getProcessingMessage(): string {
if (!processingState) { const state = processingState;
if (!state) {
return 'Processing...'; return 'Processing...';
} }
switch (processingState.status) { switch (state.status) {
case 'initializing': case 'initializing':
return 'Initializing...'; return 'Initializing...';
case 'preparing': case 'preparing':
if (processingState.progressPercent !== undefined) { if (state.progressPercent !== undefined) {
return `Processing (${processingState.progressPercent}%)`; return `Processing (${state.progressPercent}%)`;
} }
return 'Preparing response...'; return 'Preparing response...';
case 'generating': case 'generating':
if (processingState.tokensDecoded > 0) { if (state.tokensDecoded > 0) {
return `Generating... (${processingState.tokensDecoded} tokens)`; return `Generating... (${state.tokensDecoded} tokens)`;
} }
return 'Generating...'; return 'Generating...';
default: default:
@ -157,7 +131,8 @@ export function useProcessingState(): UseProcessingStateReturn {
} }
function shouldShowDetails(): boolean { function shouldShowDetails(): boolean {
return processingState !== null && processingState.status !== 'idle'; const state = processingState;
return state !== null && state.status !== 'idle';
} }
return { return {

107
tools/server/webui/src/lib/stores/chat.svelte.ts
View File

@ -5,7 +5,7 @@ import { config } from '$lib/stores/settings.svelte';
import { contextSize } from '$lib/stores/server.svelte'; import { contextSize } from '$lib/stores/server.svelte';
import { normalizeModelName } from '$lib/utils/model-names'; import { normalizeModelName } from '$lib/utils/model-names';
import { filterByLeafNodeId, findDescendantMessages, findLeafNode } from '$lib/utils/branching'; import { filterByLeafNodeId, findDescendantMessages, findLeafNode } from '$lib/utils/branching';
import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { SvelteMap } from 'svelte/reactivity';
import { DEFAULT_CONTEXT } from '$lib/constants/default-context'; import { DEFAULT_CONTEXT } from '$lib/constants/default-context';
import type { import type {
ChatMessageTimings, ChatMessageTimings,
@ -55,6 +55,7 @@ import type { DatabaseMessage, DatabaseMessageExtra } from '$lib/types/database'
* - Automatic state sync when switching between conversations * - Automatic state sync when switching between conversations
*/ */
class ChatStore { class ChatStore {
activeProcessingState = $state<ApiProcessingState | null>(null);
currentResponse = $state(''); currentResponse = $state('');
errorDialogState = $state<{ type: 'timeout' | 'server'; message: string } | null>(null); errorDialogState = $state<{ type: 'timeout' | 'server'; message: string } | null>(null);
isLoading = $state(false); isLoading = $state(false);
@ -66,10 +67,8 @@ class ChatStore {
// Processing state tracking - per-conversation timing/context info // Processing state tracking - per-conversation timing/context info
private processingStates = new SvelteMap<string, ApiProcessingState | null>(); private processingStates = new SvelteMap<string, ApiProcessingState | null>();
private processingCallbacks = new SvelteSet<(state: ApiProcessingState | null) => void>();
private activeConversationId = $state<string | null>(null); private activeConversationId = $state<string | null>(null);
private isStreamingActive = $state(false); private isStreamingActive = $state(false);
private lastKnownProcessingState = $state<ApiProcessingState | null>(null);
// ============ API Options ============ // ============ API Options ============
@ -190,7 +189,12 @@ class ChatStore {
*/ */
setActiveProcessingConversation(conversationId: string | null): void { setActiveProcessingConversation(conversationId: string | null): void {
this.activeConversationId = conversationId; this.activeConversationId = conversationId;
this.notifyProcessingCallbacks();
if (conversationId) {
this.activeProcessingState = this.processingStates.get(conversationId) || null;
} else {
this.activeProcessingState = null;
}
} }
/** /**
@ -207,24 +211,16 @@ class ChatStore {
this.processingStates.delete(conversationId); this.processingStates.delete(conversationId);
if (conversationId === this.activeConversationId) { if (conversationId === this.activeConversationId) {
this.lastKnownProcessingState = null; this.activeProcessingState = null;
this.notifyProcessingCallbacks();
} }
} }
/** /**
* Subscribe to processing state changes * Get the current processing state for the active conversation (reactive)
* Returns the direct reactive state for UI binding
*/ */
subscribeToProcessingState(callback: (state: ApiProcessingState | null) => void): () => void { getActiveProcessingState(): ApiProcessingState | null {
this.processingCallbacks.add(callback); return this.activeProcessingState;
if (this.lastKnownProcessingState) {
callback(this.lastKnownProcessingState);
}
return () => {
this.processingCallbacks.delete(callback);
};
} }
/** /**
@ -247,36 +243,28 @@ class ChatStore {
return; return;
} }
if (conversationId) { const targetId = conversationId || this.activeConversationId;
this.processingStates.set(conversationId, processingState); if (targetId) {
this.processingStates.set(targetId, processingState);
if (conversationId === this.activeConversationId) { if (targetId === this.activeConversationId) {
this.lastKnownProcessingState = processingState; this.activeProcessingState = processingState;
this.notifyProcessingCallbacks();
} }
} else {
this.lastKnownProcessingState = processingState;
this.notifyProcessingCallbacks();
} }
} }
/** /**
* Get current processing state * Get current processing state (sync version for reactive access)
*/ */
async getCurrentProcessingState(): Promise<ApiProcessingState | null> { getCurrentProcessingStateSync(): ApiProcessingState | null {
if (this.activeConversationId) { return this.activeProcessingState;
const conversationState = this.processingStates.get(this.activeConversationId); }
if (conversationState) {
return conversationState;
}
}
if (this.lastKnownProcessingState) { /**
return this.lastKnownProcessingState; * Restore processing state from last assistant message timings
} * Call this when keepStatsVisible is enabled and we need to show last known stats
*/
// Try to restore from last assistant message restoreProcessingStateFromMessages(messages: DatabaseMessage[], conversationId: string): void {
const messages = conversationsStore.activeMessages;
for (let i = messages.length - 1; i >= 0; i--) { for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i]; const message = messages[i];
if (message.role === 'assistant' && message.timings) { if (message.role === 'assistant' && message.timings) {
@ -291,32 +279,23 @@ class ChatStore {
}); });
if (restoredState) { if (restoredState) {
this.lastKnownProcessingState = restoredState; this.processingStates.set(conversationId, restoredState);
return restoredState;
if (conversationId === this.activeConversationId) {
this.activeProcessingState = restoredState;
}
return;
} }
} }
} }
return null;
}
private notifyProcessingCallbacks(): void {
const currentState = this.activeConversationId
? this.processingStates.get(this.activeConversationId) || null
: this.lastKnownProcessingState;
for (const callback of this.processingCallbacks) {
try {
callback(currentState);
} catch (error) {
console.error('Error in processing state callback:', error);
}
}
} }
private getContextTotal(): number { private getContextTotal(): number {
if (this.lastKnownProcessingState && this.lastKnownProcessingState.contextTotal > 0) { const activeState = this.getActiveProcessingState();
return this.lastKnownProcessingState.contextTotal;
if (activeState && activeState.contextTotal > 0) {
return activeState.contextTotal;
} }
const propsContextSize = contextSize(); const propsContextSize = contextSize();
@ -734,7 +713,7 @@ class ChatStore {
content: streamingState.response content: streamingState.response
}; };
if (lastMessage.thinking?.trim()) updateData.thinking = lastMessage.thinking; if (lastMessage.thinking?.trim()) updateData.thinking = lastMessage.thinking;
const lastKnownState = await this.getCurrentProcessingState(); const lastKnownState = this.getCurrentProcessingStateSync();
if (lastKnownState) { if (lastKnownState) {
updateData.timings = { updateData.timings = {
prompt_n: lastKnownState.promptTokens || 0, prompt_n: lastKnownState.promptTokens || 0,
@ -1323,9 +1302,13 @@ export const syncLoadingStateForChat = chatStore.syncLoadingStateForChat.bind(ch
export const clearUIState = chatStore.clearUIState.bind(chatStore); export const clearUIState = chatStore.clearUIState.bind(chatStore);
// Processing state (timing/context info) // Processing state (timing/context info)
export const subscribeToProcessingState = chatStore.subscribeToProcessingState.bind(chatStore);
export const getProcessingState = chatStore.getProcessingState.bind(chatStore); export const getProcessingState = chatStore.getProcessingState.bind(chatStore);
export const getCurrentProcessingState = chatStore.getCurrentProcessingState.bind(chatStore); export const getActiveProcessingState = chatStore.getActiveProcessingState.bind(chatStore);
export const activeProcessingState = () => chatStore.activeProcessingState;
export const getCurrentProcessingStateSync =
chatStore.getCurrentProcessingStateSync.bind(chatStore);
export const restoreProcessingStateFromMessages =
chatStore.restoreProcessingStateFromMessages.bind(chatStore);
export const clearProcessingState = chatStore.clearProcessingState.bind(chatStore); export const clearProcessingState = chatStore.clearProcessingState.bind(chatStore);
export const updateProcessingStateFromTimings = export const updateProcessingStateFromTimings =
chatStore.updateProcessingStateFromTimings.bind(chatStore); chatStore.updateProcessingStateFromTimings.bind(chatStore);