diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz index f1d9e30b34..e2f96687d0 100644 Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ diff --git a/tools/server/webui/src/lib/services/chat.ts b/tools/server/webui/src/lib/services/chat.ts index 927ce329ae..a7f2e0b6b0 100644 --- a/tools/server/webui/src/lib/services/chat.ts +++ b/tools/server/webui/src/lib/services/chat.ts @@ -1128,37 +1128,5 @@ export class ChatService { throw err; } } - /** - * Tokenizes the provided text using the server's tokenizer. - * - * @param content - The text content to tokenize - * @param model - Optional model name to use for tokenization (required in router mode) - * @param signal - Optional AbortSignal - * @returns {Promise} Promise that resolves to an array of token IDs - */ - static async tokenize(content: string, model?: string, signal?: AbortSignal): Promise { - try { - const body: { content: string; model?: string } = { content }; - if (model) { - body.model = model; - } - const response = await fetch('./tokenize', { - method: 'POST', - headers: getJsonHeaders(), - body: JSON.stringify(body), - signal - }); - - if (!response.ok) { - throw new Error(`Tokenize failed: ${response.statusText}`); - } - - const data = await response.json(); - return data.tokens; - } catch (error) { - console.error('Tokenize error:', error); - return []; - } - } } diff --git a/tools/server/webui/src/lib/services/tokenize.ts b/tools/server/webui/src/lib/services/tokenize.ts new file mode 100644 index 0000000000..2c6469e4f2 --- /dev/null +++ b/tools/server/webui/src/lib/services/tokenize.ts @@ -0,0 +1,39 @@ +import { getJsonHeaders } from '$lib/utils'; + +/** + * Tokenizes the provided text using the server's tokenizer. + * + * @param content - The text content to tokenize + * @param model - Optional model name to use for tokenization (required in router mode) + * @param signal - Optional AbortSignal + * @returns {Promise} Promise that resolves to an array of token IDs + */ +export async function tokenize( + content: string, + model?: string, + signal?: AbortSignal +): Promise { + try { + const body: { content: string; model?: string } = { content }; + if (model) { + body.model = model; + } + + const response = await fetch('./tokenize', { + method: 'POST', + headers: getJsonHeaders(), + body: JSON.stringify(body), + signal + }); + + if (!response.ok) { + throw new Error(`Tokenize failed: ${response.statusText}`); + } + + const data = await response.json(); + return data.tokens; + } catch (error) { + console.error('Tokenize error:', error); + return []; + } +} diff --git a/tools/server/webui/src/lib/stores/notebook.svelte.ts b/tools/server/webui/src/lib/stores/notebook.svelte.ts index 267755db08..70917def9c 100644 --- a/tools/server/webui/src/lib/stores/notebook.svelte.ts +++ b/tools/server/webui/src/lib/stores/notebook.svelte.ts @@ -1,5 +1,6 @@ import { ChatService } from '$lib/services/chat'; import { config } from '$lib/stores/settings.svelte'; +import { tokenize } from '$lib/services/tokenize'; export class NotebookStore { content = $state(''); @@ -144,7 +145,7 @@ export class NotebookStore { this.totalTokens = 0; return; } - const tokens = await ChatService.tokenize(this.content, model); + const tokens = await tokenize(this.content, model); this.totalTokens = tokens.length; }, 500); }