diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz
index f1d9e30b34..e2f96687d0 100644
Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ
diff --git a/tools/server/webui/src/lib/services/chat.ts b/tools/server/webui/src/lib/services/chat.ts
index 927ce329ae..a7f2e0b6b0 100644
--- a/tools/server/webui/src/lib/services/chat.ts
+++ b/tools/server/webui/src/lib/services/chat.ts
@@ -1128,37 +1128,5 @@ export class ChatService {
throw err;
}
}
- /**
- * Tokenizes the provided text using the server's tokenizer.
- *
- * @param content - The text content to tokenize
- * @param model - Optional model name to use for tokenization (required in router mode)
- * @param signal - Optional AbortSignal
- * @returns {Promise} Promise that resolves to an array of token IDs
- */
- static async tokenize(content: string, model?: string, signal?: AbortSignal): Promise {
- try {
- const body: { content: string; model?: string } = { content };
- if (model) {
- body.model = model;
- }
- const response = await fetch('./tokenize', {
- method: 'POST',
- headers: getJsonHeaders(),
- body: JSON.stringify(body),
- signal
- });
-
- if (!response.ok) {
- throw new Error(`Tokenize failed: ${response.statusText}`);
- }
-
- const data = await response.json();
- return data.tokens;
- } catch (error) {
- console.error('Tokenize error:', error);
- return [];
- }
- }
}
diff --git a/tools/server/webui/src/lib/services/tokenize.ts b/tools/server/webui/src/lib/services/tokenize.ts
new file mode 100644
index 0000000000..2c6469e4f2
--- /dev/null
+++ b/tools/server/webui/src/lib/services/tokenize.ts
@@ -0,0 +1,39 @@
+import { getJsonHeaders } from '$lib/utils';
+
+/**
+ * Tokenizes the provided text using the server's tokenizer.
+ *
+ * @param content - The text content to tokenize
+ * @param model - Optional model name to use for tokenization (required in router mode)
+ * @param signal - Optional AbortSignal
+ * @returns {Promise} Promise that resolves to an array of token IDs
+ */
+export async function tokenize(
+ content: string,
+ model?: string,
+ signal?: AbortSignal
+): Promise {
+ try {
+ const body: { content: string; model?: string } = { content };
+ if (model) {
+ body.model = model;
+ }
+
+ const response = await fetch('./tokenize', {
+ method: 'POST',
+ headers: getJsonHeaders(),
+ body: JSON.stringify(body),
+ signal
+ });
+
+ if (!response.ok) {
+ throw new Error(`Tokenize failed: ${response.statusText}`);
+ }
+
+ const data = await response.json();
+ return data.tokens;
+ } catch (error) {
+ console.error('Tokenize error:', error);
+ return [];
+ }
+}
diff --git a/tools/server/webui/src/lib/stores/notebook.svelte.ts b/tools/server/webui/src/lib/stores/notebook.svelte.ts
index 267755db08..70917def9c 100644
--- a/tools/server/webui/src/lib/stores/notebook.svelte.ts
+++ b/tools/server/webui/src/lib/stores/notebook.svelte.ts
@@ -1,5 +1,6 @@
import { ChatService } from '$lib/services/chat';
import { config } from '$lib/stores/settings.svelte';
+import { tokenize } from '$lib/services/tokenize';
export class NotebookStore {
content = $state('');
@@ -144,7 +145,7 @@ export class NotebookStore {
this.totalTokens = 0;
return;
}
- const tokens = await ChatService.tokenize(this.content, model);
+ const tokens = await tokenize(this.content, model);
this.totalTokens = tokens.length;
}, 500);
}