Put tokenize in a separate file

tools/server/webui/src/lib/services/chat.ts

@@ -1128,37 +1128,5 @@ export class ChatService {
 			throw err;
 		}
 	}
-	/**
-	 * Tokenizes the provided text using the server's tokenizer.
-	 *
-	 * @param content - The text content to tokenize
-	 * @param model - Optional model name to use for tokenization (required in router mode)
-	 * @param signal - Optional AbortSignal
-	 * @returns {Promise<number[]>} Promise that resolves to an array of token IDs
-	 */
-	static async tokenize(content: string, model?: string, signal?: AbortSignal): Promise<number[]> {
-		try {
-			const body: { content: string; model?: string } = { content };
-			if (model) {
-				body.model = model;
-			}
 
-			const response = await fetch('./tokenize', {
-				method: 'POST',
-				headers: getJsonHeaders(),
-				body: JSON.stringify(body),
-				signal
-			});
-
-			if (!response.ok) {
-				throw new Error(`Tokenize failed: ${response.statusText}`);
-			}
-
-			const data = await response.json();
-			return data.tokens;
-		} catch (error) {
-			console.error('Tokenize error:', error);
-			return [];
-		}
-	}
 }

tools/server/webui/src/lib/services/tokenize.ts

@@ -0,0 +1,39 @@
+import { getJsonHeaders } from '$lib/utils';
+
+/**
+ * Tokenizes the provided text using the server's tokenizer.
+ *
+ * @param content - The text content to tokenize
+ * @param model - Optional model name to use for tokenization (required in router mode)
+ * @param signal - Optional AbortSignal
+ * @returns {Promise<number[]>} Promise that resolves to an array of token IDs
+ */
+export async function tokenize(
+    content: string,
+    model?: string,
+    signal?: AbortSignal
+): Promise<number[]> {
+    try {
+        const body: { content: string; model?: string } = { content };
+        if (model) {
+            body.model = model;
+        }
+
+        const response = await fetch('./tokenize', {
+            method: 'POST',
+            headers: getJsonHeaders(),
+            body: JSON.stringify(body),
+            signal
+        });
+
+        if (!response.ok) {
+            throw new Error(`Tokenize failed: ${response.statusText}`);
+        }
+
+        const data = await response.json();
+        return data.tokens;
+    } catch (error) {
+        console.error('Tokenize error:', error);
+        return [];
+    }
+}

tools/server/webui/src/lib/stores/notebook.svelte.ts

@@ -1,5 +1,6 @@
 import { ChatService } from '$lib/services/chat';
 import { config } from '$lib/stores/settings.svelte';
+import { tokenize } from '$lib/services/tokenize';
 
 export class NotebookStore {
 	content = $state('');
@@ -144,7 +145,7 @@ export class NotebookStore {
 				this.totalTokens = 0;
 				return;
 			}
-			const tokens = await ChatService.tokenize(this.content, model);
+			const tokens = await tokenize(this.content, model);
 			this.totalTokens = tokens.length;
 		}, 500);
 	}