diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 9973572f62..01ac24200a 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -196,10 +196,14 @@ struct server_slot {
     }
 
     void init_sampler() const {
-        const int64_t t_start = ggml_time_us();
-
         common_sampler_reset(smpl.get());
 
+        if (!task->need_sampling()) {
+            return;
+        }
+
+        const int64_t t_start = ggml_time_us();
+
         int n_text = 0;
 
         for (int i = 0; i < (int) prompt.tokens.size(); i++) {