diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 2d5f95ea3e..d820e47c27 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -4982,7 +4982,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
 
         // Try to find a non-graphics compute queue and transfer-focused queues
         // On AMD GPUs with RADV driver, the graphics queue seems to be faster, so don't avoid it
-        const vk::QueueFlagBits graphics_flag = (device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eMesaRadv) ? (vk::QueueFlagBits)0 : vk::QueueFlagBits::eGraphics;
+        // Avoid on small GPUs to prevent interfering with graphics tasks
+        const vk::QueueFlagBits graphics_flag = (device->vendor_id == VK_VENDOR_ID_AMD && device->driver_id == vk::DriverId::eMesaRadv && device->shader_core_count > 16) ? (vk::QueueFlagBits)0 : vk::QueueFlagBits::eGraphics;
         const uint32_t compute_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eCompute, graphics_flag, -1, 1);
         const uint32_t transfer_queue_family_index = ggml_vk_find_queue_family_index(queue_family_props, vk::QueueFlagBits::eTransfer, vk::QueueFlagBits::eCompute | graphics_flag, compute_queue_family_index, 1);