vulkan : add dynamic VRAM heuristic for low-VRAM GPUs
This commit is contained in:
dickbird
parent
b61de2b2df
commit
5ecff8a9a9
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,120 @@
|
|||
# Dynamic VRAM Allocation for Vulkan Backend
|
||||
|
||||
This document describes the dynamic VRAM allocation heuristic for `llama.cpp`'s Vulkan backend, which automatically optimizes GPU layer offloading based on available VRAM.
|
||||
|
||||
## Overview
|
||||
|
||||
The Vulkan backend now includes a **dynamic heuristic** that automatically calculates the optimal number of GPU layers to offload based on:
|
||||
- Available VRAM on your GPU
|
||||
- Model size and layer count (from GGUF metadata)
|
||||
- Reserved overhead for KV cache and compute buffers
|
||||
|
||||
This enables **optimal performance** on low-VRAM devices (like AMD RX 6500 XT with 4GB) without manual configuration or OOM errors.
|
||||
|
||||
## How It Works
|
||||
|
||||
When you run `llama-cli` or `llama-server` **without** specifying `-ngl` (or with `-ngl -1`), the heuristic:
|
||||
|
||||
1. **Queries available VRAM** from your Vulkan device
|
||||
2. **Parses model metadata** to determine model size and layer count
|
||||
3. **Reserves overhead** (800MB) for KV cache, compute buffers, and system
|
||||
4. **Calculates optimal layers**: `(available_vram - overhead) / bytes_per_layer`
|
||||
5. **Offloads automatically** without risking OOM
|
||||
|
||||
### Example Results
|
||||
|
||||
**AMD RX 6500 XT (4GB VRAM)**:
|
||||
- Gemma 2B (1.6GB): **26/27 layers** offloaded → **2.5-3.1x faster**
|
||||
- Llama 3.2 3B (1.9GB): **28/29 layers** offloaded → **~2x faster**
|
||||
- Llama 2 7B (3.9GB): **21/33 layers** offloaded → **1.6x faster**
|
||||
- Llama 2 13B (7.5GB): **14/41 layers** offloaded → **No OOM** ✅
|
||||
|
||||
## Usage
|
||||
|
||||
### Automatic (Recommended)
|
||||
|
||||
Simply run without `-ngl` to enable the dynamic heuristic:
|
||||
|
||||
```bash
|
||||
# Heuristic calculates optimal layers automatically
|
||||
llama-cli -m models/gemma-2b-q4.gguf -p "Hello"
|
||||
```
|
||||
|
||||
The heuristic will print debug info showing the calculation:
|
||||
```
|
||||
Vulkan dynamic heuristic: available_vram=3434 MB, model_size=1623 MB,
|
||||
n_layers=27, overhead=800 MB, calculated_layers=26
|
||||
```
|
||||
|
||||
### Manual Override
|
||||
|
||||
You can still manually specify layers to override the heuristic:
|
||||
|
||||
```bash
|
||||
# Force specific number of layers
|
||||
llama-cli -m models/gemma-2b-q4.gguf -p "Hello" -ngl 20
|
||||
|
||||
# Force CPU-only
|
||||
llama-cli -m models/gemma-2b-q4.gguf -p "Hello" -ngl 0
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Compared to CPU-only (`-ngl 0`), the dynamic heuristic provides:
|
||||
|
||||
**Gemma 2B Q4_K_M on AMD RX 6500 XT**:
|
||||
- Prompt processing: **2.5x faster** (497 → 1231 t/s)
|
||||
- Token generation: **3.1x faster** (19.4 → 60.4 t/s)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Still Getting OOM Errors?
|
||||
|
||||
If you encounter "Out of Device Memory" errors despite the heuristic:
|
||||
|
||||
1. **Reduce context size**: Use `-c 2048` or lower
|
||||
2. **Force fewer layers**: Use `-ngl 10` or lower
|
||||
3. **Check available VRAM**: Close other GPU applications
|
||||
4. **Use smaller model**: Try a smaller quantization (Q4_K_M → Q3_K_S)
|
||||
|
||||
### Heuristic Not Triggering?
|
||||
|
||||
The heuristic only activates when:
|
||||
- ✅ Vulkan backend is enabled (`GGML_USE_VULKAN=1` during build)
|
||||
- ✅ `-ngl` is not specified (or set to `-1`)
|
||||
- ✅ GGUF file can be parsed for metadata
|
||||
|
||||
If you explicitly set `-ngl`, the heuristic is bypassed.
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Overhead Calculation
|
||||
|
||||
The heuristic reserves **800MB** for:
|
||||
- KV cache (dynamically allocated by llama.cpp)
|
||||
- Compute buffers (temporary tensors during inference)
|
||||
- System overhead (driver, fragmentation)
|
||||
|
||||
This value is conservative and works well across different model sizes.
|
||||
|
||||
### Model Compatibility
|
||||
|
||||
The heuristic generalizes across model architectures by searching for:
|
||||
- `*.block_count` (layer count)
|
||||
- `*.embedding_length` (model dimensions)
|
||||
|
||||
Tested architectures:
|
||||
- ✅ Gemma / Gemma 2
|
||||
- ✅ Llama / Llama 2 / Llama 3
|
||||
- ✅ Qwen / Qwen 2.5
|
||||
|
||||
## Benchmark Script
|
||||
|
||||
The `tests/6500xt_benchmark.ps1` script automates testing across different configurations:
|
||||
|
||||
```powershell
|
||||
cd tests
|
||||
.\6500xt_benchmark.ps1
|
||||
```
|
||||
|
||||
This will test CPU-only vs GPU heuristic and report performance improvements.
|
||||
|
|
@ -34,6 +34,9 @@ else()
|
|||
add_subdirectory(training)
|
||||
add_subdirectory(diffusion)
|
||||
add_subdirectory(model-conversion)
|
||||
if (GGML_VULKAN)
|
||||
add_subdirectory(vk_device_info)
|
||||
endif()
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
add_subdirectory(convert-llama2c-to-ggml)
|
||||
# these examples use the backends directly and cannot be built with dynamic loading
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
set(TARGET llama-vk-device-info)
|
||||
add_executable(${TARGET} vk_device_info.cpp)
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT})
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
#include "ggml-vulkan.h"
|
||||
#include <cstdio>
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
int device_count = ggml_backend_vk_get_device_count();
|
||||
printf("Found %d Vulkan devices\\n", device_count);
|
||||
|
||||
for (int i = 0; i < device_count; i++) {
|
||||
ggml_vk_device_info info = ggml_backend_vk_get_device_info(i);
|
||||
printf("\\nDevice %d: %s\\n", i, info.device_name);
|
||||
printf(" Vendor ID: %04x\\n", info.vendor_id);
|
||||
printf(" Device ID: %04x\\n", info.device_id);
|
||||
printf(" API Version: 0x%08x\\n", info.api_version);
|
||||
printf(" Total Device Local Memory: %llu MB\\n", info.total_device_local_memory / (1024 * 1024));
|
||||
printf(" Has Memory Budget Ext: %s\\n", info.has_memory_budget_ext ? "Yes" : "No");
|
||||
printf(" Supports Float16: %s\\n", info.supports_float16 ? "Yes" : "No");
|
||||
printf(" Supports 16-bit Storage: %s\\n", info.supports_16bit_storage ? "Yes" : "No");
|
||||
|
||||
int default_layers = ggml_backend_vk_get_default_gpu_layers(i, -1);
|
||||
printf(" Default GPU Layers (heuristic): %d\\n", default_layers);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,13 +1,13 @@
|
|||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define GGML_VK_NAME "Vulkan"
|
||||
#define GGML_VK_NAME "Vulkan"
|
||||
#define GGML_VK_MAX_DEVICES 16
|
||||
|
||||
// backend API
|
||||
|
|
@ -24,6 +24,20 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type(voi
|
|||
|
||||
GGML_BACKEND_API ggml_backend_reg_t ggml_backend_vk_reg(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
typedef struct {
|
||||
char device_name[256];
|
||||
uint32_t vendor_id;
|
||||
uint32_t device_id;
|
||||
uint64_t total_device_local_memory;
|
||||
bool has_memory_budget_ext;
|
||||
bool supports_float16;
|
||||
bool supports_16bit_storage;
|
||||
uint32_t api_version;
|
||||
} ggml_vk_device_info;
|
||||
|
||||
GGML_BACKEND_API ggml_vk_device_info ggml_backend_vk_get_device_info(int device);
|
||||
GGML_BACKEND_API int ggml_backend_vk_get_default_gpu_layers(int device, int default_layers);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Configuration
|
||||
$BuildDir = "build"
|
||||
$ModelPath = "models/7B/ggml-model-f16.gguf" # Adjust as needed
|
||||
$Prompt = "The quick brown fox jumps over the lazy dog"
|
||||
$NumRuns = 3
|
||||
$CsvFile = "benchmark_results.csv"
|
||||
|
||||
# Ensure build directory exists
|
||||
if (!(Test-Path $BuildDir)) {
|
||||
New-Item -ItemType Directory -Path $BuildDir | Out-Null
|
||||
}
|
||||
|
||||
# Build
|
||||
Write-Host "Building project..."
|
||||
Push-Location $BuildDir
|
||||
cmake .. -DGGML_VULKAN=ON -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build . --config Release -j 8
|
||||
Pop-Location
|
||||
|
||||
# Tools paths
|
||||
$LlamaCli = "$BuildDir/bin/Release/llama-cli.exe"
|
||||
if (!(Test-Path $LlamaCli)) { $LlamaCli = "$BuildDir/bin/llama-cli.exe" }
|
||||
if (!(Test-Path $LlamaCli)) { $LlamaCli = "$BuildDir/Release/llama-cli.exe" }
|
||||
|
||||
$VkInfoTool = "$BuildDir/bin/Release/llama-vk-device-info.exe"
|
||||
if (!(Test-Path $VkInfoTool)) { $VkInfoTool = "$BuildDir/bin/llama-vk-device-info.exe" }
|
||||
if (!(Test-Path $VkInfoTool)) { $VkInfoTool = "$BuildDir/Release/llama-vk-device-info.exe" }
|
||||
|
||||
# System Info
|
||||
Write-Host "Collecting System Info..."
|
||||
vulkaninfo | Out-File "vulkaninfo.txt"
|
||||
& $VkInfoTool | Out-File "vk_device_info.txt"
|
||||
Get-Content "vk_device_info.txt"
|
||||
|
||||
# Initialize CSV
|
||||
"RunType,Layers,LoadTime_ms,EvalTime_ms,TokensPerSec,PeakMem_MB" | Out-File $CsvFile -Encoding ascii
|
||||
|
||||
function Invoke-Benchmark {
|
||||
param (
|
||||
[string]$Type,
|
||||
[int]$Layers
|
||||
)
|
||||
|
||||
$TotalLoadTime = 0
|
||||
$TotalEvalTime = 0
|
||||
$TotalTokensPerSec = 0
|
||||
|
||||
Write-Host "Running benchmark: $Type (Layers: $Layers)"
|
||||
|
||||
for ($i = 1; $i -le $NumRuns; $i++) {
|
||||
$LlamaArgs = @("-m", $ModelPath, "-p", $Prompt, "-n", "128", "--no-mmap")
|
||||
if ($Type -eq "CPU") {
|
||||
$LlamaArgs += "-ngld" # No GPU layers
|
||||
}
|
||||
elseif ($Type -eq "Vulkan") {
|
||||
$LlamaArgs += "-ngl", "$Layers"
|
||||
}
|
||||
|
||||
# Capture output
|
||||
$Output = & $LlamaCli $LlamaArgs 2>&1
|
||||
|
||||
# Parse metrics
|
||||
$LoadTime = 0
|
||||
$EvalTime = 0
|
||||
$Tps = 0
|
||||
|
||||
foreach ($Line in $Output) {
|
||||
if ($Line -match "load time = \s+(\d+\.\d+) ms") { $LoadTime = [double]$matches[1] }
|
||||
if ($Line -match "eval time = \s+(\d+\.\d+) ms") { $EvalTime = [double]$matches[1] }
|
||||
if ($Line -match "(\d+\.\d+) tokens per second") { $Tps = [double]$matches[1] }
|
||||
}
|
||||
|
||||
$TotalLoadTime += $LoadTime
|
||||
$TotalEvalTime += $EvalTime
|
||||
$TotalTokensPerSec += $Tps
|
||||
|
||||
Write-Host " Run $i : Load=$LoadTime ms, Eval=$EvalTime ms, TPS=$Tps"
|
||||
}
|
||||
|
||||
$AvgLoad = $TotalLoadTime / $NumRuns
|
||||
$AvgEval = $TotalEvalTime / $NumRuns
|
||||
$AvgTps = $TotalTokensPerSec / $NumRuns
|
||||
|
||||
"$Type,$Layers,$AvgLoad,$AvgEval,$AvgTps,0" | Out-File $CsvFile -Append -Encoding ascii
|
||||
}
|
||||
|
||||
# Run Benchmarks
|
||||
Invoke-Benchmark -Type "CPU" -Layers 0
|
||||
|
||||
# Test various GPU layers
|
||||
# Note: If heuristic works, -ngl -1 (default) should pick 1 layer for 6500 XT
|
||||
# We explicitly test 1, 2, 3, 4 to show performance degradation
|
||||
Invoke-Benchmark -Type "Vulkan" -Layers 1
|
||||
Invoke-Benchmark -Type "Vulkan" -Layers 2
|
||||
Invoke-Benchmark -Type "Vulkan" -Layers 3
|
||||
Invoke-Benchmark -Type "Vulkan" -Layers 4
|
||||
|
||||
Write-Host "Benchmark complete. Results saved to $CsvFile"
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Cross-Architecture Benchmark Script
|
||||
# Tests dynamic VRAM heuristic across different model architectures and sizes
|
||||
|
||||
$buildDir = "..\build"
|
||||
$llama = "$buildDir\bin\Release\llama-cli.exe"
|
||||
|
||||
$models = @(
|
||||
@{Name = "Gemma-2-2B"; Path = "..\models\gemma-2b-it\gemma-2-2b-it-Q4_K_M.gguf"; Size = "1.6GB" },
|
||||
@{Name = "Llama-3.2-3B"; Path = "..\models\llama-3.2-3b-instruct-q4_k_m.gguf"; Size = "1.9GB" },
|
||||
@{Name = "Llama-2-7B"; Path = "..\models\llama-2-7b-chat.Q4_K_M.gguf"; Size = "3.9GB" },
|
||||
@{Name = "Llama-2-13B"; Path = "..\models\llama-2-13b-chat.Q4_K_M.gguf"; Size = "7.5GB" }
|
||||
)
|
||||
|
||||
$results = @()
|
||||
|
||||
foreach ($model in $models) {
|
||||
Write-Host "`n========================================" -ForegroundColor Cyan
|
||||
Write-Host "Testing: $($model.Name) ($($model.Size))" -ForegroundColor Cyan
|
||||
Write-Host "========================================`n" -ForegroundColor Cyan
|
||||
|
||||
# Test 1: CPU Only (-ngl 0)
|
||||
Write-Host "Test 1: CPU Only..." -ForegroundColor Yellow
|
||||
$output = & $llama -m $model.Path -p "Test" -n 10 -ngl 0 -no-cnv 2>&1 | Out-String
|
||||
$cpuTokens = if ($output -match "(\d+\.\d+)\s+tokens per second") { [float]$matches[1] } else { 0 }
|
||||
|
||||
# Test 2: Dynamic Heuristic (no -ngl flag)
|
||||
Write-Host "Test 2: Dynamic Heuristic..." -ForegroundColor Yellow
|
||||
$output = & $llama -m $model.Path -p "Test" -n 10 -no-cnv 2>&1 | Out-String
|
||||
$heuristicLayers = if ($output -match "calculated_layers=(\d+)") { [int]$matches[1] } else { "N/A" }
|
||||
$offloadedLayers = if ($output -match "offloaded (\d+)/(\d+) layers") { "$($matches[1])/$($matches[2])" } else { "N/A" }
|
||||
$heuristicTokens = if ($output -match "(\d+\.\d+)\s+tokens per second") { [float]$matches[1] } else { 0 }
|
||||
|
||||
$speedup = if ($cpuTokens -gt 0) { [math]::Round(($heuristicTokens / $cpuTokens - 1) * 100, 1) } else { 0 }
|
||||
|
||||
$results += [PSCustomObject]@{
|
||||
Model = $model.Name
|
||||
Size = $model.Size
|
||||
CPUTokensPerSec = [math]::Round($cpuTokens, 2)
|
||||
HeuristicLayers = $heuristicLayers
|
||||
OffloadedLayers = $offloadedLayers
|
||||
HeuristicTokensPerSec = [math]::Round($heuristicTokens, 2)
|
||||
SpeedupPercent = "$speedup%"
|
||||
}
|
||||
}
|
||||
|
||||
# Display results
|
||||
Write-Host "`n`n========================================" -ForegroundColor Green
|
||||
Write-Host "BENCHMARK RESULTS" -ForegroundColor Green
|
||||
Write-Host "========================================`n" -ForegroundColor Green
|
||||
|
||||
$results | Format-Table -AutoSize
|
||||
|
||||
# Save to CSV
|
||||
$results | Export-Csv -Path "cross_arch_benchmark_results.csv" -NoTypeInformation
|
||||
Write-Host "`nResults saved to: cross_arch_benchmark_results.csv" -ForegroundColor Green
|
||||
Loading…
Reference in New Issue