happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds function to relax sync requirements between input copies on 

supported backends (CUDA for now)
This commit is contained in:
aendk 2025-12-16 16:51:54 +01:00
parent 7c59ff01f2
commit 3f59431df7
1 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ggml/src/ggml-backend.cpp
View File

@ -11,6 +11,8 @@
#include "ggml-backend.h"
#include "ggml-backend-impl.h"
#include "ggml-alloc.h"
#include "ggml-cpu.h"
#include "ggml-cuda.h" // TODO add IFDEFs for CUDA-specific parts
#include "ggml-impl.h"
#include <assert.h>
@ -736,6 +738,19 @@ struct ggml_backend_sched {
int debug_prev_graph_size;
};
static void ggml_backend_synchronize_if_required(ggml_backend_t current_backend) {
// TODO add env-flag check here to auto-disable this change
// CUDA backends have an implicit order between execution and memory operations via the CUDA stream.
// Multiple parallel copies are also possible.
// There is consequently no need to synchronize in between computation and subsequent memcpys
if (ggml_backend_is_cuda(current_backend)) {
return;
}
// in all other cases, just sync.
ggml_backend_synchronize(current_backend);
}
#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)
#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]
#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]
@ -1464,7 +1479,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
} else {
ggml_backend_synchronize(split_backend);
ggml_backend_synchronize_if_required(split_backend);
}
ggml_backend_tensor_copy(input, input_cpy);
} else {
@ -1472,7 +1487,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
} else {
ggml_backend_synchronize(split_backend);
ggml_backend_synchronize_if_required(split_backend);
}
// when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used