kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-07 19:50:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

ggml: Update VMM Pool allocation ggml-cuda.cu - Turing P2P access fix (fixes #24489) (#24491)

Browse Source 
* Update ggml-cuda.cu - Turing P2P access fix.

* Add original code as fallback behaviour when NCCL or P2P is not set/true.

* Update ggml/src/ggml-cuda/ggml-cuda.cu to add comment as per suggestion

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
Vexxie
2026-07-05 18:10:09 +01:00
committed by GitHub
parent 78d2f52468
commit 7a63fdede1
1 changed files with 36 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/ggml-cuda.cu
+36 -6
View File
@@ -543,12 +543,42 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
// the memory allocation handle is no longer needed after mapping
CU_CHECK(cuMemRelease(handle));
// set access
CUmemAccessDesc access = {};
access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
access.location.id = device;
access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
CU_CHECK(cuMemSetAccess((CUdeviceptr)((char *)(pool_addr) + pool_size), reserve_size, &access, 1));
// VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build
bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;
#if defined(GGML_USE_NCCL)
use_peer_access = true;
#endif // defined(GGML_USE_NCCL)
if (use_peer_access) {
// NCCL implicitly enables peer access (cudaDeviceEnablePeerAccess), and
// GGML_CUDA_P2P enables it explicitly. Unlike cudaMalloc buffers, VMM
// allocations do not become peer-accessible from that alone, so access
// must be granted explicitly here.
std::vector<CUmemAccessDesc> access_descs;
const int device_count = ggml_cuda_info().device_count;
for (int id = 0; id < device_count; ++id) {
if (id != device) {
int can_access_peer = 0;
CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, device));
if (!can_access_peer) {
continue;
}
}
CUmemAccessDesc access = {};
access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
access.location.id = id;
access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
access_descs.push_back(access);
}
CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, access_descs.data(), access_descs.size()));
} else {
// set access for non P2P
CUmemAccessDesc access = {};
access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
access.location.id = device;
access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, &access, 1));
}
// add to the pool
pool_size += reserve_size;