A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 02:30 UTC
[Feature][Core] Support Fabric detection to adapt the MNNVL protocol for the GB series (#33540)
Signed-off-by: Kebe <mail@kebe7jun.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Thomas Vegas <tvegas@nvidia.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Kebe committed
Feb 2, 2026 at 23:55 UTC
528e9b14900fc8a012f2599172e2a4576caafe1a
1 file changed
+18
-1
csrc/cumem_allocator.cpp
+18
-1
@@ -115,11 +115,28 @@ void create_and_map(unsigned long long device, ssize_t size, CUdeviceptr d_mem,
115
if (flag) { // support GPUDirect RDMA if possible
116
prop.allocFlags.gpuDirectRDMACapable = 1;
117
}
118
+ int fab_flag = 0;
119
+ CUDA_CHECK(cuDeviceGetAttribute(
120
+ &fab_flag, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, device));
121
+ if (fab_flag) { // support fabric handle if possible
122
+ prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC;
123
+ }
124
#endif
125
126
#ifndef USE_ROCM
127
// Allocate memory using cuMemCreate
122
- CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
128
+ CUresult ret = (CUresult)cuMemCreate(p_memHandle, size, &prop, 0);
129
+ if (ret) {
130
+ if (fab_flag &&
131
+ (ret == CUDA_ERROR_NOT_PERMITTED || ret == CUDA_ERROR_NOT_SUPPORTED)) {
132
+ // Fabric allocation may fail without multi-node nvlink,
133
+ // fallback to POSIX file descriptor
134
+ prop.requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR;
135
+ CUDA_CHECK(cuMemCreate(p_memHandle, size, &prop, 0));
136
+ } else {
137
+ CUDA_CHECK(ret);
138
+ }
139
+ }
140
if (error_code != 0) {
141
return;
142
}