A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 21:30 UTC
[BugFix] Prevent orphaned process on NCCL destroy (#39846)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Jeffrey Wang committed
May 11, 2026 at 12:25 UTC
56e5810ff168bb13695d35fba2c7401256afdda4
2 files changed
+24
-2
vllm/distributed/device_communicators/pynccl.py
+15
-2
@@ -3,6 +3,8 @@
3
4
5
# ===================== import region =====================
6
+import threading
7
+
8
import torch
9
import torch.distributed as dist
10
from torch.distributed import ProcessGroup, ReduceOp
@@ -145,8 +147,19 @@ class PyNcclCommunicator:
147
148
def destroy(self):
149
if self.available and not self.disabled:
148
- with torch.accelerator.device_index(self.device.index):
149
- self.nccl.ncclCommDestroy(self.comm)
150
+ # ncclCommAbort can block until all CUDA graphs that
151
+ # captured NCCL ops on this comm are destroyed — and
152
+ # those graphs are released later in this same main-
153
+ # thread teardown, so a direct call here self-deadlocks.
154
+ # Run it in a daemon thread with a timeout: the main
155
+ # thread proceeds, the graphs drop, and the abort returns.
156
+ def _abort():
157
+ with torch.accelerator.device_index(self.device.index):
158
+ self.nccl.ncclCommAbort(self.comm)
159
+
160
+ abort_thread = threading.Thread(target=_abort, daemon=True)
161
+ abort_thread.start()
162
+ abort_thread.join(timeout=5.0)
163
self.available = False
164
self.disabled = True
165
vllm/distributed/device_communicators/pynccl_wrapper.py
+9
@@ -290,6 +290,12 @@ class NCCLLibrary:
290
# it is better not to call it at all.
291
# ncclResult_t ncclCommDestroy(ncclComm_t comm);
292
Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]),
293
+ # ncclCommAbort frees resources associated with the communicator
294
+ # without requiring a collective synchronization. Unlike
295
+ # ncclCommDestroy, it is safe to call during an uncoordinated
296
+ # shutdown when peer ranks may already be gone.
297
+ # ncclResult_t ncclCommAbort(ncclComm_t comm);
298
+ Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
299
# ncclResult_t ncclGroupStart();
300
Function("ncclGroupStart", ncclResult_t, []),
301
# ncclResult_t ncclGroupEnd();
@@ -548,6 +554,9 @@ class NCCLLibrary:
554
def ncclCommDestroy(self, comm: ncclComm_t) -> None:
555
self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
556
557
+ def ncclCommAbort(self, comm: ncclComm_t) -> None:
558
+ self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
559
+
560
def ncclGroupStart(self) -> None:
561
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
562