@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c391e4b68e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] improve robustness of pynccl (#3860)

youkaichao committed Apr 4, 2024 at 16:52 UTC c391e4b68e6694986f24ccd620d7bf07c237ab60
1 file changed +13 -8
vllm/model_executor/parallel_utils/pynccl.py
+13 -8
@@ -236,22 +236,25 @@ class NCCLCommunicator:
236 if local_rank == -1:
237 local_rank = self.rank
238 self.local_rank = local_rank
239 - torch.cuda.set_device(local_rank)
240 - if rank == 0:
239 + # don't use these args, as they can be -1
240 + # use `self.rank`, `self.local_rank` and `self.world_size` instead
241 + del world_size, rank, local_rank
242 + torch.cuda.set_device(self.local_rank)
243 + if self.rank == 0:
244 self.unique_id = ncclGetUniqueId()
245 else:
246 self.unique_id = NcclUniqueId()
244 - tensor = torch.ByteTensor(list(
245 - self.unique_id.internal)).cuda(local_rank)
247 + tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
248 + self.local_rank)
249 dist.broadcast(tensor, src=0)
250 byte_list = tensor.cpu().tolist()
251 for i, byte in enumerate(byte_list):
252 self.unique_id.internal[i] = byte
253 self.comm = ctypes.c_void_p()
251 - result = _c_ncclCommInitRank(ctypes.byref(self.comm), world_size,
252 - self.unique_id, rank)
254 + result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
255 + self.unique_id, self.rank)
256 assert result == 0
254 - self.stream = torch.cuda.Stream(device=f"cuda:{local_rank}")
257 + self.stream = torch.cuda.Stream(device=f"cuda:{self.local_rank}")
258
259 def all_reduce(self,
260 tensor: torch.Tensor,
@@ -271,4 +274,6 @@ class NCCLCommunicator:
274 # `dist` module might have been already destroyed
275 if hasattr(dist, 'destroy_process_group'):
276 dist.destroy_process_group()
274 - _c_ncclCommDestroy(self.comm)
277 + # function might have been already destroyed
278 + if _c_ncclCommDestroy is not None:
279 + _c_ncclCommDestroy(self.comm)