A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] improve robustness of pynccl (#3860)
youkaichao committed
Apr 4, 2024 at 16:52 UTC
c391e4b68e6694986f24ccd620d7bf07c237ab60
1 file changed
+13
-8
vllm/model_executor/parallel_utils/pynccl.py
+13
-8
@@ -236,22 +236,25 @@ class NCCLCommunicator:
236
if local_rank == -1:
237
local_rank = self.rank
238
self.local_rank = local_rank
239
- torch.cuda.set_device(local_rank)
240
- if rank == 0:
239
+ # don't use these args, as they can be -1
240
+ # use `self.rank`, `self.local_rank` and `self.world_size` instead
241
+ del world_size, rank, local_rank
242
+ torch.cuda.set_device(self.local_rank)
243
+ if self.rank == 0:
244
self.unique_id = ncclGetUniqueId()
245
else:
246
self.unique_id = NcclUniqueId()
244
- tensor = torch.ByteTensor(list(
245
- self.unique_id.internal)).cuda(local_rank)
247
+ tensor = torch.ByteTensor(list(self.unique_id.internal)).cuda(
248
+ self.local_rank)
249
dist.broadcast(tensor, src=0)
250
byte_list = tensor.cpu().tolist()
251
for i, byte in enumerate(byte_list):
252
self.unique_id.internal[i] = byte
253
self.comm = ctypes.c_void_p()
251
- result = _c_ncclCommInitRank(ctypes.byref(self.comm), world_size,
252
- self.unique_id, rank)
254
+ result = _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
255
+ self.unique_id, self.rank)
256
assert result == 0
254
- self.stream = torch.cuda.Stream(device=f"cuda:{local_rank}")
257
+ self.stream = torch.cuda.Stream(device=f"cuda:{self.local_rank}")
258
259
def all_reduce(self,
260
tensor: torch.Tensor,
@@ -271,4 +274,6 @@ class NCCLCommunicator:
274
# `dist` module might have been already destroyed
275
if hasattr(dist, 'destroy_process_group'):
276
dist.destroy_process_group()
274
- _c_ncclCommDestroy(self.comm)
277
+ # function might have been already destroyed
278
+ if _c_ncclCommDestroy is not None:
279
+ _c_ncclCommDestroy(self.comm)