A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] use existing torch.cuda.device (#4318)
[Core][Distributed] use existing torch.cuda.device context manager (#4318)
youkaichao committed
Apr 24, 2024 at 09:00 UTC
3cd9b5bb2d4a0d5eed07186ae140f5dc8f839708
1 file changed
+3
-5
vllm/distributed/device_communicators/pynccl.py
+3
-5
@@ -250,15 +250,13 @@ class NCCLCommunicator:
250
assert isinstance(device, torch.device)
251
self.device = device
252
# nccl communicator and stream will use this device
253
- current_device = torch.cuda.current_device()
254
- try:
255
- torch.cuda.set_device(device)
253
+ # `torch.cuda.device` is a context manager that changes the
254
+ # current cuda device to the specified one
255
+ with torch.cuda.device(device):
256
NCCL_CHECK(
257
_c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
258
self.unique_id, self.rank))
259
self.stream = torch.cuda.Stream()
260
- finally:
261
- torch.cuda.set_device(current_device)
260
261
def all_reduce(self,
262
tensor: torch.Tensor,