@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3cd9b5bb2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] use existing torch.cuda.device (#4318)

[Core][Distributed] use existing torch.cuda.device context manager (#4318)

youkaichao committed Apr 24, 2024 at 09:00 UTC 3cd9b5bb2d4a0d5eed07186ae140f5dc8f839708
1 file changed +3 -5
vllm/distributed/device_communicators/pynccl.py
+3 -5
@@ -250,15 +250,13 @@ class NCCLCommunicator:
250 assert isinstance(device, torch.device)
251 self.device = device
252 # nccl communicator and stream will use this device
253 - current_device = torch.cuda.current_device()
254 - try:
255 - torch.cuda.set_device(device)
253 + # `torch.cuda.device` is a context manager that changes the
254 + # current cuda device to the specified one
255 + with torch.cuda.device(device):
256 NCCL_CHECK(
257 _c_ncclCommInitRank(ctypes.byref(self.comm), self.world_size,
258 self.unique_id, self.rank))
259 self.stream = torch.cuda.Stream()
260 - finally:
261 - torch.cuda.set_device(current_device)
260
261 def all_reduce(self,
262 tensor: torch.Tensor,