@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 635b897246
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[distributed] remove pynccl's redundant stream (#11744)

cennn committed Jan 5, 2025 at 23:09 UTC 635b897246da121238454ed4b2bbc87cb4d4166b
3 files changed +12 -24
tests/distributed/test_pynccl.py
+2 -3
@@ -137,9 +137,8 @@ def worker_fn_with_cudagraph():
137 # run something in the default stream to initialize torch engine
138 a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
139 torch.cuda.synchronize()
140 - with torch.cuda.graph(
141 - graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
142 - enable=True):
140 + with torch.cuda.graph(graph), \
141 + pynccl_comm.change_state(enable=True):
142 a_out = pynccl_comm.all_reduce(a)
143 torch.cuda.synchronize()
144 graph.replay()
vllm/distributed/device_communicators/pynccl.py
+9 -19
@@ -51,7 +51,6 @@ class PyNcclCommunicator:
51 if self.world_size == 1:
52 self.available = False
53 self.disabled = True
54 - self.stream = None
54 return
55 try:
56 self.nccl = NCCLLibrary(library_path)
@@ -60,7 +59,6 @@ class PyNcclCommunicator:
59 # e.g. in a non-GPU environment
60 self.available = False
61 self.disabled = True
63 - self.stream = None
62 return
63
64 self.available = True
@@ -98,12 +96,12 @@ class PyNcclCommunicator:
96 with torch.cuda.device(device):
97 self.comm: ncclComm_t = self.nccl.ncclCommInitRank(
98 self.world_size, self.unique_id, self.rank)
101 - self.stream = torch.cuda.Stream()
99
100 + stream = torch.cuda.current_stream()
101 # A small all_reduce for warmup.
102 data = torch.zeros(1, device=device)
103 self.all_reduce(data)
106 - self.stream.synchronize()
104 + stream.synchronize()
105 del data
106
107 def all_reduce(self,
@@ -122,7 +120,7 @@ class PyNcclCommunicator:
120 out_tensor = torch.empty_like(in_tensor)
121
122 if stream is None:
125 - stream = self.stream
123 + stream = torch.cuda.current_stream()
124 self.nccl.ncclAllReduce(buffer_type(in_tensor.data_ptr()),
125 buffer_type(out_tensor.data_ptr()),
126 in_tensor.numel(),
@@ -144,7 +142,7 @@ class PyNcclCommunicator:
142 f"this nccl communicator is created to work on {self.device}, "
143 f"but the input tensor is on {input_tensor.device}")
144 if stream is None:
147 - stream = self.stream
145 + stream = torch.cuda.current_stream()
146 self.nccl.ncclAllGather(
147 buffer_type(input_tensor.data_ptr()),
148 buffer_type(output_tensor.data_ptr()), input_tensor.numel(),
@@ -165,7 +163,7 @@ class PyNcclCommunicator:
163 f"this nccl communicator is created to work on {self.device}, "
164 f"but the input tensor is on {input_tensor.device}")
165 if stream is None:
168 - stream = self.stream
166 + stream = torch.cuda.current_stream()
167 self.nccl.ncclReduceScatter(
168 buffer_type(input_tensor.data_ptr()),
169 buffer_type(output_tensor.data_ptr()), output_tensor.numel(),
@@ -180,7 +178,7 @@ class PyNcclCommunicator:
178 f"this nccl communicator is created to work on {self.device}, "
179 f"but the input tensor is on {tensor.device}")
180 if stream is None:
183 - stream = self.stream
181 + stream = torch.cuda.current_stream()
182 self.nccl.ncclSend(buffer_type(tensor.data_ptr()), tensor.numel(),
183 ncclDataTypeEnum.from_torch(tensor.dtype), dst,
184 self.comm, cudaStream_t(stream.cuda_stream))
@@ -192,7 +190,7 @@ class PyNcclCommunicator:
190 f"this nccl communicator is created to work on {self.device}, "
191 f"but the input tensor is on {tensor.device}")
192 if stream is None:
195 - stream = self.stream
193 + stream = torch.cuda.current_stream()
194 self.nccl.ncclRecv(buffer_type(tensor.data_ptr()), tensor.numel(),
195 ncclDataTypeEnum.from_torch(tensor.dtype), src,
196 self.comm, cudaStream_t(stream.cuda_stream))
@@ -204,7 +202,7 @@ class PyNcclCommunicator:
202 f"this nccl communicator is created to work on {self.device}, "
203 f"but the input tensor is on {tensor.device}")
204 if stream is None:
207 - stream = self.stream
205 + stream = torch.cuda.current_stream()
206 if src == self.rank:
207 sendbuff = buffer_type(tensor.data_ptr())
208 # NCCL requires the sender also to have a receive buffer
@@ -217,9 +215,7 @@ class PyNcclCommunicator:
215 self.comm, cudaStream_t(stream.cuda_stream))
216
217 @contextmanager
220 - def change_state(self,
221 - enable: Optional[bool] = None,
222 - stream: Optional[torch.cuda.Stream] = None):
218 + def change_state(self, enable: Optional[bool] = None):
219 """
220 A context manager to change the state of the communicator.
221 """
@@ -227,15 +223,9 @@ class PyNcclCommunicator:
223 # guess a default value when not specified
224 enable = self.available
225
230 - if stream is None:
231 - stream = self.stream
232 -
226 old_disable = self.disabled
234 - old_stream = self.stream
227
236 - self.stream = stream
228 self.disabled = not enable
229 yield
230
231 self.disabled = old_disable
241 - self.stream = old_stream
vllm/distributed/parallel_state.py
+1 -2
@@ -310,8 +310,7 @@ class GroupCoordinator:
310 if not pynccl_comm:
311 maybe_pynccl_context = nullcontext()
312 else:
313 - maybe_pynccl_context = pynccl_comm.change_state(
314 - stream=torch.cuda.current_stream())
313 + maybe_pynccl_context = pynccl_comm.change_state()
314 with maybe_pynccl_context:
315 yield graph_capture_context
316