@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e764e7b10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[distributed] remove pynccl's redundant change_state (#11749)

cennn committed Jan 6, 2025 at 09:05 UTC 9e764e7b105a483ebc702cad33922ba8d8c210e1
3 files changed +28 -62
tests/distributed/test_pynccl.py
+27 -37
@@ -59,8 +59,7 @@ def worker_fn():
59 device=get_world_group().device)
60 tensor = torch.ones(16, 1024, 1024,
61 dtype=torch.float32).cuda(pynccl_comm.rank)
62 - with pynccl_comm.change_state(enable=True):
63 - tensor = pynccl_comm.all_reduce(tensor)
62 + tensor = pynccl_comm.all_reduce(tensor)
63 torch.cuda.synchronize()
64 assert torch.all(tensor == pynccl_comm.world_size).cpu().item()
65
@@ -81,17 +80,16 @@ def multiple_allreduce_worker_fn():
80 group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
81 pynccl_comm = PyNcclCommunicator(group=group, device=device)
82 tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
84 - with pynccl_comm.change_state(enable=True):
85 - # two groups can communicate independently
86 - if torch.distributed.get_rank() in [0, 1]:
87 - tensor = pynccl_comm.all_reduce(tensor)
88 - tensor = pynccl_comm.all_reduce(tensor)
89 - torch.cuda.synchronize()
90 - assert torch.all(tensor == 4).cpu().item()
91 - else:
92 - tensor = pynccl_comm.all_reduce(tensor)
93 - torch.cuda.synchronize()
94 - assert torch.all(tensor == 2).cpu().item()
83 + # two groups can communicate independently
84 + if torch.distributed.get_rank() in [0, 1]:
85 + tensor = pynccl_comm.all_reduce(tensor)
86 + tensor = pynccl_comm.all_reduce(tensor)
87 + torch.cuda.synchronize()
88 + assert torch.all(tensor == 4).cpu().item()
89 + else:
90 + tensor = pynccl_comm.all_reduce(tensor)
91 + torch.cuda.synchronize()
92 + assert torch.all(tensor == 2).cpu().item()
93
94
95 @pytest.mark.skipif(torch.cuda.device_count() < 4,
@@ -137,8 +135,7 @@ def worker_fn_with_cudagraph():
135 # run something in the default stream to initialize torch engine
136 a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
137 torch.cuda.synchronize()
140 - with torch.cuda.graph(graph), \
141 - pynccl_comm.change_state(enable=True):
138 + with torch.cuda.graph(graph):
139 a_out = pynccl_comm.all_reduce(a)
140 torch.cuda.synchronize()
141 graph.replay()
@@ -167,8 +164,7 @@ def all_gather_worker_fn():
164 for r in range(world_size)
165 ]).to(device)
166
170 - with pynccl_comm.change_state(enable=True):
171 - pynccl_comm.all_gather(result, tensor)
167 + pynccl_comm.all_gather(result, tensor)
168 torch.cuda.synchronize()
169 torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
170
@@ -205,8 +201,7 @@ def reduce_scatter_worker_fn():
201 expected = sum(tensor[rank * scattered_size:(rank + 1) * scattered_size]
202 for tensor in all_tensors).to(device)
203
208 - with pynccl_comm.change_state(enable=True):
209 - pynccl_comm.reduce_scatter(result, tensor)
204 + pynccl_comm.reduce_scatter(result, tensor)
205 torch.cuda.synchronize()
206 torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
207
@@ -233,15 +228,13 @@ def send_recv_worker_fn():
228 else:
229 tensor = torch.empty(16, 1024, 1024,
230 dtype=torch.float32).cuda(pynccl_comm.rank)
236 - with pynccl_comm.change_state(enable=True):
237 - if pynccl_comm.rank == 0:
238 - pynccl_comm.send(tensor,
239 - dst=(pynccl_comm.rank + 1) %
240 - pynccl_comm.world_size)
241 - else:
242 - pynccl_comm.recv(tensor,
243 - src=(pynccl_comm.rank - 1) %
244 - pynccl_comm.world_size)
231 +
232 + if pynccl_comm.rank == 0:
233 + pynccl_comm.send(tensor,
234 + dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
235 + else:
236 + pynccl_comm.recv(tensor,
237 + src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
238 torch.cuda.synchronize()
239 assert torch.all(tensor == 1).cpu().item()
240
@@ -272,15 +265,12 @@ def multiple_send_recv_worker_fn():
265 1024,
266 dtype=torch.float32,
267 device=device)
275 - with pynccl_comm.change_state(enable=True):
276 - if torch.distributed.get_rank() in [0, 1]:
277 - pynccl_comm.send(tensor,
278 - dst=(pynccl_comm.rank + 1) %
279 - pynccl_comm.world_size)
280 - else:
281 - pynccl_comm.recv(tensor,
282 - src=(pynccl_comm.rank - 1) %
283 - pynccl_comm.world_size)
268 + if torch.distributed.get_rank() in [0, 1]:
269 + pynccl_comm.send(tensor,
270 + dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
271 + else:
272 + pynccl_comm.recv(tensor,
273 + src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
274 torch.cuda.synchronize()
275 if torch.distributed.get_rank() in [0, 2]:
276 assert torch.all(tensor == 1).cpu().item()
vllm/distributed/device_communicators/pynccl.py
-17
@@ -1,4 +1,3 @@
1 -from contextlib import contextmanager
1 from typing import Optional, Union
2
3 # ===================== import region =====================
@@ -213,19 +212,3 @@ class PyNcclCommunicator:
212 self.nccl.ncclBroadcast(sendbuff, recvbuff, tensor.numel(),
213 ncclDataTypeEnum.from_torch(tensor.dtype), src,
214 self.comm, cudaStream_t(stream.cuda_stream))
216 -
217 - @contextmanager
218 - def change_state(self, enable: Optional[bool] = None):
219 - """
220 - A context manager to change the state of the communicator.
221 - """
222 - if enable is None:
223 - # guess a default value when not specified
224 - enable = self.available
225 -
226 - old_disable = self.disabled
227 -
228 - self.disabled = not enable
229 - yield
230 -
231 - self.disabled = old_disable
vllm/distributed/parallel_state.py
+1 -8
@@ -305,14 +305,7 @@ class GroupCoordinator:
305 stream.wait_stream(curr_stream)
306
307 with torch.cuda.stream(stream), maybe_ca_context:
308 - pynccl_comm = self.pynccl_comm
309 - maybe_pynccl_context: Any
310 - if not pynccl_comm:
311 - maybe_pynccl_context = nullcontext()
312 - else:
313 - maybe_pynccl_context = pynccl_comm.change_state()
314 - with maybe_pynccl_context:
315 - yield graph_capture_context
308 + yield graph_capture_context
309
310 def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
311 """