A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[distributed] remove pynccl's redundant change_state (#11749)
cennn committed
Jan 6, 2025 at 09:05 UTC
9e764e7b105a483ebc702cad33922ba8d8c210e1
3 files changed
+28
-62
tests/distributed/test_pynccl.py
+27
-37
@@ -59,8 +59,7 @@ def worker_fn():
59
device=get_world_group().device)
60
tensor = torch.ones(16, 1024, 1024,
61
dtype=torch.float32).cuda(pynccl_comm.rank)
62
- with pynccl_comm.change_state(enable=True):
63
- tensor = pynccl_comm.all_reduce(tensor)
62
+ tensor = pynccl_comm.all_reduce(tensor)
63
torch.cuda.synchronize()
64
assert torch.all(tensor == pynccl_comm.world_size).cpu().item()
65
@@ -81,17 +80,16 @@ def multiple_allreduce_worker_fn():
80
group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
81
pynccl_comm = PyNcclCommunicator(group=group, device=device)
82
tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
84
- with pynccl_comm.change_state(enable=True):
85
- # two groups can communicate independently
86
- if torch.distributed.get_rank() in [0, 1]:
87
- tensor = pynccl_comm.all_reduce(tensor)
88
- tensor = pynccl_comm.all_reduce(tensor)
89
- torch.cuda.synchronize()
90
- assert torch.all(tensor == 4).cpu().item()
91
- else:
92
- tensor = pynccl_comm.all_reduce(tensor)
93
- torch.cuda.synchronize()
94
- assert torch.all(tensor == 2).cpu().item()
83
+ # two groups can communicate independently
84
+ if torch.distributed.get_rank() in [0, 1]:
85
+ tensor = pynccl_comm.all_reduce(tensor)
86
+ tensor = pynccl_comm.all_reduce(tensor)
87
+ torch.cuda.synchronize()
88
+ assert torch.all(tensor == 4).cpu().item()
89
+ else:
90
+ tensor = pynccl_comm.all_reduce(tensor)
91
+ torch.cuda.synchronize()
92
+ assert torch.all(tensor == 2).cpu().item()
93
94
95
@pytest.mark.skipif(torch.cuda.device_count() < 4,
@@ -137,8 +135,7 @@ def worker_fn_with_cudagraph():
135
# run something in the default stream to initialize torch engine
136
a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
137
torch.cuda.synchronize()
140
- with torch.cuda.graph(graph), \
141
- pynccl_comm.change_state(enable=True):
138
+ with torch.cuda.graph(graph):
139
a_out = pynccl_comm.all_reduce(a)
140
torch.cuda.synchronize()
141
graph.replay()
@@ -167,8 +164,7 @@ def all_gather_worker_fn():
164
for r in range(world_size)
165
]).to(device)
166
170
- with pynccl_comm.change_state(enable=True):
171
- pynccl_comm.all_gather(result, tensor)
167
+ pynccl_comm.all_gather(result, tensor)
168
torch.cuda.synchronize()
169
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
170
@@ -205,8 +201,7 @@ def reduce_scatter_worker_fn():
201
expected = sum(tensor[rank * scattered_size:(rank + 1) * scattered_size]
202
for tensor in all_tensors).to(device)
203
208
- with pynccl_comm.change_state(enable=True):
209
- pynccl_comm.reduce_scatter(result, tensor)
204
+ pynccl_comm.reduce_scatter(result, tensor)
205
torch.cuda.synchronize()
206
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
207
@@ -233,15 +228,13 @@ def send_recv_worker_fn():
228
else:
229
tensor = torch.empty(16, 1024, 1024,
230
dtype=torch.float32).cuda(pynccl_comm.rank)
236
- with pynccl_comm.change_state(enable=True):
237
- if pynccl_comm.rank == 0:
238
- pynccl_comm.send(tensor,
239
- dst=(pynccl_comm.rank + 1) %
240
- pynccl_comm.world_size)
241
- else:
242
- pynccl_comm.recv(tensor,
243
- src=(pynccl_comm.rank - 1) %
244
- pynccl_comm.world_size)
231
+
232
+ if pynccl_comm.rank == 0:
233
+ pynccl_comm.send(tensor,
234
+ dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
235
+ else:
236
+ pynccl_comm.recv(tensor,
237
+ src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
238
torch.cuda.synchronize()
239
assert torch.all(tensor == 1).cpu().item()
240
@@ -272,15 +265,12 @@ def multiple_send_recv_worker_fn():
265
1024,
266
dtype=torch.float32,
267
device=device)
275
- with pynccl_comm.change_state(enable=True):
276
- if torch.distributed.get_rank() in [0, 1]:
277
- pynccl_comm.send(tensor,
278
- dst=(pynccl_comm.rank + 1) %
279
- pynccl_comm.world_size)
280
- else:
281
- pynccl_comm.recv(tensor,
282
- src=(pynccl_comm.rank - 1) %
283
- pynccl_comm.world_size)
268
+ if torch.distributed.get_rank() in [0, 1]:
269
+ pynccl_comm.send(tensor,
270
+ dst=(pynccl_comm.rank + 1) % pynccl_comm.world_size)
271
+ else:
272
+ pynccl_comm.recv(tensor,
273
+ src=(pynccl_comm.rank - 1) % pynccl_comm.world_size)
274
torch.cuda.synchronize()
275
if torch.distributed.get_rank() in [0, 2]:
276
assert torch.all(tensor == 1).cpu().item()
vllm/distributed/device_communicators/pynccl.py
-17
@@ -1,4 +1,3 @@
1
-from contextlib import contextmanager
1
from typing import Optional, Union
2
3
# ===================== import region =====================
@@ -213,19 +212,3 @@ class PyNcclCommunicator:
212
self.nccl.ncclBroadcast(sendbuff, recvbuff, tensor.numel(),
213
ncclDataTypeEnum.from_torch(tensor.dtype), src,
214
self.comm, cudaStream_t(stream.cuda_stream))
216
-
217
- @contextmanager
218
- def change_state(self, enable: Optional[bool] = None):
219
- """
220
- A context manager to change the state of the communicator.
221
- """
222
- if enable is None:
223
- # guess a default value when not specified
224
- enable = self.available
225
-
226
- old_disable = self.disabled
227
-
228
- self.disabled = not enable
229
- yield
230
-
231
- self.disabled = old_disable
vllm/distributed/parallel_state.py
+1
-8
@@ -305,14 +305,7 @@ class GroupCoordinator:
305
stream.wait_stream(curr_stream)
306
307
with torch.cuda.stream(stream), maybe_ca_context:
308
- pynccl_comm = self.pynccl_comm
309
- maybe_pynccl_context: Any
310
- if not pynccl_comm:
311
- maybe_pynccl_context = nullcontext()
312
- else:
313
- maybe_pynccl_context = pynccl_comm.change_state()
314
- with maybe_pynccl_context:
315
- yield graph_capture_context
308
+ yield graph_capture_context
309
310
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
311
"""