A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[core][distributed] add pynccl broadcast (#10843)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 2, 2024 at 20:53 UTC
21fe7b481a3a84dc9ebe2497ec89a17002ad52c5
3 files changed
+78
-2
tests/distributed/test_pynccl.py
+43
-2
@@ -61,6 +61,7 @@ def worker_fn():
61
dtype=torch.float32).cuda(pynccl_comm.rank)
62
with pynccl_comm.change_state(enable=True):
63
tensor = pynccl_comm.all_reduce(tensor)
64
+ torch.cuda.synchronize()
65
result = tensor.mean().cpu().item()
66
assert result == pynccl_comm.world_size
67
@@ -86,10 +87,12 @@ def multiple_allreduce_worker_fn():
87
if torch.distributed.get_rank() in [0, 1]:
88
tensor = pynccl_comm.all_reduce(tensor)
89
tensor = pynccl_comm.all_reduce(tensor)
90
+ torch.cuda.synchronize()
91
result = tensor.mean().cpu().item()
92
assert result == 4
93
else:
94
tensor = pynccl_comm.all_reduce(tensor)
95
+ torch.cuda.synchronize()
96
result = tensor.mean().cpu().item()
97
assert result == 2
98
@@ -112,10 +115,12 @@ def multiple_allreduce_with_vllm_worker_fn():
115
if torch.distributed.get_rank() in [0, 1]:
116
tensor = tensor_model_parallel_all_reduce(tensor)
117
tensor = tensor_model_parallel_all_reduce(tensor)
118
+ torch.cuda.synchronize()
119
result = tensor.mean().cpu().item()
120
assert result == 4
121
else:
122
tensor = tensor_model_parallel_all_reduce(tensor)
123
+ torch.cuda.synchronize()
124
result = tensor.mean().cpu().item()
125
assert result == 2
126
@@ -141,9 +146,9 @@ def worker_fn_with_cudagraph():
146
graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
147
enable=True):
148
a_out = pynccl_comm.all_reduce(a)
144
- pynccl_comm.stream.synchronize()
149
+ torch.cuda.synchronize()
150
graph.replay()
146
- pynccl_comm.stream.synchronize()
151
+ torch.cuda.synchronize()
152
assert a_out.mean().cpu().item() == pynccl_comm.world_size**1
153
154
@@ -170,6 +175,7 @@ def all_gather_worker_fn():
175
176
with pynccl_comm.change_state(enable=True):
177
pynccl_comm.all_gather(result, tensor)
178
+ torch.cuda.synchronize()
179
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
180
181
@@ -207,6 +213,7 @@ def reduce_scatter_worker_fn():
213
214
with pynccl_comm.change_state(enable=True):
215
pynccl_comm.reduce_scatter(result, tensor)
216
+ torch.cuda.synchronize()
217
torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
218
219
@@ -241,6 +248,7 @@ def send_recv_worker_fn():
248
pynccl_comm.recv(tensor,
249
src=(pynccl_comm.rank - 1) %
250
pynccl_comm.world_size)
251
+ torch.cuda.synchronize()
252
result = tensor.mean().cpu().item()
253
assert result == 1
254
@@ -280,6 +288,7 @@ def multiple_send_recv_worker_fn():
288
pynccl_comm.recv(tensor,
289
src=(pynccl_comm.rank - 1) %
290
pynccl_comm.world_size)
291
+ torch.cuda.synchronize()
292
result = tensor.mean().cpu().item()
293
if torch.distributed.get_rank() in [0, 2]:
294
assert result == 1
@@ -293,6 +302,38 @@ def test_pynccl_multiple_send_recv():
302
distributed_run(multiple_send_recv_worker_fn, 4)
303
304
305
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
306
+ reason="Need at least 4 GPUs to run the test.")
307
+def test_pynccl_broadcast():
308
+ distributed_run(broadcast_worker_fn, 4)
309
+
310
+
311
+@worker_fn_wrapper
312
+def broadcast_worker_fn():
313
+ # Test broadcast for every root rank.
314
+ # Essentially this is an all-gather operation.
315
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
316
+ device=get_world_group().device)
317
+ recv_tensors = [
318
+ torch.empty(16,
319
+ 1024,
320
+ 1024,
321
+ dtype=torch.float32,
322
+ device=pynccl_comm.device)
323
+ for i in range(pynccl_comm.world_size)
324
+ ]
325
+ recv_tensors[pynccl_comm.rank] = torch.ones(
326
+ 16, 1024, 1024, dtype=torch.float32,
327
+ device=pynccl_comm.device) * pynccl_comm.rank
328
+
329
+ for i in range(pynccl_comm.world_size):
330
+ pynccl_comm.broadcast(recv_tensors[i], src=i)
331
+ # the broadcast op might be launched in a different stream
332
+ # need to synchronize to make sure the tensor is ready
333
+ torch.cuda.synchronize()
334
+ assert torch.all(recv_tensors[i] == i).cpu().item()
335
+
336
+
337
def test_ncclGetUniqueId():
338
lib = NCCLLibrary()
339
unique_id = lib.ncclGetUniqueId()
vllm/distributed/device_communicators/pynccl.py
+19
@@ -197,6 +197,25 @@ class PyNcclCommunicator:
197
ncclDataTypeEnum.from_torch(tensor.dtype), src,
198
self.comm, cudaStream_t(stream.cuda_stream))
199
200
+ def broadcast(self, tensor: torch.Tensor, src: int, stream=None):
201
+ if self.disabled:
202
+ return
203
+ assert tensor.device == self.device, (
204
+ f"this nccl communicator is created to work on {self.device}, "
205
+ f"but the input tensor is on {tensor.device}")
206
+ if stream is None:
207
+ stream = self.stream
208
+ if src == self.rank:
209
+ sendbuff = buffer_type(tensor.data_ptr())
210
+ # NCCL requires the sender also to have a receive buffer
211
+ recvbuff = buffer_type(tensor.data_ptr())
212
+ else:
213
+ sendbuff = buffer_type()
214
+ recvbuff = buffer_type(tensor.data_ptr())
215
+ self.nccl.ncclBroadcast(sendbuff, recvbuff, tensor.numel(),
216
+ ncclDataTypeEnum.from_torch(tensor.dtype), src,
217
+ self.comm, cudaStream_t(stream.cuda_stream))
218
+
219
@contextmanager
220
def change_state(self,
221
enable: Optional[bool] = None,
vllm/distributed/device_communicators/pynccl_wrapper.py
+16
@@ -189,6 +189,15 @@ class NCCLLibrary:
189
ncclComm_t, cudaStream_t
190
]),
191
192
+ # ncclResult_t ncclBroadcast(
193
+ # const void* sendbuff, void* recvbuff, size_t count,
194
+ # ncclDataType_t datatype, int root, ncclComm_t comm,
195
+ # cudaStream_t stream);
196
+ Function("ncclBroadcast", ncclResult_t, [
197
+ buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
198
+ ctypes.c_int, ncclComm_t, cudaStream_t
199
+ ]),
200
+
201
# be cautious! this is a collective call, it will block until all
202
# processes in the communicator have called this function.
203
# because Python object destruction can happen in random order,
@@ -312,6 +321,13 @@ class NCCLLibrary:
321
self.NCCL_CHECK(self._funcs["ncclRecv"](recvbuff, count, datatype, src,
322
comm, stream))
323
324
+ def ncclBroadcast(self, sendbuff: buffer_type, recvbuff: buffer_type,
325
+ count: int, datatype: int, root: int, comm: ncclComm_t,
326
+ stream: cudaStream_t) -> None:
327
+ self.NCCL_CHECK(self._funcs["ncclBroadcast"](sendbuff, recvbuff, count,
328
+ datatype, root, comm,
329
+ stream))
330
+
331
def ncclCommDestroy(self, comm: ncclComm_t) -> None:
332
self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
333