@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21fe7b481a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[core][distributed] add pynccl broadcast (#10843)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 2, 2024 at 20:53 UTC 21fe7b481a3a84dc9ebe2497ec89a17002ad52c5
3 files changed +78 -2
tests/distributed/test_pynccl.py
+43 -2
@@ -61,6 +61,7 @@ def worker_fn():
61 dtype=torch.float32).cuda(pynccl_comm.rank)
62 with pynccl_comm.change_state(enable=True):
63 tensor = pynccl_comm.all_reduce(tensor)
64 + torch.cuda.synchronize()
65 result = tensor.mean().cpu().item()
66 assert result == pynccl_comm.world_size
67
@@ -86,10 +87,12 @@ def multiple_allreduce_worker_fn():
87 if torch.distributed.get_rank() in [0, 1]:
88 tensor = pynccl_comm.all_reduce(tensor)
89 tensor = pynccl_comm.all_reduce(tensor)
90 + torch.cuda.synchronize()
91 result = tensor.mean().cpu().item()
92 assert result == 4
93 else:
94 tensor = pynccl_comm.all_reduce(tensor)
95 + torch.cuda.synchronize()
96 result = tensor.mean().cpu().item()
97 assert result == 2
98
@@ -112,10 +115,12 @@ def multiple_allreduce_with_vllm_worker_fn():
115 if torch.distributed.get_rank() in [0, 1]:
116 tensor = tensor_model_parallel_all_reduce(tensor)
117 tensor = tensor_model_parallel_all_reduce(tensor)
118 + torch.cuda.synchronize()
119 result = tensor.mean().cpu().item()
120 assert result == 4
121 else:
122 tensor = tensor_model_parallel_all_reduce(tensor)
123 + torch.cuda.synchronize()
124 result = tensor.mean().cpu().item()
125 assert result == 2
126
@@ -141,9 +146,9 @@ def worker_fn_with_cudagraph():
146 graph, stream=pynccl_comm.stream), pynccl_comm.change_state(
147 enable=True):
148 a_out = pynccl_comm.all_reduce(a)
144 - pynccl_comm.stream.synchronize()
149 + torch.cuda.synchronize()
150 graph.replay()
146 - pynccl_comm.stream.synchronize()
151 + torch.cuda.synchronize()
152 assert a_out.mean().cpu().item() == pynccl_comm.world_size**1
153
154
@@ -170,6 +175,7 @@ def all_gather_worker_fn():
175
176 with pynccl_comm.change_state(enable=True):
177 pynccl_comm.all_gather(result, tensor)
178 + torch.cuda.synchronize()
179 torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
180
181
@@ -207,6 +213,7 @@ def reduce_scatter_worker_fn():
213
214 with pynccl_comm.change_state(enable=True):
215 pynccl_comm.reduce_scatter(result, tensor)
216 + torch.cuda.synchronize()
217 torch.testing.assert_close(result, expected, rtol=1e-5, atol=1e-8)
218
219
@@ -241,6 +248,7 @@ def send_recv_worker_fn():
248 pynccl_comm.recv(tensor,
249 src=(pynccl_comm.rank - 1) %
250 pynccl_comm.world_size)
251 + torch.cuda.synchronize()
252 result = tensor.mean().cpu().item()
253 assert result == 1
254
@@ -280,6 +288,7 @@ def multiple_send_recv_worker_fn():
288 pynccl_comm.recv(tensor,
289 src=(pynccl_comm.rank - 1) %
290 pynccl_comm.world_size)
291 + torch.cuda.synchronize()
292 result = tensor.mean().cpu().item()
293 if torch.distributed.get_rank() in [0, 2]:
294 assert result == 1
@@ -293,6 +302,38 @@ def test_pynccl_multiple_send_recv():
302 distributed_run(multiple_send_recv_worker_fn, 4)
303
304
305 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
306 + reason="Need at least 4 GPUs to run the test.")
307 +def test_pynccl_broadcast():
308 + distributed_run(broadcast_worker_fn, 4)
309 +
310 +
311 +@worker_fn_wrapper
312 +def broadcast_worker_fn():
313 + # Test broadcast for every root rank.
314 + # Essentially this is an all-gather operation.
315 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
316 + device=get_world_group().device)
317 + recv_tensors = [
318 + torch.empty(16,
319 + 1024,
320 + 1024,
321 + dtype=torch.float32,
322 + device=pynccl_comm.device)
323 + for i in range(pynccl_comm.world_size)
324 + ]
325 + recv_tensors[pynccl_comm.rank] = torch.ones(
326 + 16, 1024, 1024, dtype=torch.float32,
327 + device=pynccl_comm.device) * pynccl_comm.rank
328 +
329 + for i in range(pynccl_comm.world_size):
330 + pynccl_comm.broadcast(recv_tensors[i], src=i)
331 + # the broadcast op might be launched in a different stream
332 + # need to synchronize to make sure the tensor is ready
333 + torch.cuda.synchronize()
334 + assert torch.all(recv_tensors[i] == i).cpu().item()
335 +
336 +
337 def test_ncclGetUniqueId():
338 lib = NCCLLibrary()
339 unique_id = lib.ncclGetUniqueId()
vllm/distributed/device_communicators/pynccl.py
+19
@@ -197,6 +197,25 @@ class PyNcclCommunicator:
197 ncclDataTypeEnum.from_torch(tensor.dtype), src,
198 self.comm, cudaStream_t(stream.cuda_stream))
199
200 + def broadcast(self, tensor: torch.Tensor, src: int, stream=None):
201 + if self.disabled:
202 + return
203 + assert tensor.device == self.device, (
204 + f"this nccl communicator is created to work on {self.device}, "
205 + f"but the input tensor is on {tensor.device}")
206 + if stream is None:
207 + stream = self.stream
208 + if src == self.rank:
209 + sendbuff = buffer_type(tensor.data_ptr())
210 + # NCCL requires the sender also to have a receive buffer
211 + recvbuff = buffer_type(tensor.data_ptr())
212 + else:
213 + sendbuff = buffer_type()
214 + recvbuff = buffer_type(tensor.data_ptr())
215 + self.nccl.ncclBroadcast(sendbuff, recvbuff, tensor.numel(),
216 + ncclDataTypeEnum.from_torch(tensor.dtype), src,
217 + self.comm, cudaStream_t(stream.cuda_stream))
218 +
219 @contextmanager
220 def change_state(self,
221 enable: Optional[bool] = None,
vllm/distributed/device_communicators/pynccl_wrapper.py
+16
@@ -189,6 +189,15 @@ class NCCLLibrary:
189 ncclComm_t, cudaStream_t
190 ]),
191
192 + # ncclResult_t ncclBroadcast(
193 + # const void* sendbuff, void* recvbuff, size_t count,
194 + # ncclDataType_t datatype, int root, ncclComm_t comm,
195 + # cudaStream_t stream);
196 + Function("ncclBroadcast", ncclResult_t, [
197 + buffer_type, buffer_type, ctypes.c_size_t, ncclDataType_t,
198 + ctypes.c_int, ncclComm_t, cudaStream_t
199 + ]),
200 +
201 # be cautious! this is a collective call, it will block until all
202 # processes in the communicator have called this function.
203 # because Python object destruction can happen in random order,
@@ -312,6 +321,13 @@ class NCCLLibrary:
321 self.NCCL_CHECK(self._funcs["ncclRecv"](recvbuff, count, datatype, src,
322 comm, stream))
323
324 + def ncclBroadcast(self, sendbuff: buffer_type, recvbuff: buffer_type,
325 + count: int, datatype: int, root: int, comm: ncclComm_t,
326 + stream: cudaStream_t) -> None:
327 + self.NCCL_CHECK(self._funcs["ncclBroadcast"](sendbuff, recvbuff, count,
328 + datatype, root, comm,
329 + stream))
330 +
331 def ncclCommDestroy(self, comm: ncclComm_t) -> None:
332 self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm))
333