@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 663874e048
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[torch.compile] improve allreduce registration (#9061)

youkaichao committed Oct 4, 2024 at 16:43 UTC 663874e048d88aa7bf087628430d50f9f5245175
2 files changed +21 -32
vllm/distributed/device_communicators/custom_all_reduce.py
+6 -9
@@ -265,24 +265,21 @@ class CustomAllreduce:
265
266 def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
267 # when custom allreduce is disabled, this will be None
268 - if self.disabled:
268 + if self.disabled or not self.should_custom_ar(input):
269 return None
270 if self._IS_CAPTURING:
271 if torch.cuda.is_current_stream_capturing():
272 - if self.should_custom_ar(input):
273 - return self.all_reduce_reg(input)
272 + return self.all_reduce_reg(input)
273 else:
275 - if self.should_custom_ar(input):
276 - # if warm up, mimic the allocation pattern
277 - # since custom allreduce is out-of-place
278 - return torch.empty_like(input)
274 + # if warm up, mimic the allocation pattern
275 + # since custom allreduce is out-of-place
276 + return torch.empty_like(input)
277 else:
278 # note: outside of cuda graph context,
279 # custom allreduce incurs a cost of cudaMemcpy, which should
280 # be small(<=1% of overall latency) compared to the performance
281 # gains of using custom kernels
284 - if self.should_custom_ar(input):
285 - return self.all_reduce_unreg(input)
282 + return self.all_reduce_unreg(input)
283
284 return None
285
vllm/distributed/parallel_state.py
+15 -23
@@ -105,7 +105,7 @@ if supports_custom_op():
105 group = _groups[group_name]()
106 if group is None:
107 raise ValueError(f"Group {group_name} is destroyed.")
108 - group._all_reduce(tensor)
108 + group._all_reduce_in_place(tensor)
109
110 @inplace_all_reduce.register_fake
111 def _(tensor: torch.Tensor, group_name: str) -> None:
@@ -118,7 +118,7 @@ if supports_custom_op():
118 group = _groups[group_name]()
119 if group is None:
120 raise ValueError(f"Group {group_name} is destroyed.")
121 - return group._all_reduce(tensor)
121 + return group._all_reduce_out_place(tensor)
122
123 @outplace_all_reduce.register_fake
124 def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
@@ -338,14 +338,17 @@ class GroupCoordinator:
338 return input_
339
340 if not supports_custom_op():
341 - return self._all_reduce(input_)
341 + self._all_reduce_in_place(input_)
342 + return input_
343
344 if self.tpu_communicator is not None and \
345 not self.tpu_communicator.disabled:
346 # TPU handles Dynamo with its own logic.
346 - return self._all_reduce(input_)
347 + return self.tpu_communicator.all_reduce(input_)
348
348 - if self.ca_comm is not None and self.ca_comm.should_custom_ar(input_):
349 + if self.ca_comm is not None and \
350 + not self.ca_comm.disabled and \
351 + self.ca_comm.should_custom_ar(input_):
352 return torch.ops.vllm.outplace_all_reduce(
353 input_, group_name=self.unique_name)
354 else:
@@ -353,25 +356,15 @@ class GroupCoordinator:
356 group_name=self.unique_name)
357 return input_
358
356 - def _all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
357 - """
358 - The actual all-reduce implementation.
359 -
360 - NOTE: This operation will be applied in-place or out-of-place.
361 - Always assume this function modifies its input, but use the return
362 - value as the output.
363 - """
359 + def _all_reduce_out_place(self, input_: torch.Tensor) -> torch.Tensor:
360 ca_comm = self.ca_comm
361 + assert ca_comm is not None
362 + assert not ca_comm.disabled
363 + out = ca_comm.custom_all_reduce(input_)
364 + assert out is not None
365 + return out
366
366 - # For TPUs, use TPU communicator.
367 - tpu_comm = self.tpu_communicator
368 - if tpu_comm is not None and not tpu_comm.disabled:
369 - return tpu_comm.all_reduce(input_)
370 -
371 - if ca_comm is not None:
372 - out = ca_comm.custom_all_reduce(input_)
373 - if out is not None:
374 - return out
367 + def _all_reduce_in_place(self, input_: torch.Tensor) -> None:
368 pynccl_comm = self.pynccl_comm
369 if (pynccl_comm is not None and not pynccl_comm.disabled):
370 pynccl_comm.all_reduce(input_)
@@ -380,7 +373,6 @@ class GroupCoordinator:
373 ipex.distributed.all_reduce(input_, group=self.device_group)
374 else:
375 torch.distributed.all_reduce(input_, group=self.device_group)
383 - return input_
376
377 def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
378 world_size = self.world_size