A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[torch.compile] improve allreduce registration (#9061)
youkaichao committed
Oct 4, 2024 at 16:43 UTC
663874e048d88aa7bf087628430d50f9f5245175
2 files changed
+21
-32
vllm/distributed/device_communicators/custom_all_reduce.py
+6
-9
@@ -265,24 +265,21 @@ class CustomAllreduce:
265
266
def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
267
# when custom allreduce is disabled, this will be None
268
- if self.disabled:
268
+ if self.disabled or not self.should_custom_ar(input):
269
return None
270
if self._IS_CAPTURING:
271
if torch.cuda.is_current_stream_capturing():
272
- if self.should_custom_ar(input):
273
- return self.all_reduce_reg(input)
272
+ return self.all_reduce_reg(input)
273
else:
275
- if self.should_custom_ar(input):
276
- # if warm up, mimic the allocation pattern
277
- # since custom allreduce is out-of-place
278
- return torch.empty_like(input)
274
+ # if warm up, mimic the allocation pattern
275
+ # since custom allreduce is out-of-place
276
+ return torch.empty_like(input)
277
else:
278
# note: outside of cuda graph context,
279
# custom allreduce incurs a cost of cudaMemcpy, which should
280
# be small(<=1% of overall latency) compared to the performance
281
# gains of using custom kernels
284
- if self.should_custom_ar(input):
285
- return self.all_reduce_unreg(input)
282
+ return self.all_reduce_unreg(input)
283
284
return None
285
vllm/distributed/parallel_state.py
+15
-23
@@ -105,7 +105,7 @@ if supports_custom_op():
105
group = _groups[group_name]()
106
if group is None:
107
raise ValueError(f"Group {group_name} is destroyed.")
108
- group._all_reduce(tensor)
108
+ group._all_reduce_in_place(tensor)
109
110
@inplace_all_reduce.register_fake
111
def _(tensor: torch.Tensor, group_name: str) -> None:
@@ -118,7 +118,7 @@ if supports_custom_op():
118
group = _groups[group_name]()
119
if group is None:
120
raise ValueError(f"Group {group_name} is destroyed.")
121
- return group._all_reduce(tensor)
121
+ return group._all_reduce_out_place(tensor)
122
123
@outplace_all_reduce.register_fake
124
def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
@@ -338,14 +338,17 @@ class GroupCoordinator:
338
return input_
339
340
if not supports_custom_op():
341
- return self._all_reduce(input_)
341
+ self._all_reduce_in_place(input_)
342
+ return input_
343
344
if self.tpu_communicator is not None and \
345
not self.tpu_communicator.disabled:
346
# TPU handles Dynamo with its own logic.
346
- return self._all_reduce(input_)
347
+ return self.tpu_communicator.all_reduce(input_)
348
348
- if self.ca_comm is not None and self.ca_comm.should_custom_ar(input_):
349
+ if self.ca_comm is not None and \
350
+ not self.ca_comm.disabled and \
351
+ self.ca_comm.should_custom_ar(input_):
352
return torch.ops.vllm.outplace_all_reduce(
353
input_, group_name=self.unique_name)
354
else:
@@ -353,25 +356,15 @@ class GroupCoordinator:
356
group_name=self.unique_name)
357
return input_
358
356
- def _all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
357
- """
358
- The actual all-reduce implementation.
359
-
360
- NOTE: This operation will be applied in-place or out-of-place.
361
- Always assume this function modifies its input, but use the return
362
- value as the output.
363
- """
359
+ def _all_reduce_out_place(self, input_: torch.Tensor) -> torch.Tensor:
360
ca_comm = self.ca_comm
361
+ assert ca_comm is not None
362
+ assert not ca_comm.disabled
363
+ out = ca_comm.custom_all_reduce(input_)
364
+ assert out is not None
365
+ return out
366
366
- # For TPUs, use TPU communicator.
367
- tpu_comm = self.tpu_communicator
368
- if tpu_comm is not None and not tpu_comm.disabled:
369
- return tpu_comm.all_reduce(input_)
370
-
371
- if ca_comm is not None:
372
- out = ca_comm.custom_all_reduce(input_)
373
- if out is not None:
374
- return out
367
+ def _all_reduce_in_place(self, input_: torch.Tensor) -> None:
368
pynccl_comm = self.pynccl_comm
369
if (pynccl_comm is not None and not pynccl_comm.disabled):
370
pynccl_comm.all_reduce(input_)
@@ -380,7 +373,6 @@ class GroupCoordinator:
373
ipex.distributed.all_reduce(input_, group=self.device_group)
374
else:
375
torch.distributed.all_reduce(input_, group=self.device_group)
383
- return input_
376
377
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
378
world_size = self.world_size