A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
PP comm optimization: replace send with partial send + allgather (#6695)
Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com>
Aurick Qiao committed
Jul 31, 2024 at 20:15 UTC
0437492ea97f0650a8b2ca39121be8864625fd70
2 files changed
+41
-5
vllm/distributed/parallel_state.py
+36
-2
@@ -569,7 +569,8 @@ class GroupCoordinator:
569
def send_tensor_dict(
570
self,
571
tensor_dict: Dict[str, Union[torch.Tensor, Any]],
572
- dst: Optional[int] = None
572
+ dst: Optional[int] = None,
573
+ all_gather_group: Optional["GroupCoordinator"] = None,
574
) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
575
"""Send the input tensor dictionary.
576
NOTE: `dst` is the local rank of the source rank.
@@ -578,6 +579,11 @@ class GroupCoordinator:
579
if not torch.distributed.is_initialized() or self.world_size == 1:
580
return tensor_dict
581
582
+ all_gather_size = (1 if all_gather_group is None else
583
+ all_gather_group.world_size)
584
+ all_gather_rank = (0 if all_gather_group is None else
585
+ all_gather_group.rank_in_group)
586
+
587
group = self.device_group
588
metadata_group = self.cpu_group
589
@@ -598,6 +604,12 @@ class GroupCoordinator:
604
if tensor.numel() == 0:
605
# Skip sending empty tensors.
606
continue
607
+
608
+ # send-allgather: send only a slice, then do allgather.
609
+ if (all_gather_group is not None
610
+ and tensor.numel() % all_gather_size == 0):
611
+ tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
612
+
613
if tensor.is_cpu:
614
# use metadata_group for CPU tensors
615
torch.distributed.send(tensor,
@@ -612,7 +624,8 @@ class GroupCoordinator:
624
625
def recv_tensor_dict(
626
self,
615
- src: Optional[int] = None
627
+ src: Optional[int] = None,
628
+ all_gather_group: Optional["GroupCoordinator"] = None,
629
) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
630
"""Recv the input tensor dictionary.
631
NOTE: `src` is the local rank of the source rank.
@@ -621,6 +634,11 @@ class GroupCoordinator:
634
if not torch.distributed.is_initialized() or self.world_size == 1:
635
return None
636
637
+ all_gather_size = (1 if all_gather_group is None else
638
+ all_gather_group.world_size)
639
+ all_gather_rank = (0 if all_gather_group is None else
640
+ all_gather_group.rank_in_group)
641
+
642
group = self.device_group
643
metadata_group = self.cpu_group
644
@@ -639,6 +657,16 @@ class GroupCoordinator:
657
# Skip broadcasting empty tensors.
658
tensor_dict[key] = tensor
659
continue
660
+
661
+ # send-allgather: send only a slice, then do allgather.
662
+ use_all_gather = (all_gather_group is not None
663
+ and tensor.numel() % all_gather_size == 0)
664
+
665
+ if use_all_gather:
666
+ orig_shape = tensor.shape
667
+ tensor = tensor.reshape(all_gather_size,
668
+ -1)[all_gather_rank]
669
+
670
if tensor.is_cpu:
671
# use metadata_group for CPU tensors
672
torch.distributed.recv(tensor,
@@ -649,6 +677,12 @@ class GroupCoordinator:
677
torch.distributed.recv(tensor,
678
src=self.ranks[src],
679
group=group)
680
+ if use_all_gather:
681
+ # do the allgather
682
+ tensor = all_gather_group.all_gather( # type: ignore
683
+ tensor, dim=0)
684
+ tensor = tensor.reshape(orig_shape)
685
+
686
tensor_dict[key] = tensor
687
else:
688
tensor_dict[key] = value
vllm/worker/worker_base.py
+5
-3
@@ -6,7 +6,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
6
7
import torch
8
9
-from vllm.distributed import broadcast_tensor_dict, get_pp_group
9
+from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.platforms import current_platform
@@ -267,7 +267,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
267
intermediate_tensors = None
268
if not get_pp_group().is_first_rank:
269
intermediate_tensors = IntermediateTensors(
270
- get_pp_group().recv_tensor_dict())
270
+ get_pp_group().recv_tensor_dict(
271
+ all_gather_group=get_tp_group()))
272
273
output = self.model_runner.execute_model(
274
model_input, self.kv_cache[worker_input.virtual_engine]
@@ -276,7 +277,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
277
278
if not get_pp_group().is_last_rank:
279
# output is IntermediateTensors
279
- get_pp_group().send_tensor_dict(output.tensors)
280
+ get_pp_group().send_tensor_dict(output.tensors,
281
+ all_gather_group=get_tp_group())
282
return [None]
283
284
# output is List[SamplerOutput]