@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0437492ea9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

PP comm optimization: replace send with partial send + allgather (#6695)

Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com>

Aurick Qiao committed Jul 31, 2024 at 20:15 UTC 0437492ea97f0650a8b2ca39121be8864625fd70
2 files changed +41 -5
vllm/distributed/parallel_state.py
+36 -2
@@ -569,7 +569,8 @@ class GroupCoordinator:
569 def send_tensor_dict(
570 self,
571 tensor_dict: Dict[str, Union[torch.Tensor, Any]],
572 - dst: Optional[int] = None
572 + dst: Optional[int] = None,
573 + all_gather_group: Optional["GroupCoordinator"] = None,
574 ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
575 """Send the input tensor dictionary.
576 NOTE: `dst` is the local rank of the source rank.
@@ -578,6 +579,11 @@ class GroupCoordinator:
579 if not torch.distributed.is_initialized() or self.world_size == 1:
580 return tensor_dict
581
582 + all_gather_size = (1 if all_gather_group is None else
583 + all_gather_group.world_size)
584 + all_gather_rank = (0 if all_gather_group is None else
585 + all_gather_group.rank_in_group)
586 +
587 group = self.device_group
588 metadata_group = self.cpu_group
589
@@ -598,6 +604,12 @@ class GroupCoordinator:
604 if tensor.numel() == 0:
605 # Skip sending empty tensors.
606 continue
607 +
608 + # send-allgather: send only a slice, then do allgather.
609 + if (all_gather_group is not None
610 + and tensor.numel() % all_gather_size == 0):
611 + tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank]
612 +
613 if tensor.is_cpu:
614 # use metadata_group for CPU tensors
615 torch.distributed.send(tensor,
@@ -612,7 +624,8 @@ class GroupCoordinator:
624
625 def recv_tensor_dict(
626 self,
615 - src: Optional[int] = None
627 + src: Optional[int] = None,
628 + all_gather_group: Optional["GroupCoordinator"] = None,
629 ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]:
630 """Recv the input tensor dictionary.
631 NOTE: `src` is the local rank of the source rank.
@@ -621,6 +634,11 @@ class GroupCoordinator:
634 if not torch.distributed.is_initialized() or self.world_size == 1:
635 return None
636
637 + all_gather_size = (1 if all_gather_group is None else
638 + all_gather_group.world_size)
639 + all_gather_rank = (0 if all_gather_group is None else
640 + all_gather_group.rank_in_group)
641 +
642 group = self.device_group
643 metadata_group = self.cpu_group
644
@@ -639,6 +657,16 @@ class GroupCoordinator:
657 # Skip broadcasting empty tensors.
658 tensor_dict[key] = tensor
659 continue
660 +
661 + # send-allgather: send only a slice, then do allgather.
662 + use_all_gather = (all_gather_group is not None
663 + and tensor.numel() % all_gather_size == 0)
664 +
665 + if use_all_gather:
666 + orig_shape = tensor.shape
667 + tensor = tensor.reshape(all_gather_size,
668 + -1)[all_gather_rank]
669 +
670 if tensor.is_cpu:
671 # use metadata_group for CPU tensors
672 torch.distributed.recv(tensor,
@@ -649,6 +677,12 @@ class GroupCoordinator:
677 torch.distributed.recv(tensor,
678 src=self.ranks[src],
679 group=group)
680 + if use_all_gather:
681 + # do the allgather
682 + tensor = all_gather_group.all_gather( # type: ignore
683 + tensor, dim=0)
684 + tensor = tensor.reshape(orig_shape)
685 +
686 tensor_dict[key] = tensor
687 else:
688 tensor_dict[key] = value
vllm/worker/worker_base.py
+5 -3
@@ -6,7 +6,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
6
7 import torch
8
9 -from vllm.distributed import broadcast_tensor_dict, get_pp_group
9 +from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.platforms import current_platform
@@ -267,7 +267,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
267 intermediate_tensors = None
268 if not get_pp_group().is_first_rank:
269 intermediate_tensors = IntermediateTensors(
270 - get_pp_group().recv_tensor_dict())
270 + get_pp_group().recv_tensor_dict(
271 + all_gather_group=get_tp_group()))
272
273 output = self.model_runner.execute_model(
274 model_input, self.kv_cache[worker_input.virtual_engine]
@@ -276,7 +277,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
277
278 if not get_pp_group().is_last_rank:
279 # output is IntermediateTensors
279 - get_pp_group().send_tensor_dict(output.tensors)
280 + get_pp_group().send_tensor_dict(output.tensors,
281 + all_gather_group=get_tp_group())
282 return [None]
283
284 # output is List[SamplerOutput]