@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3c6325f0fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[core][distributed] custom allreduce when pp size > 1 (#6117)

youkaichao committed Jul 3, 2024 at 14:41 UTC 3c6325f0fcfc46e573a107c9435abba6b6a617e8
2 files changed +17 -15
vllm/config.py
+5 -11
@@ -723,17 +723,11 @@ class ParallelConfig:
723 if self.distributed_executor_backend == "ray":
724 from vllm.executor import ray_utils
725 ray_utils.assert_ray_available()
726 - if not self.disable_custom_all_reduce and self.world_size > 1:
727 - if is_hip():
728 - self.disable_custom_all_reduce = True
729 - logger.info(
730 - "Disabled the custom all-reduce kernel because it is not "
731 - "supported on AMD GPUs.")
732 - elif self.pipeline_parallel_size > 1:
733 - self.disable_custom_all_reduce = True
734 - logger.info(
735 - "Disabled the custom all-reduce kernel because it is not "
736 - "supported with pipeline parallelism.")
726 + if is_hip():
727 + self.disable_custom_all_reduce = True
728 + logger.info(
729 + "Disabled the custom all-reduce kernel because it is not "
730 + "supported on AMD GPUs.")
731 if self.ray_workers_use_nsight and (
732 not self.distributed_executor_backend == "ray"):
733 raise ValueError("Unable to use nsight profiling unless workers "
vllm/distributed/parallel_state.py
+12 -4
@@ -719,14 +719,19 @@ def init_world_group(ranks: List[int], local_rank: int,
719 )
720
721
722 -def init_model_parallel_group(group_ranks: List[List[int]], local_rank: int,
723 - backend: str) -> GroupCoordinator:
722 +def init_model_parallel_group(
723 + group_ranks: List[List[int]],
724 + local_rank: int,
725 + backend: str,
726 + use_custom_allreduce: Optional[bool] = None) -> GroupCoordinator:
727 + if use_custom_allreduce is None:
728 + use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
729 return GroupCoordinator(
730 group_ranks=group_ranks,
731 local_rank=local_rank,
732 torch_distributed_backend=backend,
733 use_pynccl=True,
729 - use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
734 + use_custom_allreduce=use_custom_allreduce,
735 )
736
737
@@ -888,8 +893,11 @@ def initialize_model_parallel(
893 for i in range(num_pipeline_model_parallel_groups):
894 ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
895 group_ranks.append(ranks)
896 + # pipeline parallel does not need custom allreduce
897 _PP = init_model_parallel_group(group_ranks,
892 - get_world_group().local_rank, backend)
898 + get_world_group().local_rank,
899 + backend,
900 + use_custom_allreduce=False)
901
902
903 def ensure_model_parallel_initialized(