A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[core][distributed] custom allreduce when pp size > 1 (#6117)
youkaichao committed
Jul 3, 2024 at 14:41 UTC
3c6325f0fcfc46e573a107c9435abba6b6a617e8
2 files changed
+17
-15
vllm/config.py
+5
-11
@@ -723,17 +723,11 @@ class ParallelConfig:
723
if self.distributed_executor_backend == "ray":
724
from vllm.executor import ray_utils
725
ray_utils.assert_ray_available()
726
- if not self.disable_custom_all_reduce and self.world_size > 1:
727
- if is_hip():
728
- self.disable_custom_all_reduce = True
729
- logger.info(
730
- "Disabled the custom all-reduce kernel because it is not "
731
- "supported on AMD GPUs.")
732
- elif self.pipeline_parallel_size > 1:
733
- self.disable_custom_all_reduce = True
734
- logger.info(
735
- "Disabled the custom all-reduce kernel because it is not "
736
- "supported with pipeline parallelism.")
726
+ if is_hip():
727
+ self.disable_custom_all_reduce = True
728
+ logger.info(
729
+ "Disabled the custom all-reduce kernel because it is not "
730
+ "supported on AMD GPUs.")
731
if self.ray_workers_use_nsight and (
732
not self.distributed_executor_backend == "ray"):
733
raise ValueError("Unable to use nsight profiling unless workers "
vllm/distributed/parallel_state.py
+12
-4
@@ -719,14 +719,19 @@ def init_world_group(ranks: List[int], local_rank: int,
719
)
720
721
722
-def init_model_parallel_group(group_ranks: List[List[int]], local_rank: int,
723
- backend: str) -> GroupCoordinator:
722
+def init_model_parallel_group(
723
+ group_ranks: List[List[int]],
724
+ local_rank: int,
725
+ backend: str,
726
+ use_custom_allreduce: Optional[bool] = None) -> GroupCoordinator:
727
+ if use_custom_allreduce is None:
728
+ use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
729
return GroupCoordinator(
730
group_ranks=group_ranks,
731
local_rank=local_rank,
732
torch_distributed_backend=backend,
733
use_pynccl=True,
729
- use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
734
+ use_custom_allreduce=use_custom_allreduce,
735
)
736
737
@@ -888,8 +893,11 @@ def initialize_model_parallel(
893
for i in range(num_pipeline_model_parallel_groups):
894
ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
895
group_ranks.append(ranks)
896
+ # pipeline parallel does not need custom allreduce
897
_PP = init_model_parallel_group(group_ranks,
892
- get_world_group().local_rank, backend)
898
+ get_world_group().local_rank,
899
+ backend,
900
+ use_custom_allreduce=False)
901
902
903
def ensure_model_parallel_initialized(