@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 344a5d0c33
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Distributed] enable allreduce for multiple tp groups (#4566)

youkaichao committed May 2, 2024 at 17:32 UTC 344a5d0c332c3945caf336fd1d21f450f1455e6c
4 files changed +71 -22
tests/distributed/test_pynccl.py
+39 -4
@@ -3,9 +3,13 @@ import multiprocessing
3 import pytest
4 import torch
5
6 +import vllm.distributed.device_communicators.pynccl_utils as pynccl_utils
7 +from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
8 from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
9 ncclGetUniqueId)
8 -from vllm.distributed.parallel_state import init_distributed_environment
10 +from vllm.distributed.parallel_state import (
11 + ensure_model_parallel_initialized, get_tensor_model_parallel_cpu_group,
12 + init_distributed_environment, with_pynccl_for_all_reduce)
13 from vllm.utils import update_environment_variables
14
15
@@ -67,7 +71,7 @@ def multiple_tp_worker_fn():
71 ]
72 group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
73 comm = NCCLCommunicator(group=group, device=device)
70 - tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
74 + tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
75 # two groups can communicate independently
76 if torch.distributed.get_rank() in [0, 1]:
77 comm.all_reduce(tensor)
@@ -81,9 +85,40 @@ def multiple_tp_worker_fn():
85
86
87 @pytest.mark.skipif(torch.cuda.device_count() < 4,
84 - reason="Need at least 2 GPUs to run the test.")
88 + reason="Need at least 4 GPUs to run the test.")
89 def test_pynccl_multiple_tp():
86 - distributed_run(worker_fn, 4)
90 + # this tests pynccl for multiple tp groups, in a standalone way
91 + # i.e. call `comm.all_reduce` directly
92 + distributed_run(multiple_tp_worker_fn, 4)
93 +
94 +
95 +@worker_fn_wrapper
96 +def multiple_tp_with_vllm_worker_fn():
97 + device = torch.device(f"cuda:{torch.distributed.get_rank()}")
98 + torch.cuda.set_device(torch.distributed.get_rank())
99 + ensure_model_parallel_initialized(2, 2)
100 + pynccl_utils.init_process_group(
101 + group=get_tensor_model_parallel_cpu_group())
102 + tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
103 + with with_pynccl_for_all_reduce():
104 + # two tp groups can communicate independently
105 + if torch.distributed.get_rank() in [0, 1]:
106 + tensor = tensor_model_parallel_all_reduce(tensor)
107 + tensor = tensor_model_parallel_all_reduce(tensor)
108 + result = tensor.mean().cpu().item()
109 + assert result == 4
110 + else:
111 + tensor = tensor_model_parallel_all_reduce(tensor)
112 + result = tensor.mean().cpu().item()
113 + assert result == 2
114 +
115 +
116 +@pytest.mark.skipif(torch.cuda.device_count() < 4,
117 + reason="Need at least 4 GPUs to run the test.")
118 +def test_pynccl_multiple_tp_with_vllm():
119 + # this tests pynccl for multiple tp groups, together with vllm
120 + # i.e. call `tensor_model_parallel_all_reduce`
121 + distributed_run(multiple_tp_with_vllm_worker_fn, 4)
122
123
124 @worker_fn_wrapper
vllm/distributed/communication_op.py
-1
@@ -34,7 +34,6 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
34 if out is not None:
35 return out
36 if is_pynccl_enabled_for_all_reduce():
37 - # TODO: support multiple parallel groups.
37 pynccl_utils.all_reduce(input_)
38 else:
39 torch.distributed.all_reduce(input_,
vllm/distributed/parallel_state.py
+25 -11
@@ -14,7 +14,8 @@ from vllm.logger import init_logger
14 logger = init_logger(__name__)
15
16 # Tensor model parallel group that the current rank belongs to.
17 -_TENSOR_MODEL_PARALLEL_GROUP = None
17 +_TP_DEVICE_GROUP = None
18 +_TP_CPU_GROUP = None
19 # Pipeline model parallel group that the current rank belongs to.
20 _PIPELINE_MODEL_PARALLEL_GROUP = None
21
@@ -132,15 +133,17 @@ def initialize_model_parallel(
133 rank = torch.distributed.get_rank()
134
135 # Build the tensor model-parallel groups.
135 - global _TENSOR_MODEL_PARALLEL_GROUP
136 - assert _TENSOR_MODEL_PARALLEL_GROUP is None, (
136 + global _TP_DEVICE_GROUP, _TP_CPU_GROUP
137 + assert _TP_DEVICE_GROUP is None, (
138 "tensor model parallel group is already initialized")
139 for i in range(num_tensor_model_parallel_groups):
140 ranks = range(i * tensor_model_parallel_size,
141 (i + 1) * tensor_model_parallel_size)
142 group = torch.distributed.new_group(ranks, backend=backend)
143 + cpu_group = torch.distributed.new_group(ranks, backend="gloo")
144 if rank in ranks:
143 - _TENSOR_MODEL_PARALLEL_GROUP = group
145 + _TP_DEVICE_GROUP = group
146 + _TP_CPU_GROUP = cpu_group
147
148 # Build the pipeline model-parallel groups.
149 global _PIPELINE_MODEL_PARALLEL_GROUP
@@ -185,7 +188,7 @@ def ensure_model_parallel_initialized(
188
189 def model_parallel_is_initialized():
190 """Check if tensor and pipeline parallel groups are initialized."""
188 - return (_TENSOR_MODEL_PARALLEL_GROUP is not None
191 + return (_TP_DEVICE_GROUP is not None
192 and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
193
194
@@ -197,9 +200,16 @@ def get_cpu_world_group():
200
201 def get_tensor_model_parallel_group():
202 """Get the tensor model parallel group the caller rank belongs to."""
200 - assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
203 + assert _TP_DEVICE_GROUP is not None, (
204 "tensor model parallel group is not initialized")
202 - return _TENSOR_MODEL_PARALLEL_GROUP
205 + return _TP_DEVICE_GROUP
206 +
207 +
208 +def get_tensor_model_parallel_cpu_group():
209 + """Get the tensor model parallel cpu group the caller rank belongs to."""
210 + assert _TP_CPU_GROUP is not None, (
211 + "tensor model parallel cpu group is not initialized")
212 + return _TP_CPU_GROUP
213
214
215 def get_pipeline_model_parallel_group():
@@ -277,10 +287,14 @@ def get_pipeline_model_parallel_prev_rank():
287
288 def destroy_model_parallel():
289 """Set the groups to none and destroy them."""
280 - global _TENSOR_MODEL_PARALLEL_GROUP
281 - if _TENSOR_MODEL_PARALLEL_GROUP:
282 - torch.distributed.destroy_process_group(_TENSOR_MODEL_PARALLEL_GROUP)
283 - _TENSOR_MODEL_PARALLEL_GROUP = None
290 + global _TP_DEVICE_GROUP
291 + if _TP_DEVICE_GROUP:
292 + torch.distributed.destroy_process_group(_TP_DEVICE_GROUP)
293 + _TP_DEVICE_GROUP = None
294 + global _TP_CPU_GROUP
295 + if _TP_CPU_GROUP:
296 + torch.distributed.destroy_process_group(_TP_CPU_GROUP)
297 + _TP_CPU_GROUP = None
298 global _PIPELINE_MODEL_PARALLEL_GROUP
299 if _PIPELINE_MODEL_PARALLEL_GROUP:
300 torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
vllm/worker/worker.py
+7 -6
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 + get_tensor_model_parallel_cpu_group,
15 init_distributed_environment)
16 from vllm.distributed.device_communicators import pynccl_utils
17 from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -288,6 +289,9 @@ def init_worker_distributed_environment(
289 init_distributed_environment(parallel_config.world_size, rank,
290 distributed_init_method, local_rank)
291
292 + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
293 + parallel_config.pipeline_parallel_size)
294 +
295 if pynccl_utils.is_initialized():
296 pynccl_world_size = pynccl_utils.get_world_size()
297 if pynccl_world_size != parallel_config.world_size:
@@ -298,12 +302,9 @@ def init_worker_distributed_environment(
302 elif parallel_config.world_size > 1:
303 # NOTE(woosuk): We don't initialize pynccl process group when world size
304 # is 1.
301 - # NOTE(kaichao): By default, pynccl will use information inside
302 - # `parallel_state` for initialization.
303 - pynccl_utils.init_process_group()
304 -
305 - ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
306 - parallel_config.pipeline_parallel_size)
305 + # NOTE(kaichao): By default, pynccl is initialized for tp group.
306 + pynccl_utils.init_process_group(
307 + group=get_tensor_model_parallel_cpu_group())
308
309 # Initialize a custom fast all-reduce implementation.
310 if not parallel_config.disable_custom_all_reduce: