A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Distributed] enable allreduce for multiple tp groups (#4566)
youkaichao committed
May 2, 2024 at 17:32 UTC
344a5d0c332c3945caf336fd1d21f450f1455e6c
4 files changed
+71
-22
tests/distributed/test_pynccl.py
+39
-4
@@ -3,9 +3,13 @@ import multiprocessing
3
import pytest
4
import torch
5
6
+import vllm.distributed.device_communicators.pynccl_utils as pynccl_utils
7
+from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
8
from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
9
ncclGetUniqueId)
8
-from vllm.distributed.parallel_state import init_distributed_environment
10
+from vllm.distributed.parallel_state import (
11
+ ensure_model_parallel_initialized, get_tensor_model_parallel_cpu_group,
12
+ init_distributed_environment, with_pynccl_for_all_reduce)
13
from vllm.utils import update_environment_variables
14
15
@@ -67,7 +71,7 @@ def multiple_tp_worker_fn():
71
]
72
group = groups[0] if torch.distributed.get_rank() in [0, 1] else groups[1]
73
comm = NCCLCommunicator(group=group, device=device)
70
- tensor = torch.ones(16, 1024, 1024, dtype=torch.float32).cuda(comm.rank)
74
+ tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
75
# two groups can communicate independently
76
if torch.distributed.get_rank() in [0, 1]:
77
comm.all_reduce(tensor)
@@ -81,9 +85,40 @@ def multiple_tp_worker_fn():
85
86
87
@pytest.mark.skipif(torch.cuda.device_count() < 4,
84
- reason="Need at least 2 GPUs to run the test.")
88
+ reason="Need at least 4 GPUs to run the test.")
89
def test_pynccl_multiple_tp():
86
- distributed_run(worker_fn, 4)
90
+ # this tests pynccl for multiple tp groups, in a standalone way
91
+ # i.e. call `comm.all_reduce` directly
92
+ distributed_run(multiple_tp_worker_fn, 4)
93
+
94
+
95
+@worker_fn_wrapper
96
+def multiple_tp_with_vllm_worker_fn():
97
+ device = torch.device(f"cuda:{torch.distributed.get_rank()}")
98
+ torch.cuda.set_device(torch.distributed.get_rank())
99
+ ensure_model_parallel_initialized(2, 2)
100
+ pynccl_utils.init_process_group(
101
+ group=get_tensor_model_parallel_cpu_group())
102
+ tensor = torch.ones(16, 1024, 1024, dtype=torch.float32, device=device)
103
+ with with_pynccl_for_all_reduce():
104
+ # two tp groups can communicate independently
105
+ if torch.distributed.get_rank() in [0, 1]:
106
+ tensor = tensor_model_parallel_all_reduce(tensor)
107
+ tensor = tensor_model_parallel_all_reduce(tensor)
108
+ result = tensor.mean().cpu().item()
109
+ assert result == 4
110
+ else:
111
+ tensor = tensor_model_parallel_all_reduce(tensor)
112
+ result = tensor.mean().cpu().item()
113
+ assert result == 2
114
+
115
+
116
+@pytest.mark.skipif(torch.cuda.device_count() < 4,
117
+ reason="Need at least 4 GPUs to run the test.")
118
+def test_pynccl_multiple_tp_with_vllm():
119
+ # this tests pynccl for multiple tp groups, together with vllm
120
+ # i.e. call `tensor_model_parallel_all_reduce`
121
+ distributed_run(multiple_tp_with_vllm_worker_fn, 4)
122
123
124
@worker_fn_wrapper
vllm/distributed/communication_op.py
-1
@@ -34,7 +34,6 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
34
if out is not None:
35
return out
36
if is_pynccl_enabled_for_all_reduce():
37
- # TODO: support multiple parallel groups.
37
pynccl_utils.all_reduce(input_)
38
else:
39
torch.distributed.all_reduce(input_,
vllm/distributed/parallel_state.py
+25
-11
@@ -14,7 +14,8 @@ from vllm.logger import init_logger
14
logger = init_logger(__name__)
15
16
# Tensor model parallel group that the current rank belongs to.
17
-_TENSOR_MODEL_PARALLEL_GROUP = None
17
+_TP_DEVICE_GROUP = None
18
+_TP_CPU_GROUP = None
19
# Pipeline model parallel group that the current rank belongs to.
20
_PIPELINE_MODEL_PARALLEL_GROUP = None
21
@@ -132,15 +133,17 @@ def initialize_model_parallel(
133
rank = torch.distributed.get_rank()
134
135
# Build the tensor model-parallel groups.
135
- global _TENSOR_MODEL_PARALLEL_GROUP
136
- assert _TENSOR_MODEL_PARALLEL_GROUP is None, (
136
+ global _TP_DEVICE_GROUP, _TP_CPU_GROUP
137
+ assert _TP_DEVICE_GROUP is None, (
138
"tensor model parallel group is already initialized")
139
for i in range(num_tensor_model_parallel_groups):
140
ranks = range(i * tensor_model_parallel_size,
141
(i + 1) * tensor_model_parallel_size)
142
group = torch.distributed.new_group(ranks, backend=backend)
143
+ cpu_group = torch.distributed.new_group(ranks, backend="gloo")
144
if rank in ranks:
143
- _TENSOR_MODEL_PARALLEL_GROUP = group
145
+ _TP_DEVICE_GROUP = group
146
+ _TP_CPU_GROUP = cpu_group
147
148
# Build the pipeline model-parallel groups.
149
global _PIPELINE_MODEL_PARALLEL_GROUP
@@ -185,7 +188,7 @@ def ensure_model_parallel_initialized(
188
189
def model_parallel_is_initialized():
190
"""Check if tensor and pipeline parallel groups are initialized."""
188
- return (_TENSOR_MODEL_PARALLEL_GROUP is not None
191
+ return (_TP_DEVICE_GROUP is not None
192
and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
193
194
@@ -197,9 +200,16 @@ def get_cpu_world_group():
200
201
def get_tensor_model_parallel_group():
202
"""Get the tensor model parallel group the caller rank belongs to."""
200
- assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
203
+ assert _TP_DEVICE_GROUP is not None, (
204
"tensor model parallel group is not initialized")
202
- return _TENSOR_MODEL_PARALLEL_GROUP
205
+ return _TP_DEVICE_GROUP
206
+
207
+
208
+def get_tensor_model_parallel_cpu_group():
209
+ """Get the tensor model parallel cpu group the caller rank belongs to."""
210
+ assert _TP_CPU_GROUP is not None, (
211
+ "tensor model parallel cpu group is not initialized")
212
+ return _TP_CPU_GROUP
213
214
215
def get_pipeline_model_parallel_group():
@@ -277,10 +287,14 @@ def get_pipeline_model_parallel_prev_rank():
287
288
def destroy_model_parallel():
289
"""Set the groups to none and destroy them."""
280
- global _TENSOR_MODEL_PARALLEL_GROUP
281
- if _TENSOR_MODEL_PARALLEL_GROUP:
282
- torch.distributed.destroy_process_group(_TENSOR_MODEL_PARALLEL_GROUP)
283
- _TENSOR_MODEL_PARALLEL_GROUP = None
290
+ global _TP_DEVICE_GROUP
291
+ if _TP_DEVICE_GROUP:
292
+ torch.distributed.destroy_process_group(_TP_DEVICE_GROUP)
293
+ _TP_DEVICE_GROUP = None
294
+ global _TP_CPU_GROUP
295
+ if _TP_CPU_GROUP:
296
+ torch.distributed.destroy_process_group(_TP_CPU_GROUP)
297
+ _TP_CPU_GROUP = None
298
global _PIPELINE_MODEL_PARALLEL_GROUP
299
if _PIPELINE_MODEL_PARALLEL_GROUP:
300
torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
vllm/worker/worker.py
+7
-6
@@ -11,6 +11,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
+ get_tensor_model_parallel_cpu_group,
15
init_distributed_environment)
16
from vllm.distributed.device_communicators import pynccl_utils
17
from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -288,6 +289,9 @@ def init_worker_distributed_environment(
289
init_distributed_environment(parallel_config.world_size, rank,
290
distributed_init_method, local_rank)
291
292
+ ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
293
+ parallel_config.pipeline_parallel_size)
294
+
295
if pynccl_utils.is_initialized():
296
pynccl_world_size = pynccl_utils.get_world_size()
297
if pynccl_world_size != parallel_config.world_size:
@@ -298,12 +302,9 @@ def init_worker_distributed_environment(
302
elif parallel_config.world_size > 1:
303
# NOTE(woosuk): We don't initialize pynccl process group when world size
304
# is 1.
301
- # NOTE(kaichao): By default, pynccl will use information inside
302
- # `parallel_state` for initialization.
303
- pynccl_utils.init_process_group()
304
-
305
- ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
306
- parallel_config.pipeline_parallel_size)
305
+ # NOTE(kaichao): By default, pynccl is initialized for tp group.
306
+ pynccl_utils.init_process_group(
307
+ group=get_tensor_model_parallel_cpu_group())
308
309
# Initialize a custom fast all-reduce implementation.
310
if not parallel_config.disable_custom_all_reduce: