A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core][Distributed] code deduplication in tp&pp with coordinator(#5293)
[Core][Distributed] add coordinator to reduce code duplication in tp and pp (#5293)
youkaichao committed
Jun 12, 2024 at 17:27 UTC
ea3890a5f0314e49d69afca45fe706504cb14029
12 files changed
+622
-582
tests/conftest.py
+3
-1
@@ -15,7 +15,8 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
15
16
from vllm import LLM, SamplingParams
17
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
18
-from vllm.distributed import destroy_model_parallel
18
+from vllm.distributed import (destroy_distributed_environment,
19
+ destroy_model_parallel)
20
from vllm.inputs import TextPrompt
21
from vllm.logger import init_logger
22
from vllm.multimodal import MultiModalData
@@ -54,6 +55,7 @@ def _read_prompts(filename: str) -> List[str]:
55
56
def cleanup():
57
destroy_model_parallel()
58
+ destroy_distributed_environment()
59
with contextlib.suppress(AssertionError):
60
torch.distributed.destroy_process_group()
61
gc.collect()
tests/distributed/test_custom_all_reduce.py
+3
-3
@@ -7,9 +7,9 @@ import torch
7
import torch.distributed as dist
8
9
from vllm.distributed.communication_op import ( # noqa
10
- graph_capture, tensor_model_parallel_all_reduce)
10
+ tensor_model_parallel_all_reduce)
11
from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12
- get_tp_ca_communicator)
12
+ get_tp_group, graph_capture)
13
14
from ..utils import (init_test_distributed_environment,
15
multi_process_tensor_parallel)
@@ -91,7 +91,7 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
91
# communicate independently
92
num_communication = rank // tp_size + 1
93
sz = 1024
94
- fa = get_tp_ca_communicator()
94
+ fa = get_tp_group().ca_comm
95
inp = torch.ones(sz, dtype=torch.float32, device=device)
96
out = inp
97
for _ in range(num_communication):
tests/distributed/test_pynccl.py
+8
-4
@@ -6,10 +6,11 @@ import torch
6
import torch.distributed
7
8
from vllm.distributed.communication_op import ( # noqa
9
- graph_capture, tensor_model_parallel_all_reduce)
9
+ tensor_model_parallel_all_reduce)
10
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
11
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
12
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
13
+ get_world_group, graph_capture,
14
init_distributed_environment)
15
from vllm.utils import update_environment_variables
16
@@ -53,7 +54,8 @@ def worker_fn_wrapper(fn):
54
55
@worker_fn_wrapper
56
def worker_fn():
56
- pynccl_comm = PyNcclCommunicator()
57
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
58
+ device=get_world_group().device)
59
tensor = torch.ones(16, 1024, 1024,
60
dtype=torch.float32).cuda(pynccl_comm.rank)
61
with pynccl_comm.change_state(enable=True):
@@ -129,7 +131,8 @@ def test_pynccl_multiple_allreduce_with_vllm():
131
def worker_fn_with_cudagraph():
132
with torch.no_grad():
133
graph = torch.cuda.CUDAGraph()
132
- pynccl_comm = PyNcclCommunicator()
134
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
135
+ device=get_world_group().device)
136
# run something in the default stream to initialize torch engine
137
a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
138
torch.cuda.synchronize()
@@ -154,7 +157,8 @@ def test_pynccl_with_cudagraph():
157
158
@worker_fn_wrapper
159
def send_recv_worker_fn():
157
- pynccl_comm = PyNcclCommunicator()
160
+ pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
161
+ device=get_world_group().device)
162
if pynccl_comm.rank == 0:
163
tensor = torch.ones(16, 1024, 1024,
164
dtype=torch.float32).cuda(pynccl_comm.rank)
tests/lora/conftest.py
+13
-10
@@ -12,7 +12,10 @@ from huggingface_hub import snapshot_download
12
13
import vllm
14
from vllm.config import LoRAConfig
15
-from vllm.distributed import destroy_model_parallel, initialize_model_parallel
15
+from vllm.distributed import (destroy_distributed_environment,
16
+ destroy_model_parallel,
17
+ init_distributed_environment,
18
+ initialize_model_parallel)
19
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
MergedColumnParallelLinear,
21
RowParallelLinear)
@@ -35,6 +38,7 @@ LONG_LORA_INFOS = [{
38
39
def cleanup():
40
destroy_model_parallel()
41
+ destroy_distributed_environment()
42
with contextlib.suppress(AssertionError):
43
torch.distributed.destroy_process_group()
44
gc.collect()
@@ -64,15 +68,14 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
68
69
@pytest.fixture
70
def dist_init():
67
- if not torch.distributed.is_initialized():
68
- temp_file = tempfile.mkstemp()[1]
69
- torch.distributed.init_process_group(
70
- backend="nccl",
71
- world_size=1,
72
- rank=0,
73
- init_method=f"file://{temp_file}",
74
- )
75
- torch.distributed.all_reduce(torch.zeros(1).cuda())
71
+ temp_file = tempfile.mkstemp()[1]
72
+ init_distributed_environment(
73
+ world_size=1,
74
+ rank=0,
75
+ distributed_init_method=f"file://{temp_file}",
76
+ local_rank=0,
77
+ backend="nccl",
78
+ )
79
initialize_model_parallel(1, 1)
80
yield
81
cleanup()
tests/worker/test_model_runner.py
+3
-1
@@ -1,7 +1,8 @@
1
import pytest
2
import torch
3
4
-from vllm.distributed.parallel_state import init_distributed_environment
4
+from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
5
+ init_distributed_environment)
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.model_executor.sampling_metadata import SamplingMetadata
8
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
@@ -292,6 +293,7 @@ def distributed_init():
293
rank=0,
294
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
295
local_rank=0)
296
+ ensure_model_parallel_initialized(1, 1)
297
298
299
@pytest.mark.parametrize("batch_size", list(range(2, 128)))
vllm/attention/backends/pallas.py
+1
-1
@@ -110,7 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110
raise NotImplementedError("TPU version must be 4 or higher.")
111
112
self.megacore_mode = None
113
- tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
113
+ tpu_type = torch_xla.tpu.get_tp_groupu_env()["TYPE"].lower()
114
if not tpu_type.endswith("lite"):
115
if self.num_kv_heads % 2 == 0:
116
self.megacore_mode = "kv_head"
vllm/distributed/communication_op.py
+13
-298
@@ -1,317 +1,32 @@
1
-from collections import namedtuple
2
-from contextlib import contextmanager, nullcontext
3
-from dataclasses import dataclass
4
-from typing import Any, Dict, List, Optional, Tuple, Union
1
+from typing import Any, Dict, Optional, Union
2
3
import torch
7
-from torch.distributed import ProcessGroup
4
+import torch.distributed
5
9
-from .parallel_state import (get_cpu_world_group, get_pp_pynccl_communicator,
10
- get_tensor_model_parallel_group,
11
- get_tensor_model_parallel_rank,
12
- get_tensor_model_parallel_world_size,
13
- get_tp_ca_communicator,
14
- get_tp_pynccl_communicator)
15
-
16
-
17
-@dataclass
18
-class GraphCaptureContext:
19
- stream: torch.cuda.Stream
20
-
21
-
22
-@contextmanager
23
-def graph_capture():
24
- """
25
- `graph_capture` is a context manager which should surround the code that
26
- is capturing the CUDA graph. Its main purpose is to ensure that the
27
- some operations will be run after the graph is captured, before the graph
28
- is replayed. It returns a `GraphCaptureContext` object which contains the
29
- necessary data for the graph capture. Currently, it only contains the
30
- stream that the graph capture is running on. This stream is set to the
31
- current CUDA stream when the context manager is entered and reset to the
32
- default stream when the context manager is exited. This is to ensure that
33
- the graph capture is running on a separate stream from the default stream,
34
- in order to explicitly distinguish the kernels to capture
35
- from other kernels possibly launched on background in the default stream.
36
- """
37
- stream = torch.cuda.Stream()
38
- graph_capture_context = GraphCaptureContext(stream)
39
- ca_comm = get_tp_ca_communicator()
40
- maybe_ca_context = nullcontext() if ca_comm is None else ca_comm.capture()
41
- with torch.cuda.stream(stream), maybe_ca_context:
42
- # In graph mode, we have to be very careful about the collective
43
- # operations. The current status is:
44
- # allreduce \ Mode | Eager | Graph |
45
- # --------------------------------------------
46
- # custom allreduce | enabled | enabled |
47
- # PyNccl | disabled| enabled |
48
- # torch.distributed | enabled | disabled|
49
- #
50
- # Note that custom allreduce will have a runtime check, if the tensor
51
- # size is too large, it will fallback to the next available option.
52
- # In summary: When using CUDA graph, we use
53
- # either custom all-reduce kernel or pynccl. When not using CUDA
54
- # graph, we use either custom all-reduce kernel or PyTorch NCCL.
55
- # We always prioritize using custom all-reduce kernel but fall back
56
- # to PyTorch or pynccl if it is disabled or not supported.
57
- tp_pynccl_comm = get_tp_pynccl_communicator()
58
- pp_pynccl_comm = get_pp_pynccl_communicator()
59
- if not tp_pynccl_comm:
60
- maybe_tp_pynccl_context = nullcontext()
61
- else:
62
- maybe_tp_pynccl_context = tp_pynccl_comm.change_state(
63
- enable=True, stream=torch.cuda.current_stream())
64
- if not pp_pynccl_comm:
65
- maybe_pp_pynccl_context = nullcontext()
66
- else:
67
- maybe_pp_pynccl_context = pp_pynccl_comm.change_state(
68
- enable=True, stream=torch.cuda.current_stream())
69
- with maybe_tp_pynccl_context, maybe_pp_pynccl_context:
70
- yield graph_capture_context
6
+from .parallel_state import get_tp_group
7
8
9
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
74
- """All-reduce the input tensor across model parallel group.
75
-
76
- NOTE: This operation will be applied in-place on the input tensor if
77
- disable_custom_all_reduce is set to True. Otherwise, this operation may or
78
- may not be applied in place depending on whether custom all reduce is
79
- invoked for a particular tensor, which further depends on the tensor size
80
- and GPU topology.
81
-
82
- TLDR: always assume this function modifies its input, but use the return
83
- value as the output.
84
- """
85
- ca_comm = get_tp_ca_communicator()
86
-
87
- # Bypass the function if we are using only 1 GPU.
88
- if get_tensor_model_parallel_world_size() == 1:
89
- return input_
90
- if ca_comm is not None:
91
- out = ca_comm.custom_all_reduce(input_)
92
- if out is not None:
93
- return out
94
- pynccl_comm = get_tp_pynccl_communicator()
95
- if (pynccl_comm is not None and not pynccl_comm.disabled):
96
- pynccl_comm.all_reduce(input_)
97
- else:
98
- torch.distributed.all_reduce(input_,
99
- group=get_tensor_model_parallel_group())
100
- return input_
10
+ """All-reduce the input tensor across model parallel group."""
11
+ return get_tp_group().all_reduce(input_)
12
13
14
def tensor_model_parallel_all_gather(input_: torch.Tensor,
15
dim: int = -1) -> torch.Tensor:
16
"""All-gather the input tensor across model parallel group."""
106
- world_size = get_tensor_model_parallel_world_size()
107
- # Bypass the function if we are using only 1 GPU.
108
- if world_size == 1:
109
- return input_
110
- assert -input_.dim() <= dim < input_.dim(), (
111
- f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
112
- if dim < 0:
113
- # Convert negative dim to positive.
114
- dim += input_.dim()
115
- input_size = input_.size()
116
- # Allocate output tensor.
117
- output_tensor = torch.empty((world_size, ) + input_size,
118
- dtype=input_.dtype,
119
- device=input_.device)
120
- # All-gather.
121
- torch.distributed.all_gather_into_tensor(
122
- output_tensor, input_, group=get_tensor_model_parallel_group())
123
- # Reshape
124
- output_tensor = output_tensor.movedim(0, dim)
125
- output_tensor = output_tensor.reshape(input_size[:dim] +
126
- (world_size * input_size[dim], ) +
127
- input_size[dim + 1:])
128
- return output_tensor
17
+ return get_tp_group().all_gather(input_, dim)
18
19
20
def tensor_model_parallel_gather(input_: torch.Tensor,
21
dst: int = 0,
22
dim: int = -1) -> torch.Tensor:
134
- """Gather the input tensor across model parallel group.
135
-
136
- NOTE: We assume that the input tensor is on the same device across
137
- all the ranks.
138
- """
139
- world_size = get_tensor_model_parallel_world_size()
140
- # Bypass the function if we are using only 1 GPU.
141
- if world_size == 1:
142
- return input_
143
- assert -input_.dim() <= dim < input_.dim(), (
144
- f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
145
- if dim < 0:
146
- # Convert negative dim to positive.
147
- dim += input_.dim()
148
- # Allocate output tensor.
149
- if get_tensor_model_parallel_rank() == dst:
150
- gather_list = [torch.empty_like(input_) for _ in range(world_size)]
151
- else:
152
- gather_list = None
153
- # Gather.
154
- torch.distributed.gather(input_,
155
- gather_list,
156
- dst=dst,
157
- group=get_tensor_model_parallel_group())
158
- if get_tensor_model_parallel_rank() == dst:
159
- output_tensor = torch.cat(gather_list, dim=dim)
160
- else:
161
- output_tensor = None
162
- return output_tensor
163
-
164
-
165
-def broadcast(input_: torch.Tensor,
166
- src: int = 0,
167
- group: Optional[ProcessGroup] = None):
168
- """Broadcast the input tensor."""
169
- group = group or torch.distributed.group.WORLD
170
- ranks = torch.distributed.get_process_group_ranks(group)
171
- assert src in ranks, f"Invalid src rank ({src})"
172
-
173
- # Bypass the function if we are using only 1 GPU.
174
- world_size = torch.distributed.get_world_size(group=group)
175
- if world_size == 1:
176
- return input_
177
- # Broadcast.
178
- torch.distributed.broadcast(input_, src=src, group=group)
179
- return input_
23
+ """Gather the input tensor across model parallel group."""
24
+ return get_tp_group().gather(input_, dst, dim)
25
26
182
-def broadcast_object_list(obj_list: List[Any],
183
- src: int = 0,
184
- group: Optional[ProcessGroup] = None):
185
- """Broadcast the input object list."""
186
- group = group or torch.distributed.group.WORLD
187
- ranks = torch.distributed.get_process_group_ranks(group)
188
- assert src in ranks, f"Invalid src rank ({src})"
189
-
190
- # Bypass the function if we are using only 1 GPU.
191
- world_size = torch.distributed.get_world_size(group=group)
192
- if world_size == 1:
193
- return obj_list
194
- # Broadcast.
195
- torch.distributed.broadcast_object_list(obj_list, src=src, group=group)
196
- return obj_list
197
-
198
-
199
-TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
200
-
201
-
202
-def _split_tensor_dict(
203
- tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
204
-) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
205
- """Split the tensor dictionary into two parts:
206
- 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
207
- by its metadata.
208
- 2. A list of tensors.
209
- """
210
- metadata_list = []
211
- tensor_list = []
212
- for key, value in tensor_dict.items():
213
- if isinstance(value, torch.Tensor):
214
- # Note: we cannot use `value.device` here,
215
- # because it contains not only the device type but also the device
216
- # index (e.g. "cuda:0"). We only need the device type.
217
- # receiving side will set the device index.
218
- device = "cpu" if value.is_cpu else "cuda"
219
- metadata_list.append(
220
- (key, TensorMetadata(device, value.dtype, value.size())))
221
- tensor_list.append(value)
222
- else:
223
- metadata_list.append((key, value))
224
- return metadata_list, tensor_list
225
-
226
-
227
-def broadcast_tensor_dict(
228
- tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
229
- src: int = 0,
230
- group: Optional[ProcessGroup] = None,
231
- metadata_group: Optional[ProcessGroup] = None
232
-) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
233
- """Broadcast the input tensor dictionary.
234
- `group` is used to broadcast the tensors, while `metadata_group` is used
235
- to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
236
- dtypes).
237
- """
238
- # Bypass the function if we are using only 1 GPU.
239
- if (not torch.distributed.is_initialized()
240
- or torch.distributed.get_world_size(group=group) == 1):
27
+def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
28
+ Any]]] = None,
29
+ src: int = 0):
30
+ if not torch.distributed.is_initialized():
31
return tensor_dict
242
-
243
- group = group or torch.distributed.group.WORLD
244
- metadata_group = metadata_group or get_cpu_world_group()
245
- ranks = torch.distributed.get_process_group_ranks(group)
246
- assert src in ranks, f"Invalid src rank ({src})"
247
-
248
- rank = torch.distributed.get_rank()
249
- if rank == src:
250
- metadata_list: List[Tuple[Any, Any]] = []
251
- assert isinstance(
252
- tensor_dict,
253
- dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
254
- metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
255
- # `metadata_list` lives in CPU memory.
256
- # `broadcast_object_list` involves serialization and deserialization,
257
- # all happening on CPU. Therefore, we can use the CPU group.
258
- torch.distributed.broadcast_object_list([metadata_list],
259
- src=src,
260
- group=metadata_group)
261
- async_handles = []
262
- for tensor in tensor_list:
263
- if tensor.numel() == 0:
264
- # Skip broadcasting empty tensors.
265
- continue
266
- if tensor.is_cpu:
267
- # use metadata_group for CPU tensors
268
- handle = torch.distributed.broadcast(tensor,
269
- src=src,
270
- group=metadata_group,
271
- async_op=True)
272
- else:
273
- # use group for GPU tensors
274
- handle = torch.distributed.broadcast(tensor,
275
- src=src,
276
- group=group,
277
- async_op=True)
278
- async_handles.append(handle)
279
- for async_handle in async_handles:
280
- async_handle.wait()
281
-
282
- else:
283
- recv_metadata_list = [None]
284
- torch.distributed.broadcast_object_list(recv_metadata_list,
285
- src=src,
286
- group=metadata_group)
287
- assert recv_metadata_list[0] is not None
288
- tensor_dict = {}
289
- async_handles = []
290
- for key, value in recv_metadata_list[0]:
291
- if isinstance(value, TensorMetadata):
292
- tensor = torch.empty(value.size,
293
- dtype=value.dtype,
294
- device=value.device)
295
- if tensor.numel() == 0:
296
- # Skip broadcasting empty tensors.
297
- tensor_dict[key] = tensor
298
- continue
299
- if tensor.is_cpu:
300
- # use metadata_group for CPU tensors
301
- handle = torch.distributed.broadcast(tensor,
302
- src=src,
303
- group=metadata_group,
304
- async_op=True)
305
- else:
306
- # use group for GPU tensors
307
- handle = torch.distributed.broadcast(tensor,
308
- src=src,
309
- group=group,
310
- async_op=True)
311
- async_handles.append(handle)
312
- tensor_dict[key] = tensor
313
- else:
314
- tensor_dict[key] = value
315
- for async_handle in async_handles:
316
- async_handle.wait()
317
- return tensor_dict
32
+ return get_tp_group().broadcast_tensor_dict(tensor_dict, src)
vllm/distributed/device_communicators/custom_all_reduce.py
+4
-9
@@ -9,8 +9,7 @@ import vllm.envs as envs
9
from vllm import _custom_ops as ops
10
from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11
gpu_p2p_access_check)
12
-from vllm.distributed.parallel_state import (
13
- get_local_rank, get_tensor_model_parallel_cpu_group, is_in_the_same_node)
12
+from vllm.distributed.parallel_state import is_in_the_same_node
13
from vllm.logger import init_logger
14
15
try:
@@ -86,8 +85,8 @@ class CustomAllreduce:
85
86
# max_size: max supported allreduce size
87
def __init__(self,
89
- group: Optional[ProcessGroup] = None,
90
- device: Optional[Union[int, str, torch.device]] = None,
88
+ group: ProcessGroup,
89
+ device: Union[int, str, torch.device],
90
max_size=8192 * 1024) -> None:
91
"""
92
Args:
@@ -107,7 +106,6 @@ class CustomAllreduce:
106
# e.g. in a non-cuda environment
107
return
108
110
- group = group or get_tensor_model_parallel_cpu_group()
109
self.group = group
110
111
assert dist.get_backend(group) != dist.Backend.NCCL, (
@@ -134,10 +132,7 @@ class CustomAllreduce:
132
world_size, str(CustomAllreduce._SUPPORTED_WORLD_SIZES))
133
return
134
137
- if device is None:
138
- local_rank = get_local_rank()
139
- device = torch.device(f"cuda:{local_rank}")
140
- elif isinstance(device, int):
135
+ if isinstance(device, int):
136
device = torch.device(f"cuda:{device}")
137
elif isinstance(device, str):
138
device = torch.device(device)
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+3
-4
@@ -11,7 +11,6 @@ import torch.distributed as dist
11
import torch.multiprocessing as mp
12
13
import vllm.envs as envs
14
-from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
14
from vllm.logger import init_logger
15
16
logger = init_logger(__name__)
@@ -162,7 +161,8 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
161
f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
162
)
163
os.makedirs(os.path.dirname(path), exist_ok=True)
165
- if ((not is_distributed or get_local_rank() == 0)
164
+ from vllm.distributed.parallel_state import get_world_group
165
+ if ((not is_distributed or get_world_group().local_rank == 0)
166
and (not os.path.exists(path))):
167
# only the local master process (with local_rank == 0) can
168
# enter this block to calculate the cache
@@ -174,8 +174,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
174
with open(path, "w") as f:
175
json.dump(cache, f, indent=4)
176
if is_distributed:
177
- cpu_world_group = get_cpu_world_group()
178
- dist.barrier(cpu_world_group)
177
+ get_world_group().barrier()
178
logger.info("reading GPU P2P access cache from %s", path)
179
with open(path, "r") as f:
180
cache = json.load(f)
vllm/distributed/device_communicators/pynccl.py
+3
-8
@@ -9,7 +9,6 @@ from torch.distributed import ProcessGroup, ReduceOp
9
from vllm.distributed.device_communicators.pynccl_wrapper import (
10
NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11
ncclRedOpTypeEnum, ncclUniqueId)
12
-from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
12
from vllm.logger import init_logger
13
14
logger = init_logger(__name__)
@@ -19,8 +18,8 @@ class PyNcclCommunicator:
18
19
def __init__(
20
self,
22
- group: Optional[ProcessGroup] = None,
23
- device: Optional[Union[int, str, torch.device]] = None,
21
+ group: ProcessGroup,
22
+ device: Union[int, str, torch.device],
23
library_path: Optional[str] = None,
24
):
25
"""
@@ -35,7 +34,6 @@ class PyNcclCommunicator:
34
is bind to a unique device.
35
"""
36
assert dist.is_initialized()
38
- group = get_cpu_world_group() if group is None else group
37
assert dist.get_backend(group) != dist.Backend.NCCL, (
38
"PyNcclCommunicator should be attached to a non-NCCL group.")
39
self.group = group
@@ -77,10 +75,7 @@ class PyNcclCommunicator:
75
byte_list = tensor.tolist()
76
for i, byte in enumerate(byte_list):
77
self.unique_id.internal[i] = byte
80
- if device is None:
81
- local_rank = get_local_rank()
82
- device = torch.device(f"cuda:{local_rank}")
83
- elif isinstance(device, int):
78
+ if isinstance(device, int):
79
device = torch.device(f"cuda:{device}")
80
elif isinstance(device, str):
81
device = torch.device(device)
vllm/distributed/parallel_state.py
+567
-242
@@ -2,81 +2,518 @@
2
# Adapted from
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-"""Tensor and pipeline parallel groups."""
5
+"""vLLM distributed state.
6
+It takes over the control of the distributed environment from PyTorch.
7
+The typical workflow is:
8
+
9
+- call `init_distributed_environment` to initialize the distributed environment.
10
+- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
11
+ initialize the model parallel groups.
12
+
13
+- any code dealing with the distributed stuff
14
+
15
+- call `destroy_model_parallel` to destroy the model parallel groups.
16
+- call `destroy_distributed_environment` to destroy the distributed environment.
17
+
18
+If you only need to use the distributed environment without model/pipeline
19
+ parallelism, you can skip the model parallel initialization and destruction
20
+ steps.
21
+"""
22
import contextlib
23
+from collections import namedtuple
24
+from contextlib import contextmanager, nullcontext
25
+from dataclasses import dataclass
26
from multiprocessing import resource_tracker, shared_memory
8
-from typing import List, Optional
27
+from typing import Any, Dict, List, Optional, Tuple, Union
28
29
import torch
11
-from torch.distributed import ProcessGroup
30
+from torch.distributed import Backend, ProcessGroup
31
32
import vllm.envs as envs
33
from vllm.logger import init_logger
34
16
-logger = init_logger(__name__)
35
18
-_ENABLE_CUSTOM_ALL_REDUCE = True
36
+@dataclass
37
+class GraphCaptureContext:
38
+ stream: torch.cuda.Stream
39
20
-# Tensor model parallel group that the current rank belongs to.
21
-_TP_DEVICE_GROUP: Optional[ProcessGroup] = None
22
-_TP_CPU_GROUP: Optional[ProcessGroup] = None
23
-_TP_PYNCCL_COMMUNICATOR = None
24
-_TP_CA_COMMUNICATOR = None
25
-# Pipeline model parallel group that the current rank belongs to.
26
-_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
27
-_PP_CPU_GROUP: Optional[ProcessGroup] = None
28
-_PP_PYNCCL_COMMUNICATOR = None
29
-
30
-# when people blindly call `torch.distributed.all_reduce` etc,
31
-# it will use this group. It is initialized with the `backend`
32
-# parameter of `init_distributed_environment` below.
33
-# Essentially, this is `torch.distributed.group.WORLD`.
34
-# We leave a line here to note that this is device-specific.
35
-# Note that this variable is not safe to use, because when users
36
-# call `init_distributed_environment` first, and then destroy
37
-# the process group themselves, this variable will keep a reference to the
38
-# destroyed process group, which is not useful.
39
-_DEVICE_WORLD_GROUP = None
40
-
41
-# duing `init_distributed_environment`, we will also initialize a
42
-# group with `gloo` backend, to allow direct coordination between
43
-# processes through the CPU.
44
-_CPU_WORLD_GROUP = None
45
-
46
-# In summary, after calling `init_distributed_environment`, we will
47
-# always have two groups: one for device-specific (and is the default)
48
-# and one for CPU. All processes will be part of both groups.
49
-
50
-# A list of global ranks for each pipeline group to ease calculation of the
51
-# source rank when broadcasting from the first or last pipeline stage.
52
-_PP_GLOBAL_RANKS: Optional[List[int]] = None
53
-
54
-_LOCAL_RANK = -1
40
41
+TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
42
57
-def set_custom_all_reduce(enable: bool):
58
- global _ENABLE_CUSTOM_ALL_REDUCE
59
- _ENABLE_CUSTOM_ALL_REDUCE = enable
43
44
+def _split_tensor_dict(
45
+ tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
46
+) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
47
+ """Split the tensor dictionary into two parts:
48
+ 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
49
+ by its metadata.
50
+ 2. A list of tensors.
51
+ """
52
+ metadata_list = []
53
+ tensor_list = []
54
+ for key, value in tensor_dict.items():
55
+ if isinstance(value, torch.Tensor):
56
+ # Note: we cannot use `value.device` here,
57
+ # because it contains not only the device type but also the device
58
+ # index (e.g. "cuda:0"). We only need the device type.
59
+ # receiving side will set the device index.
60
+ device = "cpu" if value.is_cpu else "cuda"
61
+ metadata_list.append(
62
+ (key, TensorMetadata(device, value.dtype, value.size())))
63
+ tensor_list.append(value)
64
+ else:
65
+ metadata_list.append((key, value))
66
+ return metadata_list, tensor_list
67
62
-def get_pp_pynccl_communicator():
63
- global _PP_PYNCCL_COMMUNICATOR
64
- return _PP_PYNCCL_COMMUNICATOR
68
69
+class GroupCoordinator:
70
+ """
71
+ PyTorch ProcessGroup wrapper for a group of processes.
72
+ PyTorch ProcessGroup is bound to one specific communication backend,
73
+ e.g. NCCL, Gloo, MPI, etc.
74
+ GroupCoordinator takes charge of all the communication operations among
75
+ the processes in the group. It can route the communication to
76
+ a specific implementation (e.g. switch allreduce implementation
77
+ based on the tensor size and cuda graph mode).
78
+ """
79
67
-def get_tp_pynccl_communicator():
68
- global _TP_PYNCCL_COMMUNICATOR
69
- return _TP_PYNCCL_COMMUNICATOR
80
+ # available attributes:
81
+ rank: int # global rank
82
+ ranks: List[int] # global ranks in the group
83
+ world_size: int # size of the group
84
+ # difference between `local_rank` and `rank_in_group`:
85
+ # if we have a group of size 4 across two nodes:
86
+ # Process | Node | Rank | Local Rank | Rank in Group
87
+ # 0 | 0 | 0 | 0 | 0
88
+ # 1 | 0 | 1 | 1 | 1
89
+ # 2 | 1 | 2 | 0 | 2
90
+ # 3 | 1 | 3 | 1 | 3
91
+ local_rank: int # local rank used to assign devices
92
+ rank_in_group: int # rank inside the group
93
+ cpu_group: ProcessGroup # group for CPU communication
94
+ device_group: ProcessGroup # group for device communication
95
+ use_pynccl: bool # a hint of whether to use PyNccl
96
+ use_custom_allreduce: bool # a hint of whether to use CustomAllreduce
97
+ # communicators are only created for world size > 1
98
+ pynccl_comm: Optional[Any] # PyNccl communicator
99
+ ca_comm: Optional[Any] # Custom allreduce communicator
100
+
101
+ def __init__(
102
+ self,
103
+ group_ranks: List[List[int]],
104
+ local_rank: int,
105
+ torch_distributed_backend: Union[str, Backend],
106
+ use_pynccl: bool,
107
+ use_custom_allreduce: bool,
108
+ ):
109
+
110
+ self.rank = torch.distributed.get_rank()
111
+ self.local_rank = local_rank
112
+ self.device_group = None
113
+ self.cpu_group = None
114
+
115
+ for ranks in group_ranks:
116
+ device_group = torch.distributed.new_group(
117
+ ranks, backend=torch_distributed_backend)
118
+ # a group with `gloo` backend, to allow direct coordination between
119
+ # processes through the CPU.
120
+ cpu_group = torch.distributed.new_group(ranks, backend="gloo")
121
+ if self.rank in ranks:
122
+ self.ranks = ranks
123
+ self.world_size = len(ranks)
124
+ self.rank_in_group = ranks.index(self.rank)
125
+ self.device_group = device_group
126
+ self.cpu_group = cpu_group
127
+
128
+ assert self.cpu_group is not None
129
+ assert self.device_group is not None
130
131
+ if torch.cuda.is_available():
132
+ self.device = torch.device(f"cuda:{local_rank}")
133
+ else:
134
+ self.device = torch.device("cpu")
135
72
-def get_tp_ca_communicator():
73
- global _TP_CA_COMMUNICATOR
74
- return _TP_CA_COMMUNICATOR
136
+ self.use_pynccl = use_pynccl
137
+ self.use_custom_allreduce = use_custom_allreduce
138
+
139
+ # lazy import to avoid documentation build error
140
+ from vllm.distributed.device_communicators.custom_all_reduce import (
141
+ CustomAllreduce)
142
+ from vllm.distributed.device_communicators.pynccl import (
143
+ PyNcclCommunicator)
144
+
145
+ self.pynccl_comm: Optional[PyNcclCommunicator]
146
+ if use_pynccl and self.world_size > 1:
147
+ self.pynccl_comm = PyNcclCommunicator(
148
+ group=self.cpu_group,
149
+ device=self.device,
150
+ )
151
+ else:
152
+ self.pynccl_comm = None
153
+
154
+ self.ca_comm: Optional[CustomAllreduce]
155
+ if use_custom_allreduce and self.world_size > 1:
156
+ # Initialize a custom fast all-reduce implementation.
157
+ self.ca_comm = CustomAllreduce(
158
+ group=self.cpu_group,
159
+ device=self.device,
160
+ )
161
+ else:
162
+ self.ca_comm = None
163
+
164
+ @property
165
+ def first_rank(self):
166
+ """Return the global rank of the first process in the group"""
167
+ return self.ranks[0]
168
+
169
+ @property
170
+ def last_rank(self):
171
+ """Return the global rank of the last process in the group"""
172
+ return self.ranks[-1]
173
+
174
+ @property
175
+ def next_rank(self):
176
+ """Return the global rank of the process that follows the caller"""
177
+ rank_in_group = self.rank_in_group
178
+ world_size = self.world_size
179
+ return self.ranks[(rank_in_group + 1) % world_size]
180
+
181
+ @property
182
+ def prev_rank(self):
183
+ """Return the global rank of the process that precedes the caller"""
184
+ rank_in_group = self.rank_in_group
185
+ world_size = self.world_size
186
+ return self.ranks[(rank_in_group - 1) % world_size]
187
+
188
+ @contextmanager
189
+ def graph_capture(
190
+ self, graph_capture_context: Optional[GraphCaptureContext] = None):
191
+ if graph_capture_context is None:
192
+ stream = torch.cuda.Stream()
193
+ graph_capture_context = GraphCaptureContext(stream)
194
+ else:
195
+ stream = graph_capture_context.stream
196
+
197
+ ca_comm = self.ca_comm
198
+ maybe_ca_context = nullcontext(
199
+ ) if ca_comm is None else ca_comm.capture()
200
+ with torch.cuda.stream(stream), maybe_ca_context:
201
+ # In graph mode, we have to be very careful about the collective
202
+ # operations. The current status is:
203
+ # allreduce \ Mode | Eager | Graph |
204
+ # --------------------------------------------
205
+ # custom allreduce | enabled | enabled |
206
+ # PyNccl | disabled| enabled |
207
+ # torch.distributed | enabled | disabled|
208
+ #
209
+ # Note that custom allreduce will have a runtime check, if the
210
+ # tensor size is too large, it will fallback to the next
211
+ # available option.
212
+ # In summary: When using CUDA graph, we use
213
+ # either custom all-reduce kernel or pynccl. When not using
214
+ # CUDA graph, we use either custom all-reduce kernel or
215
+ # PyTorch NCCL. We always prioritize using custom all-reduce
216
+ # kernel but fall back to PyTorch or pynccl if it is
217
+ # disabled or not supported.
218
+ pynccl_comm = self.pynccl_comm
219
+ maybe_pynccl_context: Any
220
+ if not pynccl_comm:
221
+ maybe_pynccl_context = nullcontext()
222
+ else:
223
+ maybe_pynccl_context = pynccl_comm.change_state(
224
+ enable=True, stream=torch.cuda.current_stream())
225
+ with maybe_pynccl_context:
226
+ yield graph_capture_context
227
+
228
+ def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
229
+ """
230
+ NOTE: This operation will be applied in-place or out-of-place.
231
+ Always assume this function modifies its input, but use the return
232
+ value as the output.
233
+ """
234
+ ca_comm = self.ca_comm
235
+
236
+ # Bypass the function if we are using only 1 GPU.
237
+ if self.world_size == 1:
238
+ return input_
239
+ if ca_comm is not None:
240
+ out = ca_comm.custom_all_reduce(input_)
241
+ if out is not None:
242
+ return out
243
+ pynccl_comm = self.pynccl_comm
244
+ if (pynccl_comm is not None and not pynccl_comm.disabled):
245
+ pynccl_comm.all_reduce(input_)
246
+ else:
247
+ torch.distributed.all_reduce(input_, group=self.device_group)
248
+ return input_
249
+
250
+ def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
251
+ world_size = self.world_size
252
+ # Bypass the function if we are using only 1 GPU.
253
+ if world_size == 1:
254
+ return input_
255
+ assert -input_.dim() <= dim < input_.dim(), (
256
+ f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
257
+ if dim < 0:
258
+ # Convert negative dim to positive.
259
+ dim += input_.dim()
260
+ input_size = input_.size()
261
+ # Allocate output tensor.
262
+ output_tensor = torch.empty((world_size, ) + input_size,
263
+ dtype=input_.dtype,
264
+ device=input_.device)
265
+ # All-gather.
266
+ torch.distributed.all_gather_into_tensor(output_tensor,
267
+ input_,
268
+ group=self.device_group)
269
+ # Reshape
270
+ output_tensor = output_tensor.movedim(0, dim)
271
+ output_tensor = output_tensor.reshape(input_size[:dim] +
272
+ (world_size *
273
+ input_size[dim], ) +
274
+ input_size[dim + 1:])
275
+ return output_tensor
276
+
277
+ def gather(self,
278
+ input_: torch.Tensor,
279
+ dst: int = 0,
280
+ dim: int = -1) -> torch.Tensor:
281
+ """
282
+ NOTE: We assume that the input tensor is on the same device across
283
+ all the ranks.
284
+ NOTE: `dst` is the local rank of the destination rank.
285
+ """
286
+ world_size = self.world_size
287
+ # Bypass the function if we are using only 1 GPU.
288
+ if world_size == 1:
289
+ return input_
290
+ assert -input_.dim() <= dim < input_.dim(), (
291
+ f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
292
+ if dim < 0:
293
+ # Convert negative dim to positive.
294
+ dim += input_.dim()
295
+ # Allocate output tensor.
296
+ if self.rank_in_group == dst:
297
+ gather_list = [torch.empty_like(input_) for _ in range(world_size)]
298
+ else:
299
+ gather_list = None
300
+ # Gather.
301
+ torch.distributed.gather(input_,
302
+ gather_list,
303
+ dst=self.ranks[dst],
304
+ group=self.device_group)
305
+ if self.rank_in_group == dst:
306
+ output_tensor = torch.cat(gather_list, dim=dim)
307
+ else:
308
+ output_tensor = None
309
+ return output_tensor
310
+
311
+ def broadcast(self, input_: torch.Tensor, src: int = 0):
312
+ """Broadcast the input tensor.
313
+ NOTE: `src` is the local rank of the source rank.
314
+ """
315
+ assert src < self.world_size, f"Invalid src rank ({src})"
316
+
317
+ # Bypass the function if we are using only 1 GPU.
318
+ if self.world_size == 1:
319
+ return input_
320
+ # Broadcast.
321
+ torch.distributed.broadcast(input_,
322
+ src=self.ranks[src],
323
+ group=self.device_group)
324
+ return input_
325
+
326
+ def broadcast_object_list(self,
327
+ obj_list: List[Any],
328
+ src: int = 0,
329
+ group: Optional[ProcessGroup] = None):
330
+ """Broadcast the input object list.
331
+ NOTE: `src` is the local rank of the source rank.
332
+ """
333
+ assert src < self.world_size, f"Invalid src rank ({src})"
334
+
335
+ # Bypass the function if we are using only 1 GPU.
336
+ if self.world_size == 1:
337
+ return obj_list
338
+ # Broadcast.
339
+ torch.distributed.broadcast_object_list(obj_list,
340
+ src=self.ranks[src],
341
+ group=self.device_group)
342
+ return obj_list
343
+
344
+ def broadcast_tensor_dict(
345
+ self,
346
+ tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
347
+ src: int = 0,
348
+ group: Optional[ProcessGroup] = None,
349
+ metadata_group: Optional[ProcessGroup] = None
350
+ ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
351
+ """Broadcast the input tensor dictionary.
352
+ NOTE: `src` is the local rank of the source rank.
353
+ """
354
+ # Bypass the function if we are using only 1 GPU.
355
+ if (not torch.distributed.is_initialized() or self.world_size == 1):
356
+ return tensor_dict
357
+
358
+ group = self.device_group
359
+ metadata_group = self.cpu_group
360
+ assert src < self.world_size, f"Invalid src rank ({src})"
361
+ src = self.ranks[src]
362
+
363
+ rank = self.rank
364
+ if rank == src:
365
+ metadata_list: List[Tuple[Any, Any]] = []
366
+ assert isinstance(
367
+ tensor_dict,
368
+ dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
369
+ metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
370
+ # `metadata_list` lives in CPU memory.
371
+ # `broadcast_object_list` has serialization & deserialization,
372
+ # all happening on CPU. Therefore, we can use the CPU group.
373
+ torch.distributed.broadcast_object_list([metadata_list],
374
+ src=src,
375
+ group=metadata_group)
376
+ async_handles = []
377
+ for tensor in tensor_list:
378
+ if tensor.numel() == 0:
379
+ # Skip broadcasting empty tensors.
380
+ continue
381
+ if tensor.is_cpu:
382
+ # use metadata_group for CPU tensors
383
+ handle = torch.distributed.broadcast(tensor,
384
+ src=src,
385
+ group=metadata_group,
386
+ async_op=True)
387
+ else:
388
+ # use group for GPU tensors
389
+ handle = torch.distributed.broadcast(tensor,
390
+ src=src,
391
+ group=group,
392
+ async_op=True)
393
+ async_handles.append(handle)
394
+ for async_handle in async_handles:
395
+ async_handle.wait()
396
+
397
+ else:
398
+ recv_metadata_list = [None]
399
+ torch.distributed.broadcast_object_list(recv_metadata_list,
400
+ src=src,
401
+ group=metadata_group)
402
+ assert recv_metadata_list[0] is not None
403
+ tensor_dict = {}
404
+ async_handles = []
405
+ for key, value in recv_metadata_list[0]:
406
+ if isinstance(value, TensorMetadata):
407
+ tensor = torch.empty(value.size,
408
+ dtype=value.dtype,
409
+ device=value.device)
410
+ if tensor.numel() == 0:
411
+ # Skip broadcasting empty tensors.
412
+ tensor_dict[key] = tensor
413
+ continue
414
+ if tensor.is_cpu:
415
+ # use metadata_group for CPU tensors
416
+ handle = torch.distributed.broadcast(
417
+ tensor,
418
+ src=src,
419
+ group=metadata_group,
420
+ async_op=True)
421
+ else:
422
+ # use group for GPU tensors
423
+ handle = torch.distributed.broadcast(tensor,
424
+ src=src,
425
+ group=group,
426
+ async_op=True)
427
+ async_handles.append(handle)
428
+ tensor_dict[key] = tensor
429
+ else:
430
+ tensor_dict[key] = value
431
+ for async_handle in async_handles:
432
+ async_handle.wait()
433
+ return tensor_dict
434
+
435
+ def barrier(self):
436
+ """Barrier synchronization among the group.
437
+ NOTE: don't use `device_group` here! `barrier` in NCCL is
438
+ terrible because it is internally a broadcast operation with
439
+ secretly created GPU tensors. It is easy to mess up the current
440
+ device. Use the CPU group instead.
441
+ """
442
+ torch.distributed.barrier(group=self.cpu_group)
443
+
444
+ def destroy(self):
445
+ if self.device_group is not None:
446
+ torch.distributed.destroy_process_group(self.device_group)
447
+ self.device_group = None
448
+ if self.cpu_group is not None:
449
+ torch.distributed.destroy_process_group(self.cpu_group)
450
+ self.cpu_group = None
451
+ if self.pynccl_comm is not None:
452
+ self.pynccl_comm = None
453
+ if self.ca_comm is not None:
454
+ self.ca_comm = None
455
+
456
+
457
+_WORLD: Optional[GroupCoordinator] = None
458
+
459
+
460
+def get_world_group() -> GroupCoordinator:
461
+ assert _WORLD is not None, ("world group is not initialized")
462
+ return _WORLD
463
+
464
+
465
+_TP: Optional[GroupCoordinator] = None
466
+
467
+
468
+def get_tp_group() -> GroupCoordinator:
469
+ assert _TP is not None, ("tensor model parallel group is not initialized")
470
+ return _TP
471
+
472
+
473
+# kept for backward compatibility
474
+get_tensor_model_parallel_group = get_tp_group
475
+
476
+_PP: Optional[GroupCoordinator] = None
477
+
478
+
479
+def get_pp_group() -> GroupCoordinator:
480
+ assert _PP is not None, (
481
+ "pipeline model parallel group is not initialized")
482
+ return _PP
483
484
77
-def get_local_rank():
78
- global _LOCAL_RANK
79
- return _LOCAL_RANK
485
+# kept for backward compatibility
486
+get_pipeline_model_parallel_group = get_pp_group
487
+
488
+
489
+@contextmanager
490
+def graph_capture():
491
+ """
492
+ `graph_capture` is a context manager which should surround the code that
493
+ is capturing the CUDA graph. Its main purpose is to ensure that the
494
+ some operations will be run after the graph is captured, before the graph
495
+ is replayed. It returns a `GraphCaptureContext` object which contains the
496
+ necessary data for the graph capture. Currently, it only contains the
497
+ stream that the graph capture is running on. This stream is set to the
498
+ current CUDA stream when the context manager is entered and reset to the
499
+ default stream when the context manager is exited. This is to ensure that
500
+ the graph capture is running on a separate stream from the default stream,
501
+ in order to explicitly distinguish the kernels to capture
502
+ from other kernels possibly launched on background in the default stream.
503
+ """
504
+ with get_tp_group().graph_capture() as context, get_pp_group(
505
+ ).graph_capture(context):
506
+ yield context
507
+
508
+
509
+logger = init_logger(__name__)
510
+
511
+_ENABLE_CUSTOM_ALL_REDUCE = True
512
+
513
+
514
+def set_custom_all_reduce(enable: bool):
515
+ global _ENABLE_CUSTOM_ALL_REDUCE
516
+ _ENABLE_CUSTOM_ALL_REDUCE = enable
517
518
519
def init_distributed_environment(
@@ -100,31 +537,29 @@ def init_distributed_environment(
537
init_method=distributed_init_method,
538
world_size=world_size,
539
rank=rank)
103
- global _DEVICE_WORLD_GROUP, _CPU_WORLD_GROUP
104
- _DEVICE_WORLD_GROUP = torch.distributed.group.WORLD
540
+ # set the local rank
541
+ # local_rank is not available in torch ProcessGroup,
542
+ # see https://github.com/pytorch/pytorch/issues/122816
543
+ if local_rank == -1:
544
+ # local rank not set, this usually happens in single-node
545
+ # setting, where we can use rank as local rank
546
+ if distributed_init_method == "env://":
547
+ local_rank = envs.LOCAL_RANK
548
+ else:
549
+ local_rank = rank
550
+ global _WORLD
551
+ if _WORLD is None:
552
ranks = list(range(torch.distributed.get_world_size()))
106
- _CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
107
- backend="gloo")
108
- # set the local rank
109
- # local_rank is not available in torch ProcessGroup,
110
- # see https://github.com/pytorch/pytorch/issues/122816
111
- if local_rank == -1:
112
- # local rank not set, this usually happens in single-node
113
- # setting, where we can use rank as local rank
114
- if distributed_init_method == "env://":
115
- local_rank = envs.LOCAL_RANK
116
- else:
117
- local_rank = rank
118
- global _LOCAL_RANK
119
- _LOCAL_RANK = local_rank
120
- # A small all_reduce for warmup.
121
- data = torch.zeros(1)
122
- if torch.cuda.is_available():
123
- data = data.to(device=f"cuda:{local_rank}")
124
- torch.distributed.all_reduce(data)
125
- if torch.cuda.is_available():
126
- torch.cuda.synchronize()
127
- del data
553
+ _WORLD = GroupCoordinator(
554
+ group_ranks=[ranks],
555
+ local_rank=local_rank,
556
+ torch_distributed_backend=backend,
557
+ use_pynccl=False,
558
+ use_custom_allreduce=False,
559
+ )
560
+ else:
561
+ assert _WORLD.world_size == torch.distributed.get_world_size(), (
562
+ "world group already initialized with a different world size")
563
564
565
def initialize_model_parallel(
@@ -157,8 +592,8 @@ def initialize_model_parallel(
592
# Get world size and rank. Ensure some consistencies.
593
assert torch.distributed.is_initialized()
594
world_size: int = torch.distributed.get_world_size()
160
- # get the backend of _DEVICE_WORLD_GROUP
161
- backend = backend or torch.distributed.get_backend()
595
+ backend = backend or torch.distributed.get_backend(
596
+ get_world_group().device_group)
597
598
if (world_size !=
599
tensor_model_parallel_size * pipeline_model_parallel_size):
@@ -167,63 +602,42 @@ def initialize_model_parallel(
602
f"tensor_model_parallel_size ({tensor_model_parallel_size}) x "
603
f"pipeline_model_parallel_size ({pipeline_model_parallel_size})")
604
605
+ # Build the tensor model-parallel groups.
606
num_tensor_model_parallel_groups: int = (world_size //
607
tensor_model_parallel_size)
172
- num_pipeline_model_parallel_groups: int = (world_size //
173
- pipeline_model_parallel_size)
174
- rank = torch.distributed.get_rank()
175
-
176
- # Build the tensor model-parallel groups.
177
- global _TP_DEVICE_GROUP, _TP_CPU_GROUP
178
- global _TP_PYNCCL_COMMUNICATOR, _TP_CA_COMMUNICATOR
179
- assert _TP_DEVICE_GROUP is None, (
180
- "tensor model parallel group is already initialized")
608
+ global _TP
609
+ assert _TP is None, ("tensor model parallel group is already initialized")
610
+ group_ranks = []
611
for i in range(num_tensor_model_parallel_groups):
612
ranks = list(
613
range(i * tensor_model_parallel_size,
614
(i + 1) * tensor_model_parallel_size))
185
- group = torch.distributed.new_group(ranks, backend=backend)
186
- cpu_group = torch.distributed.new_group(ranks, backend="gloo")
187
- if rank in ranks:
188
- _TP_DEVICE_GROUP = group
189
- _TP_CPU_GROUP = cpu_group
190
-
191
- from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
192
- if tensor_model_parallel_size > 1:
193
- _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
194
- group=_TP_CPU_GROUP,
195
- device=_LOCAL_RANK,
196
- )
197
-
198
- # Initialize a custom fast all-reduce implementation.
199
- if _ENABLE_CUSTOM_ALL_REDUCE:
200
- from vllm.distributed.device_communicators.custom_all_reduce import (
201
- CustomAllreduce)
202
- _TP_CA_COMMUNICATOR = CustomAllreduce(
203
- group=_TP_CPU_GROUP,
204
- device=_LOCAL_RANK,
205
- )
615
+ group_ranks.append(ranks)
616
+ _TP = GroupCoordinator(
617
+ group_ranks=group_ranks,
618
+ local_rank=get_world_group().local_rank,
619
+ torch_distributed_backend=backend,
620
+ use_pynccl=True,
621
+ use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
622
+ )
623
624
# Build the pipeline model-parallel groups.
208
- global _PP_DEVICE_GROUP, _PP_CPU_GROUP
209
- global _PP_PYNCCL_COMMUNICATOR
210
- global _PP_GLOBAL_RANKS
211
- assert _PP_DEVICE_GROUP is None, (
625
+ num_pipeline_model_parallel_groups: int = (world_size //
626
+ pipeline_model_parallel_size)
627
+ global _PP
628
+ assert _PP is None, (
629
"pipeline model parallel group is already initialized")
630
+ group_ranks = []
631
for i in range(num_pipeline_model_parallel_groups):
632
ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
215
- group = torch.distributed.new_group(ranks, backend=backend)
216
- cpu_group = torch.distributed.new_group(ranks, backend="gloo")
217
- if rank in ranks:
218
- _PP_DEVICE_GROUP = group
219
- _PP_CPU_GROUP = cpu_group
220
- _PP_GLOBAL_RANKS = ranks
221
-
222
- if pipeline_model_parallel_size > 1:
223
- _PP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
224
- group=_PP_CPU_GROUP,
225
- device=_LOCAL_RANK,
226
- )
633
+ group_ranks.append(ranks)
634
+ _PP = GroupCoordinator(
635
+ group_ranks=group_ranks,
636
+ local_rank=get_world_group().local_rank,
637
+ torch_distributed_backend=backend,
638
+ use_pynccl=True,
639
+ use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
640
+ )
641
642
643
def ensure_model_parallel_initialized(
@@ -235,8 +649,8 @@ def ensure_model_parallel_initialized(
649
or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
650
values if the model parallel groups are initialized.
651
"""
238
- # get the backend of _DEVICE_WORLD_GROUP
239
- backend = backend or torch.distributed.get_backend()
652
+ backend = backend or torch.distributed.get_backend(
653
+ get_world_group().device_group)
654
if not model_parallel_is_initialized():
655
initialize_model_parallel(tensor_model_parallel_size,
656
pipeline_model_parallel_size, backend)
@@ -247,137 +661,48 @@ def ensure_model_parallel_initialized(
661
), ("tensor parallel group already initialized, but of unexpected size: "
662
f"{get_tensor_model_parallel_world_size()=} vs. "
663
f"{tensor_model_parallel_size=}")
250
- assert (get_pipeline_model_parallel_world_size(
251
- ) == pipeline_model_parallel_size), (
664
+ pp_world_size = get_pp_group().world_size
665
+ assert (pp_world_size == pipeline_model_parallel_size), (
666
"pipeline parallel group already initialized, but of unexpected size: "
253
- f"{get_pipeline_model_parallel_world_size()=} vs. "
667
+ f"{pp_world_size=} vs. "
668
f"{pipeline_model_parallel_size=}")
669
670
671
def model_parallel_is_initialized():
672
"""Check if tensor and pipeline parallel groups are initialized."""
259
- return (_TP_DEVICE_GROUP is not None and _PP_DEVICE_GROUP is not None)
260
-
261
-
262
-def get_cpu_world_group():
263
- """Get the CPU world group."""
264
- assert _CPU_WORLD_GROUP is not None, ("CPU world group is not initialized")
265
- return _CPU_WORLD_GROUP
266
-
267
-
268
-def get_tensor_model_parallel_group():
269
- """Get the tensor model parallel group the caller rank belongs to."""
270
- assert _TP_DEVICE_GROUP is not None, (
271
- "tensor model parallel group is not initialized")
272
- return _TP_DEVICE_GROUP
273
-
274
-
275
-def get_tensor_model_parallel_cpu_group():
276
- """Get the tensor model parallel cpu group the caller rank belongs to."""
277
- assert _TP_CPU_GROUP is not None, (
278
- "tensor model parallel cpu group is not initialized")
279
- return _TP_CPU_GROUP
280
-
281
-
282
-def get_pipeline_model_parallel_group():
283
- """Get the pipeline model parallel group the caller rank belongs to."""
284
- assert _PP_DEVICE_GROUP is not None, (
285
- "pipeline model parallel group is not initialized")
286
- return _PP_DEVICE_GROUP
287
-
288
-
289
-def get_pipeline_model_parallel_cpu_group():
290
- """Get the pipeline model parallel cpu group the caller rank belongs to."""
291
- assert _PP_CPU_GROUP is not None, (
292
- "pipeline model parallel cpu group is not initialized")
293
- return _PP_CPU_GROUP
673
+ return (_TP is not None and _PP is not None)
674
675
676
def get_tensor_model_parallel_world_size():
677
"""Return world size for the tensor model parallel group."""
298
- return torch.distributed.get_world_size(
299
- group=get_tensor_model_parallel_group())
300
-
301
-
302
-def get_pipeline_model_parallel_world_size():
303
- """Return world size for the pipeline model parallel group."""
304
- return torch.distributed.get_world_size(
305
- group=get_pipeline_model_parallel_group())
678
+ return get_tp_group().world_size
679
680
681
def get_tensor_model_parallel_rank():
682
"""Return my rank for the tensor model parallel group."""
310
- return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
311
-
312
-
313
-def get_pipeline_model_parallel_rank():
314
- """Return my rank for the pipeline model parallel group."""
315
- return torch.distributed.get_rank(
316
- group=get_pipeline_model_parallel_group())
317
-
318
-
319
-def get_tensor_model_parallel_src_rank():
320
- """Calculate the global rank corresponding to the first local rank
321
- in the tensor model parallel group."""
322
- global_rank = torch.distributed.get_rank()
323
- local_world_size = get_tensor_model_parallel_world_size()
324
- return (global_rank // local_world_size) * local_world_size
325
-
326
-
327
-def get_pipeline_model_parallel_first_rank():
328
- """Return the global rank of the first process in the pipeline for the
329
- current tensor parallel group"""
330
- assert _PP_GLOBAL_RANKS is not None, (
331
- "Pipeline parallel group is not initialized")
332
- return _PP_GLOBAL_RANKS[0]
333
-
334
-
335
-def get_pipeline_model_parallel_last_rank():
336
- """Return the global rank of the last process in the pipeline for the
337
- current tensor parallel group"""
338
- assert _PP_GLOBAL_RANKS is not None, (
339
- "Pipeline parallel group is not initialized")
340
- last_rank_local = get_pipeline_model_parallel_world_size() - 1
341
- return _PP_GLOBAL_RANKS[last_rank_local]
342
-
343
-
344
-def get_pipeline_model_parallel_next_rank():
345
- """Return the global rank that follows the caller in the pipeline"""
346
- assert _PP_GLOBAL_RANKS is not None, (
347
- "Pipeline parallel group is not initialized")
348
- rank_in_pipeline = get_pipeline_model_parallel_rank()
349
- world_size = get_pipeline_model_parallel_world_size()
350
- return _PP_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
351
-
352
-
353
-def get_pipeline_model_parallel_prev_rank():
354
- """Return the global rank that precedes the caller in the pipeline"""
355
- assert _PP_GLOBAL_RANKS is not None, (
356
- "Pipeline parallel group is not initialized")
357
- rank_in_pipeline = get_pipeline_model_parallel_rank()
358
- world_size = get_pipeline_model_parallel_world_size()
359
- return _PP_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
683
+ return get_tp_group().rank_in_group
684
685
686
def destroy_model_parallel():
687
"""Set the groups to none and destroy them."""
364
- global _TP_DEVICE_GROUP
365
- if _TP_DEVICE_GROUP:
366
- torch.distributed.destroy_process_group(_TP_DEVICE_GROUP)
367
- _TP_DEVICE_GROUP = None
368
- global _TP_CPU_GROUP
369
- if _TP_CPU_GROUP:
370
- torch.distributed.destroy_process_group(_TP_CPU_GROUP)
371
- _TP_CPU_GROUP = None
372
- global _TP_PYNCCL_COMMUNICATOR
373
- _TP_PYNCCL_COMMUNICATOR = None
374
-
375
- global _PP_DEVICE_GROUP
376
- if _PP_DEVICE_GROUP:
377
- torch.distributed.destroy_process_group(_PP_DEVICE_GROUP)
378
- _PP_DEVICE_GROUP = None
379
- global _PP_GLOBAL_RANKS
380
- _PP_GLOBAL_RANKS = None
688
+ global _TP
689
+ if _TP:
690
+ _TP.destroy()
691
+ _TP = None
692
+
693
+ global _PP
694
+ if _PP:
695
+ _PP.destroy()
696
+ _PP = None
697
+
698
+
699
+def destroy_distributed_environment():
700
+ global _WORLD
701
+ if _WORLD:
702
+ _WORLD.destroy()
703
+ _WORLD = None
704
+ if torch.distributed.is_initialized():
705
+ torch.distributed.destroy_process_group()
706
707
708
def is_in_the_same_node(pg: ProcessGroup):
vllm/worker/model_runner.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
ModelConfig, ParallelConfig, SchedulerConfig,
14
VisionLanguageConfig)
15
from vllm.distributed import broadcast_tensor_dict
16
-from vllm.distributed.communication_op import graph_capture
16
+from vllm.distributed.parallel_state import graph_capture
17
from vllm.logger import init_logger
18
from vllm.lora.layers import LoRAMapping
19
from vllm.lora.request import LoRARequest