@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ea3890a5f0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Core][Distributed] code deduplication in tp&pp with coordinator(#5293)

[Core][Distributed] add coordinator to reduce code duplication in tp and pp (#5293)

youkaichao committed Jun 12, 2024 at 17:27 UTC ea3890a5f0314e49d69afca45fe706504cb14029
12 files changed +622 -582
tests/conftest.py
+3 -1
@@ -15,7 +15,8 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
15
16 from vllm import LLM, SamplingParams
17 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
18 -from vllm.distributed import destroy_model_parallel
18 +from vllm.distributed import (destroy_distributed_environment,
19 + destroy_model_parallel)
20 from vllm.inputs import TextPrompt
21 from vllm.logger import init_logger
22 from vllm.multimodal import MultiModalData
@@ -54,6 +55,7 @@ def _read_prompts(filename: str) -> List[str]:
55
56 def cleanup():
57 destroy_model_parallel()
58 + destroy_distributed_environment()
59 with contextlib.suppress(AssertionError):
60 torch.distributed.destroy_process_group()
61 gc.collect()
tests/distributed/test_custom_all_reduce.py
+3 -3
@@ -7,9 +7,9 @@ import torch
7 import torch.distributed as dist
8
9 from vllm.distributed.communication_op import ( # noqa
10 - graph_capture, tensor_model_parallel_all_reduce)
10 + tensor_model_parallel_all_reduce)
11 from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12 - get_tp_ca_communicator)
12 + get_tp_group, graph_capture)
13
14 from ..utils import (init_test_distributed_environment,
15 multi_process_tensor_parallel)
@@ -91,7 +91,7 @@ def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
91 # communicate independently
92 num_communication = rank // tp_size + 1
93 sz = 1024
94 - fa = get_tp_ca_communicator()
94 + fa = get_tp_group().ca_comm
95 inp = torch.ones(sz, dtype=torch.float32, device=device)
96 out = inp
97 for _ in range(num_communication):
tests/distributed/test_pynccl.py
+8 -4
@@ -6,10 +6,11 @@ import torch
6 import torch.distributed
7
8 from vllm.distributed.communication_op import ( # noqa
9 - graph_capture, tensor_model_parallel_all_reduce)
9 + tensor_model_parallel_all_reduce)
10 from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
11 from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
12 from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
13 + get_world_group, graph_capture,
14 init_distributed_environment)
15 from vllm.utils import update_environment_variables
16
@@ -53,7 +54,8 @@ def worker_fn_wrapper(fn):
54
55 @worker_fn_wrapper
56 def worker_fn():
56 - pynccl_comm = PyNcclCommunicator()
57 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
58 + device=get_world_group().device)
59 tensor = torch.ones(16, 1024, 1024,
60 dtype=torch.float32).cuda(pynccl_comm.rank)
61 with pynccl_comm.change_state(enable=True):
@@ -129,7 +131,8 @@ def test_pynccl_multiple_allreduce_with_vllm():
131 def worker_fn_with_cudagraph():
132 with torch.no_grad():
133 graph = torch.cuda.CUDAGraph()
132 - pynccl_comm = PyNcclCommunicator()
134 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
135 + device=get_world_group().device)
136 # run something in the default stream to initialize torch engine
137 a = torch.ones((4, 4), device=f'cuda:{pynccl_comm.rank}')
138 torch.cuda.synchronize()
@@ -154,7 +157,8 @@ def test_pynccl_with_cudagraph():
157
158 @worker_fn_wrapper
159 def send_recv_worker_fn():
157 - pynccl_comm = PyNcclCommunicator()
160 + pynccl_comm = PyNcclCommunicator(get_world_group().cpu_group,
161 + device=get_world_group().device)
162 if pynccl_comm.rank == 0:
163 tensor = torch.ones(16, 1024, 1024,
164 dtype=torch.float32).cuda(pynccl_comm.rank)
tests/lora/conftest.py
+13 -10
@@ -12,7 +12,10 @@ from huggingface_hub import snapshot_download
12
13 import vllm
14 from vllm.config import LoRAConfig
15 -from vllm.distributed import destroy_model_parallel, initialize_model_parallel
15 +from vllm.distributed import (destroy_distributed_environment,
16 + destroy_model_parallel,
17 + init_distributed_environment,
18 + initialize_model_parallel)
19 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 MergedColumnParallelLinear,
21 RowParallelLinear)
@@ -35,6 +38,7 @@ LONG_LORA_INFOS = [{
38
39 def cleanup():
40 destroy_model_parallel()
41 + destroy_distributed_environment()
42 with contextlib.suppress(AssertionError):
43 torch.distributed.destroy_process_group()
44 gc.collect()
@@ -64,15 +68,14 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
68
69 @pytest.fixture
70 def dist_init():
67 - if not torch.distributed.is_initialized():
68 - temp_file = tempfile.mkstemp()[1]
69 - torch.distributed.init_process_group(
70 - backend="nccl",
71 - world_size=1,
72 - rank=0,
73 - init_method=f"file://{temp_file}",
74 - )
75 - torch.distributed.all_reduce(torch.zeros(1).cuda())
71 + temp_file = tempfile.mkstemp()[1]
72 + init_distributed_environment(
73 + world_size=1,
74 + rank=0,
75 + distributed_init_method=f"file://{temp_file}",
76 + local_rank=0,
77 + backend="nccl",
78 + )
79 initialize_model_parallel(1, 1)
80 yield
81 cleanup()
tests/worker/test_model_runner.py
+3 -1
@@ -1,7 +1,8 @@
1 import pytest
2 import torch
3
4 -from vllm.distributed.parallel_state import init_distributed_environment
4 +from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
5 + init_distributed_environment)
6 from vllm.engine.arg_utils import EngineArgs
7 from vllm.model_executor.sampling_metadata import SamplingMetadata
8 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
@@ -292,6 +293,7 @@ def distributed_init():
293 rank=0,
294 distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
295 local_rank=0)
296 + ensure_model_parallel_initialized(1, 1)
297
298
299 @pytest.mark.parametrize("batch_size", list(range(2, 128)))
vllm/attention/backends/pallas.py
+1 -1
@@ -110,7 +110,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
110 raise NotImplementedError("TPU version must be 4 or higher.")
111
112 self.megacore_mode = None
113 - tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
113 + tpu_type = torch_xla.tpu.get_tp_groupu_env()["TYPE"].lower()
114 if not tpu_type.endswith("lite"):
115 if self.num_kv_heads % 2 == 0:
116 self.megacore_mode = "kv_head"
vllm/distributed/communication_op.py
+13 -298
@@ -1,317 +1,32 @@
1 -from collections import namedtuple
2 -from contextlib import contextmanager, nullcontext
3 -from dataclasses import dataclass
4 -from typing import Any, Dict, List, Optional, Tuple, Union
1 +from typing import Any, Dict, Optional, Union
2
3 import torch
7 -from torch.distributed import ProcessGroup
4 +import torch.distributed
5
9 -from .parallel_state import (get_cpu_world_group, get_pp_pynccl_communicator,
10 - get_tensor_model_parallel_group,
11 - get_tensor_model_parallel_rank,
12 - get_tensor_model_parallel_world_size,
13 - get_tp_ca_communicator,
14 - get_tp_pynccl_communicator)
15 -
16 -
17 -@dataclass
18 -class GraphCaptureContext:
19 - stream: torch.cuda.Stream
20 -
21 -
22 -@contextmanager
23 -def graph_capture():
24 - """
25 - `graph_capture` is a context manager which should surround the code that
26 - is capturing the CUDA graph. Its main purpose is to ensure that the
27 - some operations will be run after the graph is captured, before the graph
28 - is replayed. It returns a `GraphCaptureContext` object which contains the
29 - necessary data for the graph capture. Currently, it only contains the
30 - stream that the graph capture is running on. This stream is set to the
31 - current CUDA stream when the context manager is entered and reset to the
32 - default stream when the context manager is exited. This is to ensure that
33 - the graph capture is running on a separate stream from the default stream,
34 - in order to explicitly distinguish the kernels to capture
35 - from other kernels possibly launched on background in the default stream.
36 - """
37 - stream = torch.cuda.Stream()
38 - graph_capture_context = GraphCaptureContext(stream)
39 - ca_comm = get_tp_ca_communicator()
40 - maybe_ca_context = nullcontext() if ca_comm is None else ca_comm.capture()
41 - with torch.cuda.stream(stream), maybe_ca_context:
42 - # In graph mode, we have to be very careful about the collective
43 - # operations. The current status is:
44 - # allreduce \ Mode | Eager | Graph |
45 - # --------------------------------------------
46 - # custom allreduce | enabled | enabled |
47 - # PyNccl | disabled| enabled |
48 - # torch.distributed | enabled | disabled|
49 - #
50 - # Note that custom allreduce will have a runtime check, if the tensor
51 - # size is too large, it will fallback to the next available option.
52 - # In summary: When using CUDA graph, we use
53 - # either custom all-reduce kernel or pynccl. When not using CUDA
54 - # graph, we use either custom all-reduce kernel or PyTorch NCCL.
55 - # We always prioritize using custom all-reduce kernel but fall back
56 - # to PyTorch or pynccl if it is disabled or not supported.
57 - tp_pynccl_comm = get_tp_pynccl_communicator()
58 - pp_pynccl_comm = get_pp_pynccl_communicator()
59 - if not tp_pynccl_comm:
60 - maybe_tp_pynccl_context = nullcontext()
61 - else:
62 - maybe_tp_pynccl_context = tp_pynccl_comm.change_state(
63 - enable=True, stream=torch.cuda.current_stream())
64 - if not pp_pynccl_comm:
65 - maybe_pp_pynccl_context = nullcontext()
66 - else:
67 - maybe_pp_pynccl_context = pp_pynccl_comm.change_state(
68 - enable=True, stream=torch.cuda.current_stream())
69 - with maybe_tp_pynccl_context, maybe_pp_pynccl_context:
70 - yield graph_capture_context
6 +from .parallel_state import get_tp_group
7
8
9 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
74 - """All-reduce the input tensor across model parallel group.
75 -
76 - NOTE: This operation will be applied in-place on the input tensor if
77 - disable_custom_all_reduce is set to True. Otherwise, this operation may or
78 - may not be applied in place depending on whether custom all reduce is
79 - invoked for a particular tensor, which further depends on the tensor size
80 - and GPU topology.
81 -
82 - TLDR: always assume this function modifies its input, but use the return
83 - value as the output.
84 - """
85 - ca_comm = get_tp_ca_communicator()
86 -
87 - # Bypass the function if we are using only 1 GPU.
88 - if get_tensor_model_parallel_world_size() == 1:
89 - return input_
90 - if ca_comm is not None:
91 - out = ca_comm.custom_all_reduce(input_)
92 - if out is not None:
93 - return out
94 - pynccl_comm = get_tp_pynccl_communicator()
95 - if (pynccl_comm is not None and not pynccl_comm.disabled):
96 - pynccl_comm.all_reduce(input_)
97 - else:
98 - torch.distributed.all_reduce(input_,
99 - group=get_tensor_model_parallel_group())
100 - return input_
10 + """All-reduce the input tensor across model parallel group."""
11 + return get_tp_group().all_reduce(input_)
12
13
14 def tensor_model_parallel_all_gather(input_: torch.Tensor,
15 dim: int = -1) -> torch.Tensor:
16 """All-gather the input tensor across model parallel group."""
106 - world_size = get_tensor_model_parallel_world_size()
107 - # Bypass the function if we are using only 1 GPU.
108 - if world_size == 1:
109 - return input_
110 - assert -input_.dim() <= dim < input_.dim(), (
111 - f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
112 - if dim < 0:
113 - # Convert negative dim to positive.
114 - dim += input_.dim()
115 - input_size = input_.size()
116 - # Allocate output tensor.
117 - output_tensor = torch.empty((world_size, ) + input_size,
118 - dtype=input_.dtype,
119 - device=input_.device)
120 - # All-gather.
121 - torch.distributed.all_gather_into_tensor(
122 - output_tensor, input_, group=get_tensor_model_parallel_group())
123 - # Reshape
124 - output_tensor = output_tensor.movedim(0, dim)
125 - output_tensor = output_tensor.reshape(input_size[:dim] +
126 - (world_size * input_size[dim], ) +
127 - input_size[dim + 1:])
128 - return output_tensor
17 + return get_tp_group().all_gather(input_, dim)
18
19
20 def tensor_model_parallel_gather(input_: torch.Tensor,
21 dst: int = 0,
22 dim: int = -1) -> torch.Tensor:
134 - """Gather the input tensor across model parallel group.
135 -
136 - NOTE: We assume that the input tensor is on the same device across
137 - all the ranks.
138 - """
139 - world_size = get_tensor_model_parallel_world_size()
140 - # Bypass the function if we are using only 1 GPU.
141 - if world_size == 1:
142 - return input_
143 - assert -input_.dim() <= dim < input_.dim(), (
144 - f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
145 - if dim < 0:
146 - # Convert negative dim to positive.
147 - dim += input_.dim()
148 - # Allocate output tensor.
149 - if get_tensor_model_parallel_rank() == dst:
150 - gather_list = [torch.empty_like(input_) for _ in range(world_size)]
151 - else:
152 - gather_list = None
153 - # Gather.
154 - torch.distributed.gather(input_,
155 - gather_list,
156 - dst=dst,
157 - group=get_tensor_model_parallel_group())
158 - if get_tensor_model_parallel_rank() == dst:
159 - output_tensor = torch.cat(gather_list, dim=dim)
160 - else:
161 - output_tensor = None
162 - return output_tensor
163 -
164 -
165 -def broadcast(input_: torch.Tensor,
166 - src: int = 0,
167 - group: Optional[ProcessGroup] = None):
168 - """Broadcast the input tensor."""
169 - group = group or torch.distributed.group.WORLD
170 - ranks = torch.distributed.get_process_group_ranks(group)
171 - assert src in ranks, f"Invalid src rank ({src})"
172 -
173 - # Bypass the function if we are using only 1 GPU.
174 - world_size = torch.distributed.get_world_size(group=group)
175 - if world_size == 1:
176 - return input_
177 - # Broadcast.
178 - torch.distributed.broadcast(input_, src=src, group=group)
179 - return input_
23 + """Gather the input tensor across model parallel group."""
24 + return get_tp_group().gather(input_, dst, dim)
25
26
182 -def broadcast_object_list(obj_list: List[Any],
183 - src: int = 0,
184 - group: Optional[ProcessGroup] = None):
185 - """Broadcast the input object list."""
186 - group = group or torch.distributed.group.WORLD
187 - ranks = torch.distributed.get_process_group_ranks(group)
188 - assert src in ranks, f"Invalid src rank ({src})"
189 -
190 - # Bypass the function if we are using only 1 GPU.
191 - world_size = torch.distributed.get_world_size(group=group)
192 - if world_size == 1:
193 - return obj_list
194 - # Broadcast.
195 - torch.distributed.broadcast_object_list(obj_list, src=src, group=group)
196 - return obj_list
197 -
198 -
199 -TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
200 -
201 -
202 -def _split_tensor_dict(
203 - tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
204 -) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
205 - """Split the tensor dictionary into two parts:
206 - 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
207 - by its metadata.
208 - 2. A list of tensors.
209 - """
210 - metadata_list = []
211 - tensor_list = []
212 - for key, value in tensor_dict.items():
213 - if isinstance(value, torch.Tensor):
214 - # Note: we cannot use `value.device` here,
215 - # because it contains not only the device type but also the device
216 - # index (e.g. "cuda:0"). We only need the device type.
217 - # receiving side will set the device index.
218 - device = "cpu" if value.is_cpu else "cuda"
219 - metadata_list.append(
220 - (key, TensorMetadata(device, value.dtype, value.size())))
221 - tensor_list.append(value)
222 - else:
223 - metadata_list.append((key, value))
224 - return metadata_list, tensor_list
225 -
226 -
227 -def broadcast_tensor_dict(
228 - tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
229 - src: int = 0,
230 - group: Optional[ProcessGroup] = None,
231 - metadata_group: Optional[ProcessGroup] = None
232 -) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
233 - """Broadcast the input tensor dictionary.
234 - `group` is used to broadcast the tensors, while `metadata_group` is used
235 - to broadcast the metadata of the dict (e.g. dict structure, tensor sizes,
236 - dtypes).
237 - """
238 - # Bypass the function if we are using only 1 GPU.
239 - if (not torch.distributed.is_initialized()
240 - or torch.distributed.get_world_size(group=group) == 1):
27 +def broadcast_tensor_dict(tensor_dict: Optional[Dict[Any, Union[torch.Tensor,
28 + Any]]] = None,
29 + src: int = 0):
30 + if not torch.distributed.is_initialized():
31 return tensor_dict
242 -
243 - group = group or torch.distributed.group.WORLD
244 - metadata_group = metadata_group or get_cpu_world_group()
245 - ranks = torch.distributed.get_process_group_ranks(group)
246 - assert src in ranks, f"Invalid src rank ({src})"
247 -
248 - rank = torch.distributed.get_rank()
249 - if rank == src:
250 - metadata_list: List[Tuple[Any, Any]] = []
251 - assert isinstance(
252 - tensor_dict,
253 - dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
254 - metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
255 - # `metadata_list` lives in CPU memory.
256 - # `broadcast_object_list` involves serialization and deserialization,
257 - # all happening on CPU. Therefore, we can use the CPU group.
258 - torch.distributed.broadcast_object_list([metadata_list],
259 - src=src,
260 - group=metadata_group)
261 - async_handles = []
262 - for tensor in tensor_list:
263 - if tensor.numel() == 0:
264 - # Skip broadcasting empty tensors.
265 - continue
266 - if tensor.is_cpu:
267 - # use metadata_group for CPU tensors
268 - handle = torch.distributed.broadcast(tensor,
269 - src=src,
270 - group=metadata_group,
271 - async_op=True)
272 - else:
273 - # use group for GPU tensors
274 - handle = torch.distributed.broadcast(tensor,
275 - src=src,
276 - group=group,
277 - async_op=True)
278 - async_handles.append(handle)
279 - for async_handle in async_handles:
280 - async_handle.wait()
281 -
282 - else:
283 - recv_metadata_list = [None]
284 - torch.distributed.broadcast_object_list(recv_metadata_list,
285 - src=src,
286 - group=metadata_group)
287 - assert recv_metadata_list[0] is not None
288 - tensor_dict = {}
289 - async_handles = []
290 - for key, value in recv_metadata_list[0]:
291 - if isinstance(value, TensorMetadata):
292 - tensor = torch.empty(value.size,
293 - dtype=value.dtype,
294 - device=value.device)
295 - if tensor.numel() == 0:
296 - # Skip broadcasting empty tensors.
297 - tensor_dict[key] = tensor
298 - continue
299 - if tensor.is_cpu:
300 - # use metadata_group for CPU tensors
301 - handle = torch.distributed.broadcast(tensor,
302 - src=src,
303 - group=metadata_group,
304 - async_op=True)
305 - else:
306 - # use group for GPU tensors
307 - handle = torch.distributed.broadcast(tensor,
308 - src=src,
309 - group=group,
310 - async_op=True)
311 - async_handles.append(handle)
312 - tensor_dict[key] = tensor
313 - else:
314 - tensor_dict[key] = value
315 - for async_handle in async_handles:
316 - async_handle.wait()
317 - return tensor_dict
32 + return get_tp_group().broadcast_tensor_dict(tensor_dict, src)
vllm/distributed/device_communicators/custom_all_reduce.py
+4 -9
@@ -9,8 +9,7 @@ import vllm.envs as envs
9 from vllm import _custom_ops as ops
10 from vllm.distributed.device_communicators.custom_all_reduce_utils import (
11 gpu_p2p_access_check)
12 -from vllm.distributed.parallel_state import (
13 - get_local_rank, get_tensor_model_parallel_cpu_group, is_in_the_same_node)
12 +from vllm.distributed.parallel_state import is_in_the_same_node
13 from vllm.logger import init_logger
14
15 try:
@@ -86,8 +85,8 @@ class CustomAllreduce:
85
86 # max_size: max supported allreduce size
87 def __init__(self,
89 - group: Optional[ProcessGroup] = None,
90 - device: Optional[Union[int, str, torch.device]] = None,
88 + group: ProcessGroup,
89 + device: Union[int, str, torch.device],
90 max_size=8192 * 1024) -> None:
91 """
92 Args:
@@ -107,7 +106,6 @@ class CustomAllreduce:
106 # e.g. in a non-cuda environment
107 return
108
110 - group = group or get_tensor_model_parallel_cpu_group()
109 self.group = group
110
111 assert dist.get_backend(group) != dist.Backend.NCCL, (
@@ -134,10 +132,7 @@ class CustomAllreduce:
132 world_size, str(CustomAllreduce._SUPPORTED_WORLD_SIZES))
133 return
134
137 - if device is None:
138 - local_rank = get_local_rank()
139 - device = torch.device(f"cuda:{local_rank}")
140 - elif isinstance(device, int):
135 + if isinstance(device, int):
136 device = torch.device(f"cuda:{device}")
137 elif isinstance(device, str):
138 device = torch.device(device)
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+3 -4
@@ -11,7 +11,6 @@ import torch.distributed as dist
11 import torch.multiprocessing as mp
12
13 import vllm.envs as envs
14 -from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
14 from vllm.logger import init_logger
15
16 logger = init_logger(__name__)
@@ -162,7 +161,8 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
161 f"{VLLM_CONFIG_ROOT}/vllm/gpu_p2p_access_cache_for_{cuda_visible_devices}.json"
162 )
163 os.makedirs(os.path.dirname(path), exist_ok=True)
165 - if ((not is_distributed or get_local_rank() == 0)
164 + from vllm.distributed.parallel_state import get_world_group
165 + if ((not is_distributed or get_world_group().local_rank == 0)
166 and (not os.path.exists(path))):
167 # only the local master process (with local_rank == 0) can
168 # enter this block to calculate the cache
@@ -174,8 +174,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
174 with open(path, "w") as f:
175 json.dump(cache, f, indent=4)
176 if is_distributed:
177 - cpu_world_group = get_cpu_world_group()
178 - dist.barrier(cpu_world_group)
177 + get_world_group().barrier()
178 logger.info("reading GPU P2P access cache from %s", path)
179 with open(path, "r") as f:
180 cache = json.load(f)
vllm/distributed/device_communicators/pynccl.py
+3 -8
@@ -9,7 +9,6 @@ from torch.distributed import ProcessGroup, ReduceOp
9 from vllm.distributed.device_communicators.pynccl_wrapper import (
10 NCCLLibrary, buffer_type, cudaStream_t, ncclComm_t, ncclDataTypeEnum,
11 ncclRedOpTypeEnum, ncclUniqueId)
12 -from vllm.distributed.parallel_state import get_cpu_world_group, get_local_rank
12 from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
@@ -19,8 +18,8 @@ class PyNcclCommunicator:
18
19 def __init__(
20 self,
22 - group: Optional[ProcessGroup] = None,
23 - device: Optional[Union[int, str, torch.device]] = None,
21 + group: ProcessGroup,
22 + device: Union[int, str, torch.device],
23 library_path: Optional[str] = None,
24 ):
25 """
@@ -35,7 +34,6 @@ class PyNcclCommunicator:
34 is bind to a unique device.
35 """
36 assert dist.is_initialized()
38 - group = get_cpu_world_group() if group is None else group
37 assert dist.get_backend(group) != dist.Backend.NCCL, (
38 "PyNcclCommunicator should be attached to a non-NCCL group.")
39 self.group = group
@@ -77,10 +75,7 @@ class PyNcclCommunicator:
75 byte_list = tensor.tolist()
76 for i, byte in enumerate(byte_list):
77 self.unique_id.internal[i] = byte
80 - if device is None:
81 - local_rank = get_local_rank()
82 - device = torch.device(f"cuda:{local_rank}")
83 - elif isinstance(device, int):
78 + if isinstance(device, int):
79 device = torch.device(f"cuda:{device}")
80 elif isinstance(device, str):
81 device = torch.device(device)
vllm/distributed/parallel_state.py
+567 -242
@@ -2,81 +2,518 @@
2 # Adapted from
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -"""Tensor and pipeline parallel groups."""
5 +"""vLLM distributed state.
6 +It takes over the control of the distributed environment from PyTorch.
7 +The typical workflow is:
8 +
9 +- call `init_distributed_environment` to initialize the distributed environment.
10 +- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
11 + initialize the model parallel groups.
12 +
13 +- any code dealing with the distributed stuff
14 +
15 +- call `destroy_model_parallel` to destroy the model parallel groups.
16 +- call `destroy_distributed_environment` to destroy the distributed environment.
17 +
18 +If you only need to use the distributed environment without model/pipeline
19 + parallelism, you can skip the model parallel initialization and destruction
20 + steps.
21 +"""
22 import contextlib
23 +from collections import namedtuple
24 +from contextlib import contextmanager, nullcontext
25 +from dataclasses import dataclass
26 from multiprocessing import resource_tracker, shared_memory
8 -from typing import List, Optional
27 +from typing import Any, Dict, List, Optional, Tuple, Union
28
29 import torch
11 -from torch.distributed import ProcessGroup
30 +from torch.distributed import Backend, ProcessGroup
31
32 import vllm.envs as envs
33 from vllm.logger import init_logger
34
16 -logger = init_logger(__name__)
35
18 -_ENABLE_CUSTOM_ALL_REDUCE = True
36 +@dataclass
37 +class GraphCaptureContext:
38 + stream: torch.cuda.Stream
39
20 -# Tensor model parallel group that the current rank belongs to.
21 -_TP_DEVICE_GROUP: Optional[ProcessGroup] = None
22 -_TP_CPU_GROUP: Optional[ProcessGroup] = None
23 -_TP_PYNCCL_COMMUNICATOR = None
24 -_TP_CA_COMMUNICATOR = None
25 -# Pipeline model parallel group that the current rank belongs to.
26 -_PP_DEVICE_GROUP: Optional[ProcessGroup] = None
27 -_PP_CPU_GROUP: Optional[ProcessGroup] = None
28 -_PP_PYNCCL_COMMUNICATOR = None
29 -
30 -# when people blindly call `torch.distributed.all_reduce` etc,
31 -# it will use this group. It is initialized with the `backend`
32 -# parameter of `init_distributed_environment` below.
33 -# Essentially, this is `torch.distributed.group.WORLD`.
34 -# We leave a line here to note that this is device-specific.
35 -# Note that this variable is not safe to use, because when users
36 -# call `init_distributed_environment` first, and then destroy
37 -# the process group themselves, this variable will keep a reference to the
38 -# destroyed process group, which is not useful.
39 -_DEVICE_WORLD_GROUP = None
40 -
41 -# duing `init_distributed_environment`, we will also initialize a
42 -# group with `gloo` backend, to allow direct coordination between
43 -# processes through the CPU.
44 -_CPU_WORLD_GROUP = None
45 -
46 -# In summary, after calling `init_distributed_environment`, we will
47 -# always have two groups: one for device-specific (and is the default)
48 -# and one for CPU. All processes will be part of both groups.
49 -
50 -# A list of global ranks for each pipeline group to ease calculation of the
51 -# source rank when broadcasting from the first or last pipeline stage.
52 -_PP_GLOBAL_RANKS: Optional[List[int]] = None
53 -
54 -_LOCAL_RANK = -1
40
41 +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
42
57 -def set_custom_all_reduce(enable: bool):
58 - global _ENABLE_CUSTOM_ALL_REDUCE
59 - _ENABLE_CUSTOM_ALL_REDUCE = enable
43
44 +def _split_tensor_dict(
45 + tensor_dict: Dict[Any, Union[torch.Tensor, Any]]
46 +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
47 + """Split the tensor dictionary into two parts:
48 + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced
49 + by its metadata.
50 + 2. A list of tensors.
51 + """
52 + metadata_list = []
53 + tensor_list = []
54 + for key, value in tensor_dict.items():
55 + if isinstance(value, torch.Tensor):
56 + # Note: we cannot use `value.device` here,
57 + # because it contains not only the device type but also the device
58 + # index (e.g. "cuda:0"). We only need the device type.
59 + # receiving side will set the device index.
60 + device = "cpu" if value.is_cpu else "cuda"
61 + metadata_list.append(
62 + (key, TensorMetadata(device, value.dtype, value.size())))
63 + tensor_list.append(value)
64 + else:
65 + metadata_list.append((key, value))
66 + return metadata_list, tensor_list
67
62 -def get_pp_pynccl_communicator():
63 - global _PP_PYNCCL_COMMUNICATOR
64 - return _PP_PYNCCL_COMMUNICATOR
68
69 +class GroupCoordinator:
70 + """
71 + PyTorch ProcessGroup wrapper for a group of processes.
72 + PyTorch ProcessGroup is bound to one specific communication backend,
73 + e.g. NCCL, Gloo, MPI, etc.
74 + GroupCoordinator takes charge of all the communication operations among
75 + the processes in the group. It can route the communication to
76 + a specific implementation (e.g. switch allreduce implementation
77 + based on the tensor size and cuda graph mode).
78 + """
79
67 -def get_tp_pynccl_communicator():
68 - global _TP_PYNCCL_COMMUNICATOR
69 - return _TP_PYNCCL_COMMUNICATOR
80 + # available attributes:
81 + rank: int # global rank
82 + ranks: List[int] # global ranks in the group
83 + world_size: int # size of the group
84 + # difference between `local_rank` and `rank_in_group`:
85 + # if we have a group of size 4 across two nodes:
86 + # Process | Node | Rank | Local Rank | Rank in Group
87 + # 0 | 0 | 0 | 0 | 0
88 + # 1 | 0 | 1 | 1 | 1
89 + # 2 | 1 | 2 | 0 | 2
90 + # 3 | 1 | 3 | 1 | 3
91 + local_rank: int # local rank used to assign devices
92 + rank_in_group: int # rank inside the group
93 + cpu_group: ProcessGroup # group for CPU communication
94 + device_group: ProcessGroup # group for device communication
95 + use_pynccl: bool # a hint of whether to use PyNccl
96 + use_custom_allreduce: bool # a hint of whether to use CustomAllreduce
97 + # communicators are only created for world size > 1
98 + pynccl_comm: Optional[Any] # PyNccl communicator
99 + ca_comm: Optional[Any] # Custom allreduce communicator
100 +
101 + def __init__(
102 + self,
103 + group_ranks: List[List[int]],
104 + local_rank: int,
105 + torch_distributed_backend: Union[str, Backend],
106 + use_pynccl: bool,
107 + use_custom_allreduce: bool,
108 + ):
109 +
110 + self.rank = torch.distributed.get_rank()
111 + self.local_rank = local_rank
112 + self.device_group = None
113 + self.cpu_group = None
114 +
115 + for ranks in group_ranks:
116 + device_group = torch.distributed.new_group(
117 + ranks, backend=torch_distributed_backend)
118 + # a group with `gloo` backend, to allow direct coordination between
119 + # processes through the CPU.
120 + cpu_group = torch.distributed.new_group(ranks, backend="gloo")
121 + if self.rank in ranks:
122 + self.ranks = ranks
123 + self.world_size = len(ranks)
124 + self.rank_in_group = ranks.index(self.rank)
125 + self.device_group = device_group
126 + self.cpu_group = cpu_group
127 +
128 + assert self.cpu_group is not None
129 + assert self.device_group is not None
130
131 + if torch.cuda.is_available():
132 + self.device = torch.device(f"cuda:{local_rank}")
133 + else:
134 + self.device = torch.device("cpu")
135
72 -def get_tp_ca_communicator():
73 - global _TP_CA_COMMUNICATOR
74 - return _TP_CA_COMMUNICATOR
136 + self.use_pynccl = use_pynccl
137 + self.use_custom_allreduce = use_custom_allreduce
138 +
139 + # lazy import to avoid documentation build error
140 + from vllm.distributed.device_communicators.custom_all_reduce import (
141 + CustomAllreduce)
142 + from vllm.distributed.device_communicators.pynccl import (
143 + PyNcclCommunicator)
144 +
145 + self.pynccl_comm: Optional[PyNcclCommunicator]
146 + if use_pynccl and self.world_size > 1:
147 + self.pynccl_comm = PyNcclCommunicator(
148 + group=self.cpu_group,
149 + device=self.device,
150 + )
151 + else:
152 + self.pynccl_comm = None
153 +
154 + self.ca_comm: Optional[CustomAllreduce]
155 + if use_custom_allreduce and self.world_size > 1:
156 + # Initialize a custom fast all-reduce implementation.
157 + self.ca_comm = CustomAllreduce(
158 + group=self.cpu_group,
159 + device=self.device,
160 + )
161 + else:
162 + self.ca_comm = None
163 +
164 + @property
165 + def first_rank(self):
166 + """Return the global rank of the first process in the group"""
167 + return self.ranks[0]
168 +
169 + @property
170 + def last_rank(self):
171 + """Return the global rank of the last process in the group"""
172 + return self.ranks[-1]
173 +
174 + @property
175 + def next_rank(self):
176 + """Return the global rank of the process that follows the caller"""
177 + rank_in_group = self.rank_in_group
178 + world_size = self.world_size
179 + return self.ranks[(rank_in_group + 1) % world_size]
180 +
181 + @property
182 + def prev_rank(self):
183 + """Return the global rank of the process that precedes the caller"""
184 + rank_in_group = self.rank_in_group
185 + world_size = self.world_size
186 + return self.ranks[(rank_in_group - 1) % world_size]
187 +
188 + @contextmanager
189 + def graph_capture(
190 + self, graph_capture_context: Optional[GraphCaptureContext] = None):
191 + if graph_capture_context is None:
192 + stream = torch.cuda.Stream()
193 + graph_capture_context = GraphCaptureContext(stream)
194 + else:
195 + stream = graph_capture_context.stream
196 +
197 + ca_comm = self.ca_comm
198 + maybe_ca_context = nullcontext(
199 + ) if ca_comm is None else ca_comm.capture()
200 + with torch.cuda.stream(stream), maybe_ca_context:
201 + # In graph mode, we have to be very careful about the collective
202 + # operations. The current status is:
203 + # allreduce \ Mode | Eager | Graph |
204 + # --------------------------------------------
205 + # custom allreduce | enabled | enabled |
206 + # PyNccl | disabled| enabled |
207 + # torch.distributed | enabled | disabled|
208 + #
209 + # Note that custom allreduce will have a runtime check, if the
210 + # tensor size is too large, it will fallback to the next
211 + # available option.
212 + # In summary: When using CUDA graph, we use
213 + # either custom all-reduce kernel or pynccl. When not using
214 + # CUDA graph, we use either custom all-reduce kernel or
215 + # PyTorch NCCL. We always prioritize using custom all-reduce
216 + # kernel but fall back to PyTorch or pynccl if it is
217 + # disabled or not supported.
218 + pynccl_comm = self.pynccl_comm
219 + maybe_pynccl_context: Any
220 + if not pynccl_comm:
221 + maybe_pynccl_context = nullcontext()
222 + else:
223 + maybe_pynccl_context = pynccl_comm.change_state(
224 + enable=True, stream=torch.cuda.current_stream())
225 + with maybe_pynccl_context:
226 + yield graph_capture_context
227 +
228 + def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
229 + """
230 + NOTE: This operation will be applied in-place or out-of-place.
231 + Always assume this function modifies its input, but use the return
232 + value as the output.
233 + """
234 + ca_comm = self.ca_comm
235 +
236 + # Bypass the function if we are using only 1 GPU.
237 + if self.world_size == 1:
238 + return input_
239 + if ca_comm is not None:
240 + out = ca_comm.custom_all_reduce(input_)
241 + if out is not None:
242 + return out
243 + pynccl_comm = self.pynccl_comm
244 + if (pynccl_comm is not None and not pynccl_comm.disabled):
245 + pynccl_comm.all_reduce(input_)
246 + else:
247 + torch.distributed.all_reduce(input_, group=self.device_group)
248 + return input_
249 +
250 + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
251 + world_size = self.world_size
252 + # Bypass the function if we are using only 1 GPU.
253 + if world_size == 1:
254 + return input_
255 + assert -input_.dim() <= dim < input_.dim(), (
256 + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
257 + if dim < 0:
258 + # Convert negative dim to positive.
259 + dim += input_.dim()
260 + input_size = input_.size()
261 + # Allocate output tensor.
262 + output_tensor = torch.empty((world_size, ) + input_size,
263 + dtype=input_.dtype,
264 + device=input_.device)
265 + # All-gather.
266 + torch.distributed.all_gather_into_tensor(output_tensor,
267 + input_,
268 + group=self.device_group)
269 + # Reshape
270 + output_tensor = output_tensor.movedim(0, dim)
271 + output_tensor = output_tensor.reshape(input_size[:dim] +
272 + (world_size *
273 + input_size[dim], ) +
274 + input_size[dim + 1:])
275 + return output_tensor
276 +
277 + def gather(self,
278 + input_: torch.Tensor,
279 + dst: int = 0,
280 + dim: int = -1) -> torch.Tensor:
281 + """
282 + NOTE: We assume that the input tensor is on the same device across
283 + all the ranks.
284 + NOTE: `dst` is the local rank of the destination rank.
285 + """
286 + world_size = self.world_size
287 + # Bypass the function if we are using only 1 GPU.
288 + if world_size == 1:
289 + return input_
290 + assert -input_.dim() <= dim < input_.dim(), (
291 + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
292 + if dim < 0:
293 + # Convert negative dim to positive.
294 + dim += input_.dim()
295 + # Allocate output tensor.
296 + if self.rank_in_group == dst:
297 + gather_list = [torch.empty_like(input_) for _ in range(world_size)]
298 + else:
299 + gather_list = None
300 + # Gather.
301 + torch.distributed.gather(input_,
302 + gather_list,
303 + dst=self.ranks[dst],
304 + group=self.device_group)
305 + if self.rank_in_group == dst:
306 + output_tensor = torch.cat(gather_list, dim=dim)
307 + else:
308 + output_tensor = None
309 + return output_tensor
310 +
311 + def broadcast(self, input_: torch.Tensor, src: int = 0):
312 + """Broadcast the input tensor.
313 + NOTE: `src` is the local rank of the source rank.
314 + """
315 + assert src < self.world_size, f"Invalid src rank ({src})"
316 +
317 + # Bypass the function if we are using only 1 GPU.
318 + if self.world_size == 1:
319 + return input_
320 + # Broadcast.
321 + torch.distributed.broadcast(input_,
322 + src=self.ranks[src],
323 + group=self.device_group)
324 + return input_
325 +
326 + def broadcast_object_list(self,
327 + obj_list: List[Any],
328 + src: int = 0,
329 + group: Optional[ProcessGroup] = None):
330 + """Broadcast the input object list.
331 + NOTE: `src` is the local rank of the source rank.
332 + """
333 + assert src < self.world_size, f"Invalid src rank ({src})"
334 +
335 + # Bypass the function if we are using only 1 GPU.
336 + if self.world_size == 1:
337 + return obj_list
338 + # Broadcast.
339 + torch.distributed.broadcast_object_list(obj_list,
340 + src=self.ranks[src],
341 + group=self.device_group)
342 + return obj_list
343 +
344 + def broadcast_tensor_dict(
345 + self,
346 + tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
347 + src: int = 0,
348 + group: Optional[ProcessGroup] = None,
349 + metadata_group: Optional[ProcessGroup] = None
350 + ) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
351 + """Broadcast the input tensor dictionary.
352 + NOTE: `src` is the local rank of the source rank.
353 + """
354 + # Bypass the function if we are using only 1 GPU.
355 + if (not torch.distributed.is_initialized() or self.world_size == 1):
356 + return tensor_dict
357 +
358 + group = self.device_group
359 + metadata_group = self.cpu_group
360 + assert src < self.world_size, f"Invalid src rank ({src})"
361 + src = self.ranks[src]
362 +
363 + rank = self.rank
364 + if rank == src:
365 + metadata_list: List[Tuple[Any, Any]] = []
366 + assert isinstance(
367 + tensor_dict,
368 + dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
369 + metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
370 + # `metadata_list` lives in CPU memory.
371 + # `broadcast_object_list` has serialization & deserialization,
372 + # all happening on CPU. Therefore, we can use the CPU group.
373 + torch.distributed.broadcast_object_list([metadata_list],
374 + src=src,
375 + group=metadata_group)
376 + async_handles = []
377 + for tensor in tensor_list:
378 + if tensor.numel() == 0:
379 + # Skip broadcasting empty tensors.
380 + continue
381 + if tensor.is_cpu:
382 + # use metadata_group for CPU tensors
383 + handle = torch.distributed.broadcast(tensor,
384 + src=src,
385 + group=metadata_group,
386 + async_op=True)
387 + else:
388 + # use group for GPU tensors
389 + handle = torch.distributed.broadcast(tensor,
390 + src=src,
391 + group=group,
392 + async_op=True)
393 + async_handles.append(handle)
394 + for async_handle in async_handles:
395 + async_handle.wait()
396 +
397 + else:
398 + recv_metadata_list = [None]
399 + torch.distributed.broadcast_object_list(recv_metadata_list,
400 + src=src,
401 + group=metadata_group)
402 + assert recv_metadata_list[0] is not None
403 + tensor_dict = {}
404 + async_handles = []
405 + for key, value in recv_metadata_list[0]:
406 + if isinstance(value, TensorMetadata):
407 + tensor = torch.empty(value.size,
408 + dtype=value.dtype,
409 + device=value.device)
410 + if tensor.numel() == 0:
411 + # Skip broadcasting empty tensors.
412 + tensor_dict[key] = tensor
413 + continue
414 + if tensor.is_cpu:
415 + # use metadata_group for CPU tensors
416 + handle = torch.distributed.broadcast(
417 + tensor,
418 + src=src,
419 + group=metadata_group,
420 + async_op=True)
421 + else:
422 + # use group for GPU tensors
423 + handle = torch.distributed.broadcast(tensor,
424 + src=src,
425 + group=group,
426 + async_op=True)
427 + async_handles.append(handle)
428 + tensor_dict[key] = tensor
429 + else:
430 + tensor_dict[key] = value
431 + for async_handle in async_handles:
432 + async_handle.wait()
433 + return tensor_dict
434 +
435 + def barrier(self):
436 + """Barrier synchronization among the group.
437 + NOTE: don't use `device_group` here! `barrier` in NCCL is
438 + terrible because it is internally a broadcast operation with
439 + secretly created GPU tensors. It is easy to mess up the current
440 + device. Use the CPU group instead.
441 + """
442 + torch.distributed.barrier(group=self.cpu_group)
443 +
444 + def destroy(self):
445 + if self.device_group is not None:
446 + torch.distributed.destroy_process_group(self.device_group)
447 + self.device_group = None
448 + if self.cpu_group is not None:
449 + torch.distributed.destroy_process_group(self.cpu_group)
450 + self.cpu_group = None
451 + if self.pynccl_comm is not None:
452 + self.pynccl_comm = None
453 + if self.ca_comm is not None:
454 + self.ca_comm = None
455 +
456 +
457 +_WORLD: Optional[GroupCoordinator] = None
458 +
459 +
460 +def get_world_group() -> GroupCoordinator:
461 + assert _WORLD is not None, ("world group is not initialized")
462 + return _WORLD
463 +
464 +
465 +_TP: Optional[GroupCoordinator] = None
466 +
467 +
468 +def get_tp_group() -> GroupCoordinator:
469 + assert _TP is not None, ("tensor model parallel group is not initialized")
470 + return _TP
471 +
472 +
473 +# kept for backward compatibility
474 +get_tensor_model_parallel_group = get_tp_group
475 +
476 +_PP: Optional[GroupCoordinator] = None
477 +
478 +
479 +def get_pp_group() -> GroupCoordinator:
480 + assert _PP is not None, (
481 + "pipeline model parallel group is not initialized")
482 + return _PP
483
484
77 -def get_local_rank():
78 - global _LOCAL_RANK
79 - return _LOCAL_RANK
485 +# kept for backward compatibility
486 +get_pipeline_model_parallel_group = get_pp_group
487 +
488 +
489 +@contextmanager
490 +def graph_capture():
491 + """
492 + `graph_capture` is a context manager which should surround the code that
493 + is capturing the CUDA graph. Its main purpose is to ensure that the
494 + some operations will be run after the graph is captured, before the graph
495 + is replayed. It returns a `GraphCaptureContext` object which contains the
496 + necessary data for the graph capture. Currently, it only contains the
497 + stream that the graph capture is running on. This stream is set to the
498 + current CUDA stream when the context manager is entered and reset to the
499 + default stream when the context manager is exited. This is to ensure that
500 + the graph capture is running on a separate stream from the default stream,
501 + in order to explicitly distinguish the kernels to capture
502 + from other kernels possibly launched on background in the default stream.
503 + """
504 + with get_tp_group().graph_capture() as context, get_pp_group(
505 + ).graph_capture(context):
506 + yield context
507 +
508 +
509 +logger = init_logger(__name__)
510 +
511 +_ENABLE_CUSTOM_ALL_REDUCE = True
512 +
513 +
514 +def set_custom_all_reduce(enable: bool):
515 + global _ENABLE_CUSTOM_ALL_REDUCE
516 + _ENABLE_CUSTOM_ALL_REDUCE = enable
517
518
519 def init_distributed_environment(
@@ -100,31 +537,29 @@ def init_distributed_environment(
537 init_method=distributed_init_method,
538 world_size=world_size,
539 rank=rank)
103 - global _DEVICE_WORLD_GROUP, _CPU_WORLD_GROUP
104 - _DEVICE_WORLD_GROUP = torch.distributed.group.WORLD
540 + # set the local rank
541 + # local_rank is not available in torch ProcessGroup,
542 + # see https://github.com/pytorch/pytorch/issues/122816
543 + if local_rank == -1:
544 + # local rank not set, this usually happens in single-node
545 + # setting, where we can use rank as local rank
546 + if distributed_init_method == "env://":
547 + local_rank = envs.LOCAL_RANK
548 + else:
549 + local_rank = rank
550 + global _WORLD
551 + if _WORLD is None:
552 ranks = list(range(torch.distributed.get_world_size()))
106 - _CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
107 - backend="gloo")
108 - # set the local rank
109 - # local_rank is not available in torch ProcessGroup,
110 - # see https://github.com/pytorch/pytorch/issues/122816
111 - if local_rank == -1:
112 - # local rank not set, this usually happens in single-node
113 - # setting, where we can use rank as local rank
114 - if distributed_init_method == "env://":
115 - local_rank = envs.LOCAL_RANK
116 - else:
117 - local_rank = rank
118 - global _LOCAL_RANK
119 - _LOCAL_RANK = local_rank
120 - # A small all_reduce for warmup.
121 - data = torch.zeros(1)
122 - if torch.cuda.is_available():
123 - data = data.to(device=f"cuda:{local_rank}")
124 - torch.distributed.all_reduce(data)
125 - if torch.cuda.is_available():
126 - torch.cuda.synchronize()
127 - del data
553 + _WORLD = GroupCoordinator(
554 + group_ranks=[ranks],
555 + local_rank=local_rank,
556 + torch_distributed_backend=backend,
557 + use_pynccl=False,
558 + use_custom_allreduce=False,
559 + )
560 + else:
561 + assert _WORLD.world_size == torch.distributed.get_world_size(), (
562 + "world group already initialized with a different world size")
563
564
565 def initialize_model_parallel(
@@ -157,8 +592,8 @@ def initialize_model_parallel(
592 # Get world size and rank. Ensure some consistencies.
593 assert torch.distributed.is_initialized()
594 world_size: int = torch.distributed.get_world_size()
160 - # get the backend of _DEVICE_WORLD_GROUP
161 - backend = backend or torch.distributed.get_backend()
595 + backend = backend or torch.distributed.get_backend(
596 + get_world_group().device_group)
597
598 if (world_size !=
599 tensor_model_parallel_size * pipeline_model_parallel_size):
@@ -167,63 +602,42 @@ def initialize_model_parallel(
602 f"tensor_model_parallel_size ({tensor_model_parallel_size}) x "
603 f"pipeline_model_parallel_size ({pipeline_model_parallel_size})")
604
605 + # Build the tensor model-parallel groups.
606 num_tensor_model_parallel_groups: int = (world_size //
607 tensor_model_parallel_size)
172 - num_pipeline_model_parallel_groups: int = (world_size //
173 - pipeline_model_parallel_size)
174 - rank = torch.distributed.get_rank()
175 -
176 - # Build the tensor model-parallel groups.
177 - global _TP_DEVICE_GROUP, _TP_CPU_GROUP
178 - global _TP_PYNCCL_COMMUNICATOR, _TP_CA_COMMUNICATOR
179 - assert _TP_DEVICE_GROUP is None, (
180 - "tensor model parallel group is already initialized")
608 + global _TP
609 + assert _TP is None, ("tensor model parallel group is already initialized")
610 + group_ranks = []
611 for i in range(num_tensor_model_parallel_groups):
612 ranks = list(
613 range(i * tensor_model_parallel_size,
614 (i + 1) * tensor_model_parallel_size))
185 - group = torch.distributed.new_group(ranks, backend=backend)
186 - cpu_group = torch.distributed.new_group(ranks, backend="gloo")
187 - if rank in ranks:
188 - _TP_DEVICE_GROUP = group
189 - _TP_CPU_GROUP = cpu_group
190 -
191 - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
192 - if tensor_model_parallel_size > 1:
193 - _TP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
194 - group=_TP_CPU_GROUP,
195 - device=_LOCAL_RANK,
196 - )
197 -
198 - # Initialize a custom fast all-reduce implementation.
199 - if _ENABLE_CUSTOM_ALL_REDUCE:
200 - from vllm.distributed.device_communicators.custom_all_reduce import (
201 - CustomAllreduce)
202 - _TP_CA_COMMUNICATOR = CustomAllreduce(
203 - group=_TP_CPU_GROUP,
204 - device=_LOCAL_RANK,
205 - )
615 + group_ranks.append(ranks)
616 + _TP = GroupCoordinator(
617 + group_ranks=group_ranks,
618 + local_rank=get_world_group().local_rank,
619 + torch_distributed_backend=backend,
620 + use_pynccl=True,
621 + use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
622 + )
623
624 # Build the pipeline model-parallel groups.
208 - global _PP_DEVICE_GROUP, _PP_CPU_GROUP
209 - global _PP_PYNCCL_COMMUNICATOR
210 - global _PP_GLOBAL_RANKS
211 - assert _PP_DEVICE_GROUP is None, (
625 + num_pipeline_model_parallel_groups: int = (world_size //
626 + pipeline_model_parallel_size)
627 + global _PP
628 + assert _PP is None, (
629 "pipeline model parallel group is already initialized")
630 + group_ranks = []
631 for i in range(num_pipeline_model_parallel_groups):
632 ranks = list(range(i, world_size, num_pipeline_model_parallel_groups))
215 - group = torch.distributed.new_group(ranks, backend=backend)
216 - cpu_group = torch.distributed.new_group(ranks, backend="gloo")
217 - if rank in ranks:
218 - _PP_DEVICE_GROUP = group
219 - _PP_CPU_GROUP = cpu_group
220 - _PP_GLOBAL_RANKS = ranks
221 -
222 - if pipeline_model_parallel_size > 1:
223 - _PP_PYNCCL_COMMUNICATOR = PyNcclCommunicator(
224 - group=_PP_CPU_GROUP,
225 - device=_LOCAL_RANK,
226 - )
633 + group_ranks.append(ranks)
634 + _PP = GroupCoordinator(
635 + group_ranks=group_ranks,
636 + local_rank=get_world_group().local_rank,
637 + torch_distributed_backend=backend,
638 + use_pynccl=True,
639 + use_custom_allreduce=_ENABLE_CUSTOM_ALL_REDUCE,
640 + )
641
642
643 def ensure_model_parallel_initialized(
@@ -235,8 +649,8 @@ def ensure_model_parallel_initialized(
649 or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
650 values if the model parallel groups are initialized.
651 """
238 - # get the backend of _DEVICE_WORLD_GROUP
239 - backend = backend or torch.distributed.get_backend()
652 + backend = backend or torch.distributed.get_backend(
653 + get_world_group().device_group)
654 if not model_parallel_is_initialized():
655 initialize_model_parallel(tensor_model_parallel_size,
656 pipeline_model_parallel_size, backend)
@@ -247,137 +661,48 @@ def ensure_model_parallel_initialized(
661 ), ("tensor parallel group already initialized, but of unexpected size: "
662 f"{get_tensor_model_parallel_world_size()=} vs. "
663 f"{tensor_model_parallel_size=}")
250 - assert (get_pipeline_model_parallel_world_size(
251 - ) == pipeline_model_parallel_size), (
664 + pp_world_size = get_pp_group().world_size
665 + assert (pp_world_size == pipeline_model_parallel_size), (
666 "pipeline parallel group already initialized, but of unexpected size: "
253 - f"{get_pipeline_model_parallel_world_size()=} vs. "
667 + f"{pp_world_size=} vs. "
668 f"{pipeline_model_parallel_size=}")
669
670
671 def model_parallel_is_initialized():
672 """Check if tensor and pipeline parallel groups are initialized."""
259 - return (_TP_DEVICE_GROUP is not None and _PP_DEVICE_GROUP is not None)
260 -
261 -
262 -def get_cpu_world_group():
263 - """Get the CPU world group."""
264 - assert _CPU_WORLD_GROUP is not None, ("CPU world group is not initialized")
265 - return _CPU_WORLD_GROUP
266 -
267 -
268 -def get_tensor_model_parallel_group():
269 - """Get the tensor model parallel group the caller rank belongs to."""
270 - assert _TP_DEVICE_GROUP is not None, (
271 - "tensor model parallel group is not initialized")
272 - return _TP_DEVICE_GROUP
273 -
274 -
275 -def get_tensor_model_parallel_cpu_group():
276 - """Get the tensor model parallel cpu group the caller rank belongs to."""
277 - assert _TP_CPU_GROUP is not None, (
278 - "tensor model parallel cpu group is not initialized")
279 - return _TP_CPU_GROUP
280 -
281 -
282 -def get_pipeline_model_parallel_group():
283 - """Get the pipeline model parallel group the caller rank belongs to."""
284 - assert _PP_DEVICE_GROUP is not None, (
285 - "pipeline model parallel group is not initialized")
286 - return _PP_DEVICE_GROUP
287 -
288 -
289 -def get_pipeline_model_parallel_cpu_group():
290 - """Get the pipeline model parallel cpu group the caller rank belongs to."""
291 - assert _PP_CPU_GROUP is not None, (
292 - "pipeline model parallel cpu group is not initialized")
293 - return _PP_CPU_GROUP
673 + return (_TP is not None and _PP is not None)
674
675
676 def get_tensor_model_parallel_world_size():
677 """Return world size for the tensor model parallel group."""
298 - return torch.distributed.get_world_size(
299 - group=get_tensor_model_parallel_group())
300 -
301 -
302 -def get_pipeline_model_parallel_world_size():
303 - """Return world size for the pipeline model parallel group."""
304 - return torch.distributed.get_world_size(
305 - group=get_pipeline_model_parallel_group())
678 + return get_tp_group().world_size
679
680
681 def get_tensor_model_parallel_rank():
682 """Return my rank for the tensor model parallel group."""
310 - return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
311 -
312 -
313 -def get_pipeline_model_parallel_rank():
314 - """Return my rank for the pipeline model parallel group."""
315 - return torch.distributed.get_rank(
316 - group=get_pipeline_model_parallel_group())
317 -
318 -
319 -def get_tensor_model_parallel_src_rank():
320 - """Calculate the global rank corresponding to the first local rank
321 - in the tensor model parallel group."""
322 - global_rank = torch.distributed.get_rank()
323 - local_world_size = get_tensor_model_parallel_world_size()
324 - return (global_rank // local_world_size) * local_world_size
325 -
326 -
327 -def get_pipeline_model_parallel_first_rank():
328 - """Return the global rank of the first process in the pipeline for the
329 - current tensor parallel group"""
330 - assert _PP_GLOBAL_RANKS is not None, (
331 - "Pipeline parallel group is not initialized")
332 - return _PP_GLOBAL_RANKS[0]
333 -
334 -
335 -def get_pipeline_model_parallel_last_rank():
336 - """Return the global rank of the last process in the pipeline for the
337 - current tensor parallel group"""
338 - assert _PP_GLOBAL_RANKS is not None, (
339 - "Pipeline parallel group is not initialized")
340 - last_rank_local = get_pipeline_model_parallel_world_size() - 1
341 - return _PP_GLOBAL_RANKS[last_rank_local]
342 -
343 -
344 -def get_pipeline_model_parallel_next_rank():
345 - """Return the global rank that follows the caller in the pipeline"""
346 - assert _PP_GLOBAL_RANKS is not None, (
347 - "Pipeline parallel group is not initialized")
348 - rank_in_pipeline = get_pipeline_model_parallel_rank()
349 - world_size = get_pipeline_model_parallel_world_size()
350 - return _PP_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
351 -
352 -
353 -def get_pipeline_model_parallel_prev_rank():
354 - """Return the global rank that precedes the caller in the pipeline"""
355 - assert _PP_GLOBAL_RANKS is not None, (
356 - "Pipeline parallel group is not initialized")
357 - rank_in_pipeline = get_pipeline_model_parallel_rank()
358 - world_size = get_pipeline_model_parallel_world_size()
359 - return _PP_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
683 + return get_tp_group().rank_in_group
684
685
686 def destroy_model_parallel():
687 """Set the groups to none and destroy them."""
364 - global _TP_DEVICE_GROUP
365 - if _TP_DEVICE_GROUP:
366 - torch.distributed.destroy_process_group(_TP_DEVICE_GROUP)
367 - _TP_DEVICE_GROUP = None
368 - global _TP_CPU_GROUP
369 - if _TP_CPU_GROUP:
370 - torch.distributed.destroy_process_group(_TP_CPU_GROUP)
371 - _TP_CPU_GROUP = None
372 - global _TP_PYNCCL_COMMUNICATOR
373 - _TP_PYNCCL_COMMUNICATOR = None
374 -
375 - global _PP_DEVICE_GROUP
376 - if _PP_DEVICE_GROUP:
377 - torch.distributed.destroy_process_group(_PP_DEVICE_GROUP)
378 - _PP_DEVICE_GROUP = None
379 - global _PP_GLOBAL_RANKS
380 - _PP_GLOBAL_RANKS = None
688 + global _TP
689 + if _TP:
690 + _TP.destroy()
691 + _TP = None
692 +
693 + global _PP
694 + if _PP:
695 + _PP.destroy()
696 + _PP = None
697 +
698 +
699 +def destroy_distributed_environment():
700 + global _WORLD
701 + if _WORLD:
702 + _WORLD.destroy()
703 + _WORLD = None
704 + if torch.distributed.is_initialized():
705 + torch.distributed.destroy_process_group()
706
707
708 def is_in_the_same_node(pg: ProcessGroup):
vllm/worker/model_runner.py
+1 -1
@@ -13,7 +13,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 ModelConfig, ParallelConfig, SchedulerConfig,
14 VisionLanguageConfig)
15 from vllm.distributed import broadcast_tensor_dict
16 -from vllm.distributed.communication_op import graph_capture
16 +from vllm.distributed.parallel_state import graph_capture
17 from vllm.logger import init_logger
18 from vllm.lora.layers import LoRAMapping
19 from vllm.lora.request import LoRARequest