@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d592eb430
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] separate distributed_init from worker (#3904)

youkaichao committed Apr 9, 2024 at 01:49 UTC 6d592eb430a37a7f8f5f9beb2dbc014bf3aa76bc
4 files changed +85 -58
vllm/model_executor/parallel_utils/parallel_state.py
+60 -3
@@ -4,6 +4,7 @@
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 """Tensor and pipeline parallel groups."""
6 import contextlib
7 +from typing import Optional
8
9 import torch
10
@@ -14,14 +15,59 @@ _TENSOR_MODEL_PARALLEL_GROUP = None
15 # Pipeline model parallel group that the current rank belongs to.
16 _PIPELINE_MODEL_PARALLEL_GROUP = None
17
18 +# when people blindly call `torch.distributed.all_reduce` etc,
19 +# it will use this group. It is initialized with the `backend`
20 +# parameter of `init_distributed_environment` below.
21 +# Essentially, this is `torch.distributed.group.WORLD`.
22 +# We leave a line here to note that this is device-specific.
23 +# Note that this variable is not safe to use, because when users
24 +# call `init_distributed_environment` first, and then destroy
25 +# the process group themselves, this variable will keep a reference to the
26 +# destroyed process group, which is not useful.
27 +_DEVICE_WORLD_GROUP = None
28 +
29 +# duing `init_distributed_environment`, we will also initialize a
30 +# group with `gloo` backend, to allow direct coordination between
31 +# processes through the CPU.
32 +_CPU_WORLD_GROUP = None
33 +
34 +# In summary, after calling `init_distributed_environment`, we will
35 +# always have two groups: one for device-specific (and is the default)
36 +# and one for CPU. All processes will be part of both groups.
37 +
38 # A list of global ranks for each pipeline group to ease calculation of the
39 # source rank when broadcasting from the first or last pipeline stage.
40 _PIPELINE_GLOBAL_RANKS = None
41
42
43 +def init_distributed_environment(
44 + world_size: int,
45 + rank: int,
46 + distributed_init_method: Optional[str] = None,
47 + local_rank: int = -1,
48 + backend: str = "nccl",
49 +):
50 + if not torch.distributed.is_initialized():
51 + assert distributed_init_method is not None, (
52 + "distributed_init_method must be provided when initializing "
53 + "distributed environment")
54 + # this backend is used for WORLD
55 + torch.distributed.init_process_group(
56 + backend=backend,
57 + init_method=distributed_init_method,
58 + world_size=world_size,
59 + rank=rank)
60 + global _DEVICE_WORLD_GROUP, _CPU_WORLD_GROUP
61 + _DEVICE_WORLD_GROUP = torch.distributed.group.WORLD
62 + ranks = list(range(torch.distributed.get_world_size()))
63 + _CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
64 + backend="gloo")
65 +
66 +
67 def initialize_model_parallel(
68 tensor_model_parallel_size: int = 1,
69 pipeline_model_parallel_size: int = 1,
70 + backend: Optional[str] = None,
71 ) -> None:
72 """
73 Initialize model parallel groups.
@@ -48,6 +94,8 @@ def initialize_model_parallel(
94 # Get world size and rank. Ensure some consistencies.
95 assert torch.distributed.is_initialized()
96 world_size: int = torch.distributed.get_world_size()
97 + # get the backend of _DEVICE_WORLD_GROUP
98 + backend = backend or torch.distributed.get_backend()
99
100 if (world_size !=
101 tensor_model_parallel_size * pipeline_model_parallel_size):
@@ -69,7 +117,7 @@ def initialize_model_parallel(
117 for i in range(num_tensor_model_parallel_groups):
118 ranks = range(i * tensor_model_parallel_size,
119 (i + 1) * tensor_model_parallel_size)
72 - group = torch.distributed.new_group(ranks)
120 + group = torch.distributed.new_group(ranks, backend=backend)
121 if rank in ranks:
122 _TENSOR_MODEL_PARALLEL_GROUP = group
123
@@ -80,7 +128,7 @@ def initialize_model_parallel(
128 "pipeline model parallel group is already initialized")
129 for i in range(num_pipeline_model_parallel_groups):
130 ranks = range(i, world_size, num_pipeline_model_parallel_groups)
83 - group = torch.distributed.new_group(ranks)
131 + group = torch.distributed.new_group(ranks, backend=backend)
132 if rank in ranks:
133 _PIPELINE_MODEL_PARALLEL_GROUP = group
134 _PIPELINE_GLOBAL_RANKS = ranks
@@ -89,14 +137,17 @@ def initialize_model_parallel(
137 def ensure_model_parallel_initialized(
138 tensor_model_parallel_size: int,
139 pipeline_model_parallel_size: int,
140 + backend: Optional[str] = None,
141 ) -> None:
142 """Helper to initialize model parallel groups if they are not initialized,
143 or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
144 values if the model parallel groups are initialized.
145 """
146 + # get the backend of _DEVICE_WORLD_GROUP
147 + backend = backend or torch.distributed.get_backend()
148 if not model_parallel_is_initialized():
149 initialize_model_parallel(tensor_model_parallel_size,
99 - pipeline_model_parallel_size)
150 + pipeline_model_parallel_size, backend)
151 return
152
153 assert (
@@ -117,6 +168,12 @@ def model_parallel_is_initialized():
168 and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
169
170
171 +def get_cpu_world_group():
172 + """Get the CPU world group."""
173 + assert _CPU_WORLD_GROUP is not None, ("CPU world group is not initialized")
174 + return _CPU_WORLD_GROUP
175 +
176 +
177 def get_tensor_model_parallel_group():
178 """Get the tensor model parallel group the caller rank belongs to."""
179 assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
vllm/test_utils.py
+6 -7
@@ -1,8 +1,8 @@
1 import ray
2
3 -from vllm.config import ParallelConfig
3 +from vllm.model_executor.parallel_utils.parallel_state import (
4 + ensure_model_parallel_initialized, init_distributed_environment)
5 from vllm.utils import get_open_port
5 -from vllm.worker.worker import init_distributed_environment
6
7
8 def init_test_distributed_environment(
@@ -12,15 +12,14 @@ def init_test_distributed_environment(
12 distributed_init_port: str,
13 local_rank: int = -1,
14 ) -> None:
15 - parallel_config = ParallelConfig(pipeline_parallel_size,
16 - tensor_parallel_size,
17 - worker_use_ray=True)
15 distributed_init_method = f"tcp://localhost:{distributed_init_port}"
16 init_distributed_environment(
20 - parallel_config,
21 - rank,
17 + world_size=pipeline_parallel_size * tensor_parallel_size,
18 + rank=rank,
19 distributed_init_method=distributed_init_method,
20 local_rank=local_rank)
21 + ensure_model_parallel_initialized(tensor_parallel_size,
22 + pipeline_parallel_size)
23
24
25 def multi_process_tensor_parallel(
vllm/worker/cpu_worker.py
+7 -21
@@ -13,7 +13,7 @@ from vllm.model_executor.model_loader import get_model
13 from vllm.model_executor.parallel_utils.communication_op import (
14 broadcast_tensor_dict)
15 from vllm.model_executor.parallel_utils.parallel_state import (
16 - ensure_model_parallel_initialized)
16 + ensure_model_parallel_initialized, init_distributed_environment)
17 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19 from vllm.worker.model_runner import ModelRunner
@@ -251,26 +251,12 @@ class CPUWorker:
251 parallel_config = self.parallel_config
252 rank = self.rank
253 distributed_init_method = self.distributed_init_method
254 -
255 - if torch.distributed.is_initialized():
256 - torch_world_size = torch.distributed.get_world_size()
257 - if torch_world_size != parallel_config.world_size:
258 - raise RuntimeError(
259 - "torch.distributed is already initialized but the torch "
260 - "world size does not match parallel_config.world_size "
261 - f"({torch_world_size} vs. {parallel_config.world_size}).")
262 - elif not distributed_init_method:
263 - raise ValueError(
264 - "distributed_init_method must be set if torch.distributed "
265 - "is not already initialized")
266 - else:
267 - backend = "gloo"
268 - torch.distributed.init_process_group(
269 - backend=backend,
270 - world_size=parallel_config.world_size,
271 - rank=rank,
272 - init_method=distributed_init_method,
273 - )
254 + init_distributed_environment(
255 + world_size=parallel_config.world_size,
256 + rank=rank,
257 + distributed_init_method=distributed_init_method,
258 + backend="gloo",
259 + )
260
261 # A small all_reduce for warmup.
262 torch.distributed.all_reduce(torch.zeros(1).cpu())
vllm/worker/worker.py
+12 -27
@@ -15,7 +15,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
15 broadcast_tensor_dict)
16 from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
17 from vllm.model_executor.parallel_utils.parallel_state import (
18 - ensure_model_parallel_initialized)
18 + ensure_model_parallel_initialized, init_distributed_environment)
19 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.model_runner import ModelRunner
@@ -97,9 +97,9 @@ class Worker:
97 raise RuntimeError(
98 f"Not support device type: {self.device_config.device}")
99 # Initialize the distributed environment.
100 - init_distributed_environment(self.parallel_config, self.rank,
101 - self.distributed_init_method,
102 - self.local_rank)
100 + init_worker_distributed_environment(self.parallel_config, self.rank,
101 + self.distributed_init_method,
102 + self.local_rank)
103 # Set random seed.
104 set_random_seed(self.model_config.seed)
105
@@ -248,31 +248,15 @@ class Worker:
248 self.parallel_config)
249
250
251 -def init_distributed_environment(
251 +def init_worker_distributed_environment(
252 parallel_config: ParallelConfig,
253 rank: int,
254 distributed_init_method: Optional[str] = None,
255 local_rank: int = -1,
256 ) -> None:
257 """Initialize the distributed environment."""
258 - if torch.distributed.is_initialized():
259 - torch_world_size = torch.distributed.get_world_size()
260 - if torch_world_size != parallel_config.world_size:
261 - raise RuntimeError(
262 - "torch.distributed is already initialized but the torch world "
263 - "size does not match parallel_config.world_size "
264 - f"({torch_world_size} vs. {parallel_config.world_size}).")
265 - elif not distributed_init_method:
266 - raise ValueError(
267 - "distributed_init_method must be set if torch.distributed "
268 - "is not already initialized")
269 - else:
270 - torch.distributed.init_process_group(
271 - backend="nccl",
272 - world_size=parallel_config.world_size,
273 - rank=rank,
274 - init_method=distributed_init_method,
275 - )
258 + init_distributed_environment(parallel_config.world_size, rank,
259 + distributed_init_method, local_rank)
260
261 if pynccl_utils.is_initialized():
262 pynccl_world_size = pynccl_utils.get_world_size()
@@ -291,10 +275,6 @@ def init_distributed_environment(
275 init_method=distributed_init_method,
276 )
277
294 - # A small all_reduce for warmup.
295 - torch.distributed.all_reduce(torch.zeros(1).cuda())
296 - if pynccl_utils.is_initialized():
297 - pynccl_utils.all_reduce(torch.zeros(1).cuda())
278 ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
279 parallel_config.pipeline_parallel_size)
280
@@ -302,6 +282,11 @@ def init_distributed_environment(
282 if not parallel_config.disable_custom_all_reduce:
283 init_custom_ar()
284
285 + # A small all_reduce for warmup.
286 + torch.distributed.all_reduce(torch.zeros(1).cuda())
287 + if pynccl_utils.is_initialized():
288 + pynccl_utils.all_reduce(torch.zeros(1).cuda())
289 +
290
291 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
292 # Check if the GPU supports the dtype.