A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] separate distributed_init from worker (#3904)
youkaichao committed
Apr 9, 2024 at 01:49 UTC
6d592eb430a37a7f8f5f9beb2dbc014bf3aa76bc
4 files changed
+85
-58
vllm/model_executor/parallel_utils/parallel_state.py
+60
-3
@@ -4,6 +4,7 @@
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
"""Tensor and pipeline parallel groups."""
6
import contextlib
7
+from typing import Optional
8
9
import torch
10
@@ -14,14 +15,59 @@ _TENSOR_MODEL_PARALLEL_GROUP = None
15
# Pipeline model parallel group that the current rank belongs to.
16
_PIPELINE_MODEL_PARALLEL_GROUP = None
17
18
+# when people blindly call `torch.distributed.all_reduce` etc,
19
+# it will use this group. It is initialized with the `backend`
20
+# parameter of `init_distributed_environment` below.
21
+# Essentially, this is `torch.distributed.group.WORLD`.
22
+# We leave a line here to note that this is device-specific.
23
+# Note that this variable is not safe to use, because when users
24
+# call `init_distributed_environment` first, and then destroy
25
+# the process group themselves, this variable will keep a reference to the
26
+# destroyed process group, which is not useful.
27
+_DEVICE_WORLD_GROUP = None
28
+
29
+# duing `init_distributed_environment`, we will also initialize a
30
+# group with `gloo` backend, to allow direct coordination between
31
+# processes through the CPU.
32
+_CPU_WORLD_GROUP = None
33
+
34
+# In summary, after calling `init_distributed_environment`, we will
35
+# always have two groups: one for device-specific (and is the default)
36
+# and one for CPU. All processes will be part of both groups.
37
+
38
# A list of global ranks for each pipeline group to ease calculation of the
39
# source rank when broadcasting from the first or last pipeline stage.
40
_PIPELINE_GLOBAL_RANKS = None
41
42
43
+def init_distributed_environment(
44
+ world_size: int,
45
+ rank: int,
46
+ distributed_init_method: Optional[str] = None,
47
+ local_rank: int = -1,
48
+ backend: str = "nccl",
49
+):
50
+ if not torch.distributed.is_initialized():
51
+ assert distributed_init_method is not None, (
52
+ "distributed_init_method must be provided when initializing "
53
+ "distributed environment")
54
+ # this backend is used for WORLD
55
+ torch.distributed.init_process_group(
56
+ backend=backend,
57
+ init_method=distributed_init_method,
58
+ world_size=world_size,
59
+ rank=rank)
60
+ global _DEVICE_WORLD_GROUP, _CPU_WORLD_GROUP
61
+ _DEVICE_WORLD_GROUP = torch.distributed.group.WORLD
62
+ ranks = list(range(torch.distributed.get_world_size()))
63
+ _CPU_WORLD_GROUP = torch.distributed.new_group(ranks=ranks,
64
+ backend="gloo")
65
+
66
+
67
def initialize_model_parallel(
68
tensor_model_parallel_size: int = 1,
69
pipeline_model_parallel_size: int = 1,
70
+ backend: Optional[str] = None,
71
) -> None:
72
"""
73
Initialize model parallel groups.
@@ -48,6 +94,8 @@ def initialize_model_parallel(
94
# Get world size and rank. Ensure some consistencies.
95
assert torch.distributed.is_initialized()
96
world_size: int = torch.distributed.get_world_size()
97
+ # get the backend of _DEVICE_WORLD_GROUP
98
+ backend = backend or torch.distributed.get_backend()
99
100
if (world_size !=
101
tensor_model_parallel_size * pipeline_model_parallel_size):
@@ -69,7 +117,7 @@ def initialize_model_parallel(
117
for i in range(num_tensor_model_parallel_groups):
118
ranks = range(i * tensor_model_parallel_size,
119
(i + 1) * tensor_model_parallel_size)
72
- group = torch.distributed.new_group(ranks)
120
+ group = torch.distributed.new_group(ranks, backend=backend)
121
if rank in ranks:
122
_TENSOR_MODEL_PARALLEL_GROUP = group
123
@@ -80,7 +128,7 @@ def initialize_model_parallel(
128
"pipeline model parallel group is already initialized")
129
for i in range(num_pipeline_model_parallel_groups):
130
ranks = range(i, world_size, num_pipeline_model_parallel_groups)
83
- group = torch.distributed.new_group(ranks)
131
+ group = torch.distributed.new_group(ranks, backend=backend)
132
if rank in ranks:
133
_PIPELINE_MODEL_PARALLEL_GROUP = group
134
_PIPELINE_GLOBAL_RANKS = ranks
@@ -89,14 +137,17 @@ def initialize_model_parallel(
137
def ensure_model_parallel_initialized(
138
tensor_model_parallel_size: int,
139
pipeline_model_parallel_size: int,
140
+ backend: Optional[str] = None,
141
) -> None:
142
"""Helper to initialize model parallel groups if they are not initialized,
143
or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
144
values if the model parallel groups are initialized.
145
"""
146
+ # get the backend of _DEVICE_WORLD_GROUP
147
+ backend = backend or torch.distributed.get_backend()
148
if not model_parallel_is_initialized():
149
initialize_model_parallel(tensor_model_parallel_size,
99
- pipeline_model_parallel_size)
150
+ pipeline_model_parallel_size, backend)
151
return
152
153
assert (
@@ -117,6 +168,12 @@ def model_parallel_is_initialized():
168
and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
169
170
171
+def get_cpu_world_group():
172
+ """Get the CPU world group."""
173
+ assert _CPU_WORLD_GROUP is not None, ("CPU world group is not initialized")
174
+ return _CPU_WORLD_GROUP
175
+
176
+
177
def get_tensor_model_parallel_group():
178
"""Get the tensor model parallel group the caller rank belongs to."""
179
assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
vllm/test_utils.py
+6
-7
@@ -1,8 +1,8 @@
1
import ray
2
3
-from vllm.config import ParallelConfig
3
+from vllm.model_executor.parallel_utils.parallel_state import (
4
+ ensure_model_parallel_initialized, init_distributed_environment)
5
from vllm.utils import get_open_port
5
-from vllm.worker.worker import init_distributed_environment
6
7
8
def init_test_distributed_environment(
@@ -12,15 +12,14 @@ def init_test_distributed_environment(
12
distributed_init_port: str,
13
local_rank: int = -1,
14
) -> None:
15
- parallel_config = ParallelConfig(pipeline_parallel_size,
16
- tensor_parallel_size,
17
- worker_use_ray=True)
15
distributed_init_method = f"tcp://localhost:{distributed_init_port}"
16
init_distributed_environment(
20
- parallel_config,
21
- rank,
17
+ world_size=pipeline_parallel_size * tensor_parallel_size,
18
+ rank=rank,
19
distributed_init_method=distributed_init_method,
20
local_rank=local_rank)
21
+ ensure_model_parallel_initialized(tensor_parallel_size,
22
+ pipeline_parallel_size)
23
24
25
def multi_process_tensor_parallel(
vllm/worker/cpu_worker.py
+7
-21
@@ -13,7 +13,7 @@ from vllm.model_executor.model_loader import get_model
13
from vllm.model_executor.parallel_utils.communication_op import (
14
broadcast_tensor_dict)
15
from vllm.model_executor.parallel_utils.parallel_state import (
16
- ensure_model_parallel_initialized)
16
+ ensure_model_parallel_initialized, init_distributed_environment)
17
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
from vllm.worker.model_runner import ModelRunner
@@ -251,26 +251,12 @@ class CPUWorker:
251
parallel_config = self.parallel_config
252
rank = self.rank
253
distributed_init_method = self.distributed_init_method
254
-
255
- if torch.distributed.is_initialized():
256
- torch_world_size = torch.distributed.get_world_size()
257
- if torch_world_size != parallel_config.world_size:
258
- raise RuntimeError(
259
- "torch.distributed is already initialized but the torch "
260
- "world size does not match parallel_config.world_size "
261
- f"({torch_world_size} vs. {parallel_config.world_size}).")
262
- elif not distributed_init_method:
263
- raise ValueError(
264
- "distributed_init_method must be set if torch.distributed "
265
- "is not already initialized")
266
- else:
267
- backend = "gloo"
268
- torch.distributed.init_process_group(
269
- backend=backend,
270
- world_size=parallel_config.world_size,
271
- rank=rank,
272
- init_method=distributed_init_method,
273
- )
254
+ init_distributed_environment(
255
+ world_size=parallel_config.world_size,
256
+ rank=rank,
257
+ distributed_init_method=distributed_init_method,
258
+ backend="gloo",
259
+ )
260
261
# A small all_reduce for warmup.
262
torch.distributed.all_reduce(torch.zeros(1).cpu())
vllm/worker/worker.py
+12
-27
@@ -15,7 +15,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
15
broadcast_tensor_dict)
16
from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
17
from vllm.model_executor.parallel_utils.parallel_state import (
18
- ensure_model_parallel_initialized)
18
+ ensure_model_parallel_initialized, init_distributed_environment)
19
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.model_runner import ModelRunner
@@ -97,9 +97,9 @@ class Worker:
97
raise RuntimeError(
98
f"Not support device type: {self.device_config.device}")
99
# Initialize the distributed environment.
100
- init_distributed_environment(self.parallel_config, self.rank,
101
- self.distributed_init_method,
102
- self.local_rank)
100
+ init_worker_distributed_environment(self.parallel_config, self.rank,
101
+ self.distributed_init_method,
102
+ self.local_rank)
103
# Set random seed.
104
set_random_seed(self.model_config.seed)
105
@@ -248,31 +248,15 @@ class Worker:
248
self.parallel_config)
249
250
251
-def init_distributed_environment(
251
+def init_worker_distributed_environment(
252
parallel_config: ParallelConfig,
253
rank: int,
254
distributed_init_method: Optional[str] = None,
255
local_rank: int = -1,
256
) -> None:
257
"""Initialize the distributed environment."""
258
- if torch.distributed.is_initialized():
259
- torch_world_size = torch.distributed.get_world_size()
260
- if torch_world_size != parallel_config.world_size:
261
- raise RuntimeError(
262
- "torch.distributed is already initialized but the torch world "
263
- "size does not match parallel_config.world_size "
264
- f"({torch_world_size} vs. {parallel_config.world_size}).")
265
- elif not distributed_init_method:
266
- raise ValueError(
267
- "distributed_init_method must be set if torch.distributed "
268
- "is not already initialized")
269
- else:
270
- torch.distributed.init_process_group(
271
- backend="nccl",
272
- world_size=parallel_config.world_size,
273
- rank=rank,
274
- init_method=distributed_init_method,
275
- )
258
+ init_distributed_environment(parallel_config.world_size, rank,
259
+ distributed_init_method, local_rank)
260
261
if pynccl_utils.is_initialized():
262
pynccl_world_size = pynccl_utils.get_world_size()
@@ -291,10 +275,6 @@ def init_distributed_environment(
275
init_method=distributed_init_method,
276
)
277
294
- # A small all_reduce for warmup.
295
- torch.distributed.all_reduce(torch.zeros(1).cuda())
296
- if pynccl_utils.is_initialized():
297
- pynccl_utils.all_reduce(torch.zeros(1).cuda())
278
ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
279
parallel_config.pipeline_parallel_size)
280
@@ -302,6 +282,11 @@ def init_distributed_environment(
282
if not parallel_config.disable_custom_all_reduce:
283
init_custom_ar()
284
285
+ # A small all_reduce for warmup.
286
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
287
+ if pynccl_utils.is_initialized():
288
+ pynccl_utils.all_reduce(torch.zeros(1).cuda())
289
+
290
291
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
292
# Check if the GPU supports the dtype.