A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Multiprocessing Pipeline Parallel support (#6130)
Co-authored-by: Murali Andoorveedu <muralidhar.andoorveedu@centml.ai>
Nick Hill committed
Jul 18, 2024 at 19:15 UTC
b5672a112c0e6c550f2dfa75c6cbe940a3f933bc
9 files changed
+152
-99
.buildkite/test-pipeline.yaml
+3
-3
@@ -54,7 +54,7 @@ steps:
54
- label: Core Test
55
mirror_hardwares: [amd]
56
fast_check: true
57
- commands:
57
+ commands:
58
- pytest -v -s core
59
- pytest -v -s distributed/test_parallel_state.py
60
@@ -73,7 +73,7 @@ steps:
73
commands:
74
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
75
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
76
- - pytest -v -s distributed/test_pipeline_parallel.py
76
+ - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
77
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
78
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
79
@@ -123,7 +123,7 @@ steps:
123
124
- label: Engine Test
125
mirror_hardwares: [amd]
126
- commands:
126
+ commands:
127
- pytest -v -s engine test_sequence.py test_config.py test_logger.py
128
# OOM in the CI unless we run this separately
129
- pytest -v -s tokenization
tests/distributed/test_pipeline_parallel.py
+23
-9
@@ -1,28 +1,42 @@
1
+import os
2
+
3
import pytest
4
5
from ..utils import compare_two_settings
6
7
+VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
8
+
9
10
@pytest.mark.parametrize(
7
- "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8
- (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9
- (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10
- (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
11
- (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12
- (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
11
+ "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME, DIST_BACKEND",
12
+ [
13
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
14
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
15
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
16
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
17
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
18
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
19
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
20
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
21
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
22
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
23
])
14
-def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
24
+def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
25
+ DIST_BACKEND):
26
+ if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
27
+ pytest.skip("Skipping multi-node pipeline parallel test for "
28
+ "multiprocessing distributed backend")
29
30
pp_args = [
31
# use half precision for speed and memory savings in CI environment
32
"--dtype",
19
- "bfloat16",
33
+ "float16",
34
"--pipeline-parallel-size",
35
str(PP_SIZE),
36
"--tensor-parallel-size",
37
str(TP_SIZE),
38
"--distributed-executor-backend",
25
- "ray",
39
+ DIST_BACKEND,
40
]
41
42
# compare without pipeline parallelism
vllm/config.py
-4
@@ -712,10 +712,6 @@ class ParallelConfig:
712
self.rank = 0
713
714
def _verify_args(self) -> None:
715
- if (self.pipeline_parallel_size > 1
716
- and self.distributed_executor_backend == "mp"):
717
- raise NotImplementedError("Pipeline parallelism is not supported "
718
- "yet with multiprocessing.")
715
if self.distributed_executor_backend not in ("ray", "mp", None):
716
raise ValueError(
717
"Unrecognized distributed executor backend. Supported values "
vllm/executor/executor_base.py
-21
@@ -1,4 +1,3 @@
1
-import asyncio
1
from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
@@ -132,26 +131,6 @@ class ExecutorBase(ABC):
131
132
class ExecutorAsyncBase(ExecutorBase):
133
135
- def __init__(
136
- self,
137
- model_config: ModelConfig,
138
- cache_config: CacheConfig,
139
- parallel_config: ParallelConfig,
140
- scheduler_config: SchedulerConfig,
141
- device_config: DeviceConfig,
142
- load_config: LoadConfig,
143
- lora_config: Optional[LoRAConfig],
144
- multimodal_config: Optional[MultiModalConfig],
145
- speculative_config: Optional[SpeculativeConfig],
146
- prompt_adapter_config: Optional[PromptAdapterConfig],
147
- ) -> None:
148
- self.pp_locks: Optional[List[asyncio.Lock]] = None
149
-
150
- super().__init__(model_config, cache_config, parallel_config,
151
- scheduler_config, device_config, load_config,
152
- lora_config, multimodal_config, speculative_config,
153
- prompt_adapter_config)
154
-
134
@abstractmethod
135
async def execute_model_async(
136
self,
vllm/executor/gpu_executor.py
+29
-15
@@ -12,6 +12,15 @@ from vllm.worker.worker_base import WorkerWrapperBase
12
logger = init_logger(__name__)
13
14
15
+def create_worker(worker_module_name, worker_class_name, **kwargs):
16
+ wrapper = WorkerWrapperBase(
17
+ worker_module_name=worker_module_name,
18
+ worker_class_name=worker_class_name,
19
+ )
20
+ wrapper.init_worker(**kwargs)
21
+ return wrapper.worker
22
+
23
+
24
class GPUExecutor(ExecutorBase):
25
26
def _init_executor(self) -> None:
@@ -51,25 +60,30 @@ class GPUExecutor(ExecutorBase):
60
or (rank % self.parallel_config.tensor_parallel_size == 0),
61
)
62
63
+ def _get_create_worker_kwargs(
64
+ self,
65
+ local_rank: int = 0,
66
+ rank: int = 0,
67
+ distributed_init_method: Optional[str] = None) -> Dict:
68
+ worker_kwargs = self._get_worker_kwargs(local_rank, rank,
69
+ distributed_init_method)
70
+ if self.speculative_config is None:
71
+ worker_kwargs.update(worker_module_name="vllm.worker.worker",
72
+ worker_class_name="Worker")
73
+ else:
74
+ worker_kwargs.update(
75
+ worker_module_name="vllm.spec_decode.spec_decode_worker",
76
+ worker_class_name="create_spec_worker")
77
+ return worker_kwargs
78
+
79
def _create_worker(self,
80
local_rank: int = 0,
81
rank: int = 0,
82
distributed_init_method: Optional[str] = None):
58
-
59
- if self.speculative_config is None:
60
- worker_module_name = "vllm.worker.worker"
61
- worker_class_name = "Worker"
62
- else:
63
- worker_module_name = "vllm.spec_decode.spec_decode_worker"
64
- worker_class_name = "create_spec_worker"
65
-
66
- wrapper = WorkerWrapperBase(
67
- worker_module_name=worker_module_name,
68
- worker_class_name=worker_class_name,
69
- )
70
- wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
71
- distributed_init_method))
72
- return wrapper.worker
83
+ return create_worker(**self._get_create_worker_kwargs(
84
+ local_rank=local_rank,
85
+ rank=rank,
86
+ distributed_init_method=distributed_init_method))
87
88
def determine_num_available_blocks(self) -> Tuple[int, int]:
89
"""Determine the number of available KV blocks by invoking the
vllm/executor/multiproc_gpu_executor.py
+75
-20
@@ -7,12 +7,13 @@ from typing import Any, List, Optional
7
8
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
9
DistributedGPUExecutor, DistributedGPUExecutorAsync)
10
+from vllm.executor.gpu_executor import create_worker
11
from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12
ResultHandler, WorkerMonitor)
13
from vllm.logger import init_logger
14
from vllm.sequence import ExecuteModelRequest, SamplerOutput
15
from vllm.triton_utils import maybe_set_triton_cache_manager
15
-from vllm.utils import (cuda_device_count_stateless,
16
+from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
17
error_on_invalid_device_count_status,
18
get_distributed_init_method, get_open_port,
19
get_vllm_instance_id, make_async,
@@ -26,7 +27,8 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
27
28
def _init_executor(self) -> None:
29
# Create the parallel GPU workers.
29
- world_size = self.parallel_config.tensor_parallel_size
30
+ world_size = self.parallel_config.world_size
31
+ tensor_parallel_size = self.parallel_config.tensor_parallel_size
32
33
# Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
34
if "CUDA_VISIBLE_DEVICES" not in os.environ:
@@ -49,8 +51,15 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
51
if world_size > 1:
52
maybe_set_triton_cache_manager()
53
52
- assert world_size <= cuda_device_count_stateless(), (
53
- "please set tensor_parallel_size to less than max local gpu count")
54
+ cuda_device_count = cuda_device_count_stateless()
55
+ # Use confusing message for more common TP-only case.
56
+ assert tensor_parallel_size <= cuda_device_count, (
57
+ f"please set tensor_parallel_size ({tensor_parallel_size}) "
58
+ f"to less than max local gpu count ({cuda_device_count})")
59
+
60
+ assert world_size <= cuda_device_count, (
61
+ f"please ensure that world_size ({world_size}) "
62
+ f"is less than than max local gpu count ({cuda_device_count})")
63
64
error_on_invalid_device_count_status()
65
@@ -60,21 +69,35 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
69
distributed_init_method = get_distributed_init_method(
70
"127.0.0.1", get_open_port())
71
72
+ self.workers: List[ProcessWorkerWrapper] = []
73
+ # This is the list of workers that are rank 0 of each TP group EXCEPT
74
+ # global rank 0. These are the workers that will broadcast to the
75
+ # rest of the workers.
76
+ self.tp_driver_workers: List[ProcessWorkerWrapper] = []
77
+ # This is the list of workers that are not drivers and not the first
78
+ # worker in a TP group. These are the workers that will be
79
+ # broadcasted to.
80
+ self.non_driver_workers: List[ProcessWorkerWrapper] = []
81
+
82
if world_size == 1:
64
- self.workers = []
83
self.worker_monitor = None
84
else:
85
result_handler = ResultHandler()
68
- self.workers = [
69
- ProcessWorkerWrapper(
86
+ for rank in range(1, world_size):
87
+ worker = ProcessWorkerWrapper(
88
result_handler,
89
partial(
72
- self._create_worker,
73
- rank=rank,
74
- local_rank=rank,
75
- distributed_init_method=distributed_init_method,
76
- )) for rank in range(1, world_size)
77
- ]
90
+ create_worker,
91
+ **self._get_create_worker_kwargs(
92
+ rank=rank,
93
+ local_rank=rank,
94
+ distributed_init_method=distributed_init_method,
95
+ )))
96
+ self.workers.append(worker)
97
+ if rank % tensor_parallel_size == 0:
98
+ self.tp_driver_workers.append(worker)
99
+ else:
100
+ self.non_driver_workers.append(worker)
101
102
self.worker_monitor = WorkerMonitor(self.workers, result_handler)
103
result_handler.start()
@@ -136,16 +159,19 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
159
raise NotImplementedError(
160
"max_concurrent_workers is not supported yet.")
161
139
- # Start the workers first.
162
+ if async_run_tensor_parallel_workers_only:
163
+ # Run only non-driver workers and just return futures.
164
+ return [
165
+ worker.execute_method(method, *args, **kwargs)
166
+ for worker in self.non_driver_workers
167
+ ]
168
+
169
+ # Start all remote workers first.
170
worker_outputs = [
171
worker.execute_method(method, *args, **kwargs)
172
for worker in self.workers
173
]
174
145
- if async_run_tensor_parallel_workers_only:
146
- # Just return futures
147
- return worker_outputs
148
-
175
driver_worker_method = getattr(self.driver_worker, method)
176
driver_worker_output = driver_worker_method(*args, **kwargs)
177
@@ -172,16 +198,45 @@ class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
198
def __init__(self, *args, **kwargs):
199
super().__init__(*args, **kwargs)
200
self.driver_exec_model = make_async(self.driver_worker.execute_model)
201
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
202
203
async def _driver_execute_model_async(
204
self,
205
execute_model_req: Optional[ExecuteModelRequest] = None
206
) -> List[SamplerOutput]:
180
- return await self.driver_exec_model(execute_model_req)
207
+ if not self.tp_driver_workers:
208
+ return await self.driver_exec_model(execute_model_req)
209
+
210
+ if self.pp_locks is None:
211
+ # This locks each pipeline parallel stage so multiple virtual
212
+ # engines can't execute on the same stage at the same time
213
+ # We create the locks here to avoid creating them in the constructor
214
+ # which uses a different asyncio loop.
215
+ self.pp_locks = [
216
+ asyncio.Lock()
217
+ for _ in range(self.parallel_config.pipeline_parallel_size)
218
+ ]
219
+
220
+ tasks = [
221
+ asyncio.create_task(
222
+ _run_task_with_lock(self.driver_exec_model, self.pp_locks[0],
223
+ execute_model_req))
224
+ ]
225
+ for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
226
+ start=1):
227
+ tasks.append(
228
+ asyncio.create_task(
229
+ _run_task_with_lock(driver_worker.execute_method_async,
230
+ self.pp_locks[pp_rank],
231
+ "execute_model", execute_model_req)))
232
+ results = await asyncio.gather(*tasks)
233
+
234
+ # Only the last PP stage has the final results.
235
+ return results[-1]
236
237
async def _start_worker_execution_loop(self):
238
coros = [
239
worker.execute_method_async("start_worker_execution_loop")
185
- for worker in self.workers
240
+ for worker in self.non_driver_workers
241
]
242
return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+13
-25
@@ -10,7 +10,8 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10
from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
from vllm.logger import init_logger
12
from vllm.sequence import ExecuteModelRequest, SamplerOutput
13
-from vllm.utils import (error_on_invalid_device_count_status,
13
+from vllm.utils import (_run_task_with_lock,
14
+ error_on_invalid_device_count_status,
15
get_distributed_init_method, get_ip, get_open_port,
16
get_vllm_instance_id, make_async)
17
@@ -240,27 +241,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
241
# broadcasted to.
242
self.non_driver_workers: List[RayWorkerWrapper] = []
243
243
- tp_driver_worker_ranks = []
244
- non_driver_worker_ranks = []
245
- for idx, rank in enumerate(worker_ranks[1:]):
244
+ # Enforce rank order for correct rank to return final output.
245
+ for rank, worker in sorted(zip(worker_ranks[1:], self.workers)):
246
# We need to skip the driver worker, which we
247
# do by skipping worker_ranks[0] which is always 0.
248
if rank % self.parallel_config.tensor_parallel_size == 0:
249
- self.tp_driver_workers.append(self.workers[idx])
250
- tp_driver_worker_ranks.append(rank)
249
+ self.tp_driver_workers.append(worker)
250
else:
252
- self.non_driver_workers.append(self.workers[idx])
253
- non_driver_worker_ranks.append(rank)
254
-
255
- # Enforce rank order for correct rank to return final output.
256
- self.tp_driver_workers = [
257
- worker for _, worker in sorted(
258
- zip(tp_driver_worker_ranks, self.tp_driver_workers))
259
- ]
260
- self.non_driver_workers = [
261
- worker for _, worker in sorted(
262
- zip(non_driver_worker_ranks, self.non_driver_workers))
263
- ]
251
+ self.non_driver_workers.append(worker)
252
253
def _driver_execute_model(
254
self, execute_model_req: Optional[ExecuteModelRequest]
@@ -413,6 +401,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
401
402
def __init__(self, *args, **kwargs):
403
super().__init__(*args, **kwargs)
404
+ self.pp_locks: Optional[List[asyncio.Lock]] = None
405
self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
406
if not self.use_ray_compiled_dag:
407
self.driver_exec_method = make_async(
@@ -437,6 +426,9 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
426
) -> List[SamplerOutput]:
427
assert not self.use_ray_spmd_worker, (
428
"driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
429
+ if not self.tp_driver_workers:
430
+ return await self.driver_exec_method("execute_model",
431
+ execute_model_req)
432
if self.pp_locks is None:
433
# This locks each pipeline parallel stage so multiple virtual
434
# engines can't execute on the same stage at the same time
@@ -447,15 +439,11 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
439
for _ in range(self.parallel_config.pipeline_parallel_size)
440
]
441
450
- async def _run_task_with_lock(task, lock, *args, **kwargs):
451
- async with lock:
452
- return await task(*args, **kwargs)
453
-
454
- tasks = []
455
- tasks.append(
442
+ tasks = [
443
asyncio.create_task(
444
_run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
458
- "execute_model", execute_model_req)))
445
+ "execute_model", execute_model_req))
446
+ ]
447
for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
448
start=1):
449
tasks.append(
vllm/utils.py
+7
@@ -939,3 +939,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
939
processed_args.append(arg)
940
941
return super().parse_args(processed_args, namespace)
942
+
943
+
944
+async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
945
+ **kwargs):
946
+ """Utility function to run async task in a lock"""
947
+ async with lock:
948
+ return await task(*args, **kwargs)
vllm/worker/worker_base.py
+2
-2
@@ -274,11 +274,11 @@ class LocalOrDistributedWorkerBase(WorkerBase):
274
num_steps)
275
276
if not get_pp_group().is_last_rank:
277
+ # output is IntermediateTensors
278
get_pp_group().send_tensor_dict(output.tensors)
279
return [None]
280
280
- # Worker only supports single-step execution. Wrap the output in a
281
- # list to conform to interface.
281
+ # output is List[SamplerOutput]
282
return output
283
284
def _execute_model_spmd(