@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5672a112c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Multiprocessing Pipeline Parallel support (#6130)

Co-authored-by: Murali Andoorveedu <muralidhar.andoorveedu@centml.ai>

Nick Hill committed Jul 18, 2024 at 19:15 UTC b5672a112c0e6c550f2dfa75c6cbe940a3f933bc
9 files changed +152 -99
.buildkite/test-pipeline.yaml
+3 -3
@@ -54,7 +54,7 @@ steps:
54 - label: Core Test
55 mirror_hardwares: [amd]
56 fast_check: true
57 - commands:
57 + commands:
58 - pytest -v -s core
59 - pytest -v -s distributed/test_parallel_state.py
60
@@ -73,7 +73,7 @@ steps:
73 commands:
74 - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
75 - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
76 - - pytest -v -s distributed/test_pipeline_parallel.py
76 + - VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py
77 - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
78 - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
79
@@ -123,7 +123,7 @@ steps:
123
124 - label: Engine Test
125 mirror_hardwares: [amd]
126 - commands:
126 + commands:
127 - pytest -v -s engine test_sequence.py test_config.py test_logger.py
128 # OOM in the CI unless we run this separately
129 - pytest -v -s tokenization
tests/distributed/test_pipeline_parallel.py
+23 -9
@@ -1,28 +1,42 @@
1 +import os
2 +
3 import pytest
4
5 from ..utils import compare_two_settings
6
7 +VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
8 +
9
10 @pytest.mark.parametrize(
7 - "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8 - (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9 - (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10 - (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
11 - (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12 - (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
11 + "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME, DIST_BACKEND",
12 + [
13 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
14 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
15 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
16 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
17 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
18 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
19 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
20 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
21 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
22 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
23 ])
14 -def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
24 +def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
25 + DIST_BACKEND):
26 + if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
27 + pytest.skip("Skipping multi-node pipeline parallel test for "
28 + "multiprocessing distributed backend")
29
30 pp_args = [
31 # use half precision for speed and memory savings in CI environment
32 "--dtype",
19 - "bfloat16",
33 + "float16",
34 "--pipeline-parallel-size",
35 str(PP_SIZE),
36 "--tensor-parallel-size",
37 str(TP_SIZE),
38 "--distributed-executor-backend",
25 - "ray",
39 + DIST_BACKEND,
40 ]
41
42 # compare without pipeline parallelism
vllm/config.py
-4
@@ -712,10 +712,6 @@ class ParallelConfig:
712 self.rank = 0
713
714 def _verify_args(self) -> None:
715 - if (self.pipeline_parallel_size > 1
716 - and self.distributed_executor_backend == "mp"):
717 - raise NotImplementedError("Pipeline parallelism is not supported "
718 - "yet with multiprocessing.")
715 if self.distributed_executor_backend not in ("ray", "mp", None):
716 raise ValueError(
717 "Unrecognized distributed executor backend. Supported values "
vllm/executor/executor_base.py
-21
@@ -1,4 +1,3 @@
1 -import asyncio
1 from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
@@ -132,26 +131,6 @@ class ExecutorBase(ABC):
131
132 class ExecutorAsyncBase(ExecutorBase):
133
135 - def __init__(
136 - self,
137 - model_config: ModelConfig,
138 - cache_config: CacheConfig,
139 - parallel_config: ParallelConfig,
140 - scheduler_config: SchedulerConfig,
141 - device_config: DeviceConfig,
142 - load_config: LoadConfig,
143 - lora_config: Optional[LoRAConfig],
144 - multimodal_config: Optional[MultiModalConfig],
145 - speculative_config: Optional[SpeculativeConfig],
146 - prompt_adapter_config: Optional[PromptAdapterConfig],
147 - ) -> None:
148 - self.pp_locks: Optional[List[asyncio.Lock]] = None
149 -
150 - super().__init__(model_config, cache_config, parallel_config,
151 - scheduler_config, device_config, load_config,
152 - lora_config, multimodal_config, speculative_config,
153 - prompt_adapter_config)
154 -
134 @abstractmethod
135 async def execute_model_async(
136 self,
vllm/executor/gpu_executor.py
+29 -15
@@ -12,6 +12,15 @@ from vllm.worker.worker_base import WorkerWrapperBase
12 logger = init_logger(__name__)
13
14
15 +def create_worker(worker_module_name, worker_class_name, **kwargs):
16 + wrapper = WorkerWrapperBase(
17 + worker_module_name=worker_module_name,
18 + worker_class_name=worker_class_name,
19 + )
20 + wrapper.init_worker(**kwargs)
21 + return wrapper.worker
22 +
23 +
24 class GPUExecutor(ExecutorBase):
25
26 def _init_executor(self) -> None:
@@ -51,25 +60,30 @@ class GPUExecutor(ExecutorBase):
60 or (rank % self.parallel_config.tensor_parallel_size == 0),
61 )
62
63 + def _get_create_worker_kwargs(
64 + self,
65 + local_rank: int = 0,
66 + rank: int = 0,
67 + distributed_init_method: Optional[str] = None) -> Dict:
68 + worker_kwargs = self._get_worker_kwargs(local_rank, rank,
69 + distributed_init_method)
70 + if self.speculative_config is None:
71 + worker_kwargs.update(worker_module_name="vllm.worker.worker",
72 + worker_class_name="Worker")
73 + else:
74 + worker_kwargs.update(
75 + worker_module_name="vllm.spec_decode.spec_decode_worker",
76 + worker_class_name="create_spec_worker")
77 + return worker_kwargs
78 +
79 def _create_worker(self,
80 local_rank: int = 0,
81 rank: int = 0,
82 distributed_init_method: Optional[str] = None):
58 -
59 - if self.speculative_config is None:
60 - worker_module_name = "vllm.worker.worker"
61 - worker_class_name = "Worker"
62 - else:
63 - worker_module_name = "vllm.spec_decode.spec_decode_worker"
64 - worker_class_name = "create_spec_worker"
65 -
66 - wrapper = WorkerWrapperBase(
67 - worker_module_name=worker_module_name,
68 - worker_class_name=worker_class_name,
69 - )
70 - wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
71 - distributed_init_method))
72 - return wrapper.worker
83 + return create_worker(**self._get_create_worker_kwargs(
84 + local_rank=local_rank,
85 + rank=rank,
86 + distributed_init_method=distributed_init_method))
87
88 def determine_num_available_blocks(self) -> Tuple[int, int]:
89 """Determine the number of available KV blocks by invoking the
vllm/executor/multiproc_gpu_executor.py
+75 -20
@@ -7,12 +7,13 @@ from typing import Any, List, Optional
7
8 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
9 DistributedGPUExecutor, DistributedGPUExecutorAsync)
10 +from vllm.executor.gpu_executor import create_worker
11 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
12 ResultHandler, WorkerMonitor)
13 from vllm.logger import init_logger
14 from vllm.sequence import ExecuteModelRequest, SamplerOutput
15 from vllm.triton_utils import maybe_set_triton_cache_manager
15 -from vllm.utils import (cuda_device_count_stateless,
16 +from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
17 error_on_invalid_device_count_status,
18 get_distributed_init_method, get_open_port,
19 get_vllm_instance_id, make_async,
@@ -26,7 +27,8 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
27
28 def _init_executor(self) -> None:
29 # Create the parallel GPU workers.
29 - world_size = self.parallel_config.tensor_parallel_size
30 + world_size = self.parallel_config.world_size
31 + tensor_parallel_size = self.parallel_config.tensor_parallel_size
32
33 # Set CUDA_VISIBLE_DEVICES for the driver, inherited by workers
34 if "CUDA_VISIBLE_DEVICES" not in os.environ:
@@ -49,8 +51,15 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
51 if world_size > 1:
52 maybe_set_triton_cache_manager()
53
52 - assert world_size <= cuda_device_count_stateless(), (
53 - "please set tensor_parallel_size to less than max local gpu count")
54 + cuda_device_count = cuda_device_count_stateless()
55 + # Use confusing message for more common TP-only case.
56 + assert tensor_parallel_size <= cuda_device_count, (
57 + f"please set tensor_parallel_size ({tensor_parallel_size}) "
58 + f"to less than max local gpu count ({cuda_device_count})")
59 +
60 + assert world_size <= cuda_device_count, (
61 + f"please ensure that world_size ({world_size}) "
62 + f"is less than than max local gpu count ({cuda_device_count})")
63
64 error_on_invalid_device_count_status()
65
@@ -60,21 +69,35 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
69 distributed_init_method = get_distributed_init_method(
70 "127.0.0.1", get_open_port())
71
72 + self.workers: List[ProcessWorkerWrapper] = []
73 + # This is the list of workers that are rank 0 of each TP group EXCEPT
74 + # global rank 0. These are the workers that will broadcast to the
75 + # rest of the workers.
76 + self.tp_driver_workers: List[ProcessWorkerWrapper] = []
77 + # This is the list of workers that are not drivers and not the first
78 + # worker in a TP group. These are the workers that will be
79 + # broadcasted to.
80 + self.non_driver_workers: List[ProcessWorkerWrapper] = []
81 +
82 if world_size == 1:
64 - self.workers = []
83 self.worker_monitor = None
84 else:
85 result_handler = ResultHandler()
68 - self.workers = [
69 - ProcessWorkerWrapper(
86 + for rank in range(1, world_size):
87 + worker = ProcessWorkerWrapper(
88 result_handler,
89 partial(
72 - self._create_worker,
73 - rank=rank,
74 - local_rank=rank,
75 - distributed_init_method=distributed_init_method,
76 - )) for rank in range(1, world_size)
77 - ]
90 + create_worker,
91 + **self._get_create_worker_kwargs(
92 + rank=rank,
93 + local_rank=rank,
94 + distributed_init_method=distributed_init_method,
95 + )))
96 + self.workers.append(worker)
97 + if rank % tensor_parallel_size == 0:
98 + self.tp_driver_workers.append(worker)
99 + else:
100 + self.non_driver_workers.append(worker)
101
102 self.worker_monitor = WorkerMonitor(self.workers, result_handler)
103 result_handler.start()
@@ -136,16 +159,19 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
159 raise NotImplementedError(
160 "max_concurrent_workers is not supported yet.")
161
139 - # Start the workers first.
162 + if async_run_tensor_parallel_workers_only:
163 + # Run only non-driver workers and just return futures.
164 + return [
165 + worker.execute_method(method, *args, **kwargs)
166 + for worker in self.non_driver_workers
167 + ]
168 +
169 + # Start all remote workers first.
170 worker_outputs = [
171 worker.execute_method(method, *args, **kwargs)
172 for worker in self.workers
173 ]
174
145 - if async_run_tensor_parallel_workers_only:
146 - # Just return futures
147 - return worker_outputs
148 -
175 driver_worker_method = getattr(self.driver_worker, method)
176 driver_worker_output = driver_worker_method(*args, **kwargs)
177
@@ -172,16 +198,45 @@ class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
198 def __init__(self, *args, **kwargs):
199 super().__init__(*args, **kwargs)
200 self.driver_exec_model = make_async(self.driver_worker.execute_model)
201 + self.pp_locks: Optional[List[asyncio.Lock]] = None
202
203 async def _driver_execute_model_async(
204 self,
205 execute_model_req: Optional[ExecuteModelRequest] = None
206 ) -> List[SamplerOutput]:
180 - return await self.driver_exec_model(execute_model_req)
207 + if not self.tp_driver_workers:
208 + return await self.driver_exec_model(execute_model_req)
209 +
210 + if self.pp_locks is None:
211 + # This locks each pipeline parallel stage so multiple virtual
212 + # engines can't execute on the same stage at the same time
213 + # We create the locks here to avoid creating them in the constructor
214 + # which uses a different asyncio loop.
215 + self.pp_locks = [
216 + asyncio.Lock()
217 + for _ in range(self.parallel_config.pipeline_parallel_size)
218 + ]
219 +
220 + tasks = [
221 + asyncio.create_task(
222 + _run_task_with_lock(self.driver_exec_model, self.pp_locks[0],
223 + execute_model_req))
224 + ]
225 + for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
226 + start=1):
227 + tasks.append(
228 + asyncio.create_task(
229 + _run_task_with_lock(driver_worker.execute_method_async,
230 + self.pp_locks[pp_rank],
231 + "execute_model", execute_model_req)))
232 + results = await asyncio.gather(*tasks)
233 +
234 + # Only the last PP stage has the final results.
235 + return results[-1]
236
237 async def _start_worker_execution_loop(self):
238 coros = [
239 worker.execute_method_async("start_worker_execution_loop")
185 - for worker in self.workers
240 + for worker in self.non_driver_workers
241 ]
242 return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+13 -25
@@ -10,7 +10,8 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
10 from vllm.executor.ray_utils import RayWorkerWrapper, ray
11 from vllm.logger import init_logger
12 from vllm.sequence import ExecuteModelRequest, SamplerOutput
13 -from vllm.utils import (error_on_invalid_device_count_status,
13 +from vllm.utils import (_run_task_with_lock,
14 + error_on_invalid_device_count_status,
15 get_distributed_init_method, get_ip, get_open_port,
16 get_vllm_instance_id, make_async)
17
@@ -240,27 +241,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
241 # broadcasted to.
242 self.non_driver_workers: List[RayWorkerWrapper] = []
243
243 - tp_driver_worker_ranks = []
244 - non_driver_worker_ranks = []
245 - for idx, rank in enumerate(worker_ranks[1:]):
244 + # Enforce rank order for correct rank to return final output.
245 + for rank, worker in sorted(zip(worker_ranks[1:], self.workers)):
246 # We need to skip the driver worker, which we
247 # do by skipping worker_ranks[0] which is always 0.
248 if rank % self.parallel_config.tensor_parallel_size == 0:
249 - self.tp_driver_workers.append(self.workers[idx])
250 - tp_driver_worker_ranks.append(rank)
249 + self.tp_driver_workers.append(worker)
250 else:
252 - self.non_driver_workers.append(self.workers[idx])
253 - non_driver_worker_ranks.append(rank)
254 -
255 - # Enforce rank order for correct rank to return final output.
256 - self.tp_driver_workers = [
257 - worker for _, worker in sorted(
258 - zip(tp_driver_worker_ranks, self.tp_driver_workers))
259 - ]
260 - self.non_driver_workers = [
261 - worker for _, worker in sorted(
262 - zip(non_driver_worker_ranks, self.non_driver_workers))
263 - ]
251 + self.non_driver_workers.append(worker)
252
253 def _driver_execute_model(
254 self, execute_model_req: Optional[ExecuteModelRequest]
@@ -413,6 +401,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
401
402 def __init__(self, *args, **kwargs):
403 super().__init__(*args, **kwargs)
404 + self.pp_locks: Optional[List[asyncio.Lock]] = None
405 self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
406 if not self.use_ray_compiled_dag:
407 self.driver_exec_method = make_async(
@@ -437,6 +426,9 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
426 ) -> List[SamplerOutput]:
427 assert not self.use_ray_spmd_worker, (
428 "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
429 + if not self.tp_driver_workers:
430 + return await self.driver_exec_method("execute_model",
431 + execute_model_req)
432 if self.pp_locks is None:
433 # This locks each pipeline parallel stage so multiple virtual
434 # engines can't execute on the same stage at the same time
@@ -447,15 +439,11 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
439 for _ in range(self.parallel_config.pipeline_parallel_size)
440 ]
441
450 - async def _run_task_with_lock(task, lock, *args, **kwargs):
451 - async with lock:
452 - return await task(*args, **kwargs)
453 -
454 - tasks = []
455 - tasks.append(
442 + tasks = [
443 asyncio.create_task(
444 _run_task_with_lock(self.driver_exec_method, self.pp_locks[0],
458 - "execute_model", execute_model_req)))
445 + "execute_model", execute_model_req))
446 + ]
447 for pp_rank, driver_worker in enumerate(self.tp_driver_workers,
448 start=1):
449 tasks.append(
vllm/utils.py
+7
@@ -939,3 +939,10 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
939 processed_args.append(arg)
940
941 return super().parse_args(processed_args, namespace)
942 +
943 +
944 +async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
945 + **kwargs):
946 + """Utility function to run async task in a lock"""
947 + async with lock:
948 + return await task(*args, **kwargs)
vllm/worker/worker_base.py
+2 -2
@@ -274,11 +274,11 @@ class LocalOrDistributedWorkerBase(WorkerBase):
274 num_steps)
275
276 if not get_pp_group().is_last_rank:
277 + # output is IntermediateTensors
278 get_pp_group().send_tensor_dict(output.tensors)
279 return [None]
280
280 - # Worker only supports single-step execution. Wrap the output in a
281 - # list to conform to interface.
281 + # output is List[SamplerOutput]
282 return output
283
284 def _execute_model_spmd(