@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb6d3c264d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Eliminate parallel worker per-step task scheduling overhead (#4894)

Nick Hill committed May 22, 2024 at 14:17 UTC eb6d3c264d0cd8e44dec16bca7947fbe96415ce9
12 files changed +348 -209
vllm/engine/async_llm_engine.py
+9 -1
@@ -234,6 +234,14 @@ class _AsyncLLMEngine(LLMEngine):
234 # Log stats.
235 self.do_log_stats(scheduler_outputs, output)
236
237 + if not request_outputs:
238 + # Stop the execute model loop in parallel workers until there are
239 + # more requests to process. This avoids waiting indefinitely in
240 + # torch.distributed ops which may otherwise timeout, and unblocks
241 + # the RPC thread in the workers so that they can process any other
242 + # queued control plane messages, such as add/remove lora adapters.
243 + await self.model_executor.stop_remote_worker_execution_loop_async()
244 +
245 return request_outputs
246
247 async def encode_request_async(
@@ -687,7 +695,7 @@ class AsyncLLMEngine:
695 multi_modal_data: Multi modal data per request.
696
697 Yields:
690 - The output `EmbeddingRequestOutput` objects from the LLMEngine
698 + The output `EmbeddingRequestOutput` objects from the LLMEngine
699 for the request.
700
701 Details:
vllm/engine/llm_engine.py
+8
@@ -692,6 +692,14 @@ class LLMEngine:
692 # Log stats.
693 self.do_log_stats(scheduler_outputs, output)
694
695 + if not request_outputs:
696 + # Stop the execute model loop in parallel workers until there are
697 + # more requests to process. This avoids waiting indefinitely in
698 + # torch.distributed ops which may otherwise timeout, and unblocks
699 + # the RPC thread in the workers so that they can process any other
700 + # queued control plane messages, such as add/remove lora adapters.
701 + self.model_executor.stop_remote_worker_execution_loop()
702 +
703 return request_outputs
704
705 def do_log_stats(
vllm/executor/distributed_gpu_executor.py
+97 -26
@@ -1,11 +1,12 @@
1 +import asyncio
2 from abc import abstractmethod
2 -from typing import Any, Dict, List, Optional, Set, Tuple
3 +from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4
5 from vllm.executor.executor_base import ExecutorAsyncBase
6 from vllm.executor.gpu_executor import GPUExecutor
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
8 -from vllm.sequence import SamplerOutput
9 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
10
11 logger = init_logger(__name__)
12
@@ -13,6 +14,16 @@ logger = init_logger(__name__)
14 class DistributedGPUExecutor(GPUExecutor):
15 """Abstract superclass of multi-GPU executor implementations."""
16
17 + def __init__(self, *args, **kwargs):
18 + # This is non-None when the execute model loop is running
19 + # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
20 + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
21 + # Updated by implementations that require additional args to be passed
22 + # to the _run_workers execute_model call
23 + self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
24 +
25 + super().__init__(*args, **kwargs)
26 +
27 def determine_num_available_blocks(self) -> Tuple[int, int]:
28 """Determine the number of available KV blocks.
29
@@ -52,13 +63,28 @@ class DistributedGPUExecutor(GPUExecutor):
63 num_gpu_blocks=num_gpu_blocks,
64 num_cpu_blocks=num_cpu_blocks)
65
55 - def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
56 - all_outputs = self._run_workers("execute_model",
57 - driver_args=args,
58 - driver_kwargs=kwargs)
66 + def execute_model(
67 + self,
68 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
69 + if self.parallel_worker_tasks is None:
70 + self.parallel_worker_tasks = self._run_workers(
71 + "start_worker_execution_loop",
72 + async_run_remote_workers_only=True,
73 + **self.extra_execute_model_run_workers_kwargs)
74
75 # Only the driver worker returns the sampling results.
61 - return all_outputs[0]
76 + return self._driver_execute_model(execute_model_req)
77 +
78 + def stop_remote_worker_execution_loop(self) -> None:
79 + if self.parallel_worker_tasks is None:
80 + return
81 +
82 + self._driver_execute_model()
83 + parallel_worker_tasks = self.parallel_worker_tasks
84 + self.parallel_worker_tasks = None
85 + # Ensure that workers exit model loop cleanly
86 + # (this will raise otherwise)
87 + self._wait_for_tasks_completion(parallel_worker_tasks)
88
89 def add_lora(self, lora_request: LoRARequest) -> bool:
90 assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
@@ -88,39 +114,84 @@ class DistributedGPUExecutor(GPUExecutor):
114 pattern=pattern,
115 max_size=max_size)
116
117 + @abstractmethod
118 + def _driver_execute_model(
119 + self,
120 + execute_model_req: Optional[ExecuteModelRequest] = None
121 + ) -> List[SamplerOutput]:
122 + """Run execute_model in the driver worker.
123 +
124 + Passing None will cause the driver to stop the model execution
125 + loop running in each of the remote workers.
126 + """
127 + raise NotImplementedError
128 +
129 @abstractmethod
130 def _run_workers(
131 self,
132 method: str,
133 *args,
96 - driver_args: Optional[Tuple[Any, ...]] = None,
97 - driver_kwargs: Optional[Dict[str, Any]] = None,
134 + async_run_remote_workers_only: bool = False,
135 max_concurrent_workers: Optional[int] = None,
136 **kwargs,
137 ) -> Any:
101 - """Runs the given method on all workers."""
138 + """Runs the given method on all workers.
139 +
140 + Args:
141 + async_run_remote_workers_only: If True the method will be run only
142 + in the remote workers, not the driver worker. It will also be
143 + run asynchronously and return a list of futures rather than
144 + blocking on the results.
145 + """
146 + raise NotImplementedError
147 +
148 + @abstractmethod
149 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
150 + """Wait for futures returned from _run_workers() with
151 + async_run_remote_workers_only to complete."""
152 raise NotImplementedError
153
154
155 class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
156
157 + async def execute_model_async(
158 + self,
159 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
160 + if self.parallel_worker_tasks is None:
161 + # Start model execution loop running in the parallel workers
162 + self.parallel_worker_tasks = asyncio.create_task(
163 + self._start_worker_execution_loop())
164 +
165 + # Only the driver worker returns the sampling results.
166 + return await self._driver_execute_model_async(execute_model_req)
167 +
168 + async def stop_remote_worker_execution_loop_async(self) -> None:
169 + if self.parallel_worker_tasks is None:
170 + return
171 +
172 + await self._driver_execute_model_async()
173 + parallel_worker_tasks = self.parallel_worker_tasks
174 + self.parallel_worker_tasks = None
175 + # Ensure that workers exit model loop cleanly
176 + # (this will raise otherwise)
177 + await parallel_worker_tasks
178 +
179 @abstractmethod
108 - async def _run_workers_async(
180 + async def _driver_execute_model_async(
181 self,
110 - method: str,
111 - *args,
112 - driver_args: Optional[Tuple[Any, ...]] = None,
113 - driver_kwargs: Optional[Dict[str, Any]] = None,
114 - **kwargs,
115 - ) -> Any:
116 - """Runs the given method on all workers."""
117 - raise NotImplementedError
182 + execute_model_req: Optional[ExecuteModelRequest] = None
183 + ) -> List[SamplerOutput]:
184 + """Execute the model asynchronously in the driver worker.
185
119 - async def execute_model_async(self, *args,
120 - **kwargs) -> List[SamplerOutput]:
121 - all_outputs = await self._run_workers_async("execute_model",
122 - driver_args=args,
123 - driver_kwargs=kwargs)
186 + Passing None will cause the driver to stop the model execution
187 + loop running in each of the remote workers.
188 + """
189 + raise NotImplementedError
190
125 - # Only the driver worker returns the sampling results.
126 - return all_outputs[0]
191 + @abstractmethod
192 + async def _start_worker_execution_loop(self):
193 + """Run execution loop on all workers. It guarantees all workers run
194 + the loop or None of them is running the loop. Loop can be stopped by
195 + `stop_remote_worker_execution_loop`.
196 + The API is idempotent (guarantee only 1 loop run at any moment)."""
197 + raise NotImplementedError
vllm/executor/executor_base.py
+8
@@ -74,6 +74,10 @@ class ExecutorBase(ABC):
74 """Executes at least one model step on the given sequences."""
75 raise NotImplementedError
76
77 + def stop_remote_worker_execution_loop(self) -> None:
78 + """Releases parallel workers from model loop."""
79 + return
80 +
81 @abstractmethod
82 def add_lora(self, lora_request: LoRARequest) -> bool:
83 raise NotImplementedError
@@ -109,6 +113,10 @@ class ExecutorAsyncBase(ExecutorBase):
113 """Executes one model step on the given sequences."""
114 raise NotImplementedError
115
116 + async def stop_remote_worker_execution_loop_async(self) -> None:
117 + """Releases parallel workers from model loop."""
118 + return
119 +
120 async def check_health_async(self) -> None:
121 """Checks if the executor is healthy. If not, it should raise an
122 exception."""
vllm/executor/multiproc_gpu_executor.py
+44 -29
@@ -1,13 +1,14 @@
1 import asyncio
2 import os
3 from functools import partial
4 -from typing import Any, Dict, Optional, Tuple
4 +from typing import Any, List, Optional
5
6 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7 DistributedGPUExecutor, DistributedGPUExecutorAsync)
8 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9 ResultHandler, WorkerMonitor)
10 from vllm.logger import init_logger
11 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 get_vllm_instance_id, make_async)
14
@@ -71,16 +72,34 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
72 None)) is not None:
73 worker_monitor.close()
74
75 + def _driver_execute_model(
76 + self,
77 + execute_model_req: Optional[ExecuteModelRequest] = None
78 + ) -> List[SamplerOutput]:
79 + """Run execute_model in the driver worker.
80 +
81 + Passing None will cause the driver to stop the model execution
82 + loop running in each of the remote workers.
83 + """
84 + return self.driver_worker.execute_model(
85 + execute_model_req=execute_model_req)
86 +
87 def _run_workers(
88 self,
89 method: str,
90 *args,
78 - driver_args: Optional[Tuple[Any, ...]] = None,
79 - driver_kwargs: Optional[Dict[str, Any]] = None,
91 + async_run_remote_workers_only: bool = False,
92 max_concurrent_workers: Optional[int] = None,
93 **kwargs,
94 ) -> Any:
83 - """Runs the given method on all workers."""
95 + """Runs the given method on all workers.
96 +
97 + Args:
98 + async_run_remote_workers_only: If True the method will be run only
99 + in the remote workers, not the driver worker. It will also be
100 + run asynchronously and return a list of futures rather than
101 + blocking on the results.
102 + """
103
104 if max_concurrent_workers:
105 raise NotImplementedError(
@@ -92,15 +111,12 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
111 for worker in self.workers
112 ]
113
95 - if driver_args is None:
96 - driver_args = args
97 - if driver_kwargs is None:
98 - driver_kwargs = kwargs
114 + if async_run_remote_workers_only:
115 + # Just return futures
116 + return worker_outputs
117
100 - # Start the driver worker after all the ray workers.
118 driver_worker_method = getattr(self.driver_worker, method)
102 - driver_worker_output = driver_worker_method(*driver_args,
103 - **driver_kwargs)
119 + driver_worker_output = driver_worker_method(*args, **kwargs)
120
121 # Get the results of the workers.
122 return [driver_worker_output
@@ -111,30 +127,29 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
127 if not self.worker_monitor.is_alive():
128 raise RuntimeError("Worker processes are not running")
129
130 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
131 + """Wait for futures returned from _run_workers() with
132 + async_run_remote_workers_only to complete."""
133 + for result in parallel_worker_tasks:
134 + result.get()
135 +
136
137 class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
138 DistributedGPUExecutorAsync):
139
118 - async def _run_workers_async(
119 - self,
120 - method: str,
121 - *args,
122 - driver_args: Optional[Tuple[Any, ...]] = None,
123 - driver_kwargs: Optional[Dict[str, Any]] = None,
124 - **kwargs,
125 - ) -> Any:
126 - """Runs the given method on all workers."""
127 - if driver_args is None:
128 - driver_args = args
129 - if driver_kwargs is None:
130 - driver_kwargs = kwargs
140 + def __init__(self, *args, **kwargs):
141 + super().__init__(*args, **kwargs)
142 + self.driver_exec_model = make_async(self.driver_worker.execute_model)
143
132 - driver_executor = make_async(getattr(self.driver_worker, method))
144 + async def _driver_execute_model_async(
145 + self,
146 + execute_model_req: Optional[ExecuteModelRequest] = None
147 + ) -> List[SamplerOutput]:
148 + return await self.driver_exec_model(execute_model_req)
149
134 - # Run all the workers asynchronously.
135 - coros = [driver_executor(*driver_args, **driver_kwargs)] + [
136 - worker.execute_method_async(method, *args, **kwargs)
150 + async def _start_worker_execution_loop(self):
151 + coros = [
152 + worker.execute_method_async("start_worker_execution_loop")
153 for worker in self.workers
154 ]
139 -
155 return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+43 -43
@@ -42,6 +42,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
42 self.forward_dag = None
43 if USE_RAY_COMPILED_DAG:
44 self.forward_dag = self._compiled_ray_dag()
45 + self.extra_execute_model_run_workers_kwargs[
46 + "use_ray_compiled_dag"] = True
47
48 def _configure_ray_workers_use_nsight(self,
49 ray_remote_kwargs) -> Dict[str, Any]:
@@ -171,23 +173,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
173 max_concurrent_workers=self.parallel_config.
174 max_parallel_loading_workers)
175
174 - def execute_model(
175 - self,
176 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
177 - all_outputs = self._run_workers(
178 - "execute_model",
179 - driver_kwargs={"execute_model_req": execute_model_req},
180 - use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
176 + def _driver_execute_model(
177 + self,
178 + execute_model_req: Optional[ExecuteModelRequest] = None
179 + ) -> List[SamplerOutput]:
180 + """Run execute_model in the driver worker.
181
182 - # Only the driver worker returns the sampling results.
183 - return all_outputs[0]
182 + Passing None will cause the driver to stop the model execution
183 + loop running in each of the remote workers.
184 + """
185 + return self.driver_worker.execute_method("execute_model",
186 + execute_model_req)
187
188 def _run_workers(
189 self,
190 method: str,
191 *args,
189 - driver_args: Optional[Tuple[Any, ...]] = None,
190 - driver_kwargs: Optional[Dict[str, Any]] = None,
192 + async_run_remote_workers_only: bool = False,
193 all_args: Optional[List[Tuple[Any, ...]]] = None,
194 all_kwargs: Optional[List[Dict[str, Any]]] = None,
195 use_dummy_driver: bool = False,
@@ -198,9 +200,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
200 """Runs the given method on all workers. Can be used in the following
201 ways:
202
203 + - async_run_remote_workers_only: If True the method will be run only
204 + in the remote workers, not the driver worker. It will also be
205 + run asynchronously and return a list of futures rather than blocking
206 + on the results.
207 - args/kwargs: All workers share the same args/kwargs
202 - - args/kwargs and driver_args/driver_kwargs: Driver worker has
203 - different args
208 - all_args/all_kwargs: args/kwargs for each worker are specified
209 individually
210 """
@@ -209,11 +213,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
213 raise NotImplementedError(
214 "max_concurrent_workers is not supported yet.")
215
212 - if driver_args is None:
213 - driver_args = args if all_args is None else all_args[0]
214 - if driver_kwargs is None:
215 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
216 -
216 count = len(self.workers)
217 all_worker_args = repeat(args, count) if all_args is None \
218 else islice(all_args, 1, None)
@@ -225,6 +224,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
224 # input. TODO(sang): Fix it.
225 assert self.forward_dag is not None
226 output_channels = self.forward_dag.execute(1)
227 + ray_worker_outputs = []
228 else:
229 # Start the ray workers first.
230 ray_worker_outputs = [
@@ -234,6 +234,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
234 ) in zip(self.workers, all_worker_args, all_worker_kwargs)
235 ]
236
237 + if async_run_remote_workers_only:
238 + # Just return futures
239 + return ray_worker_outputs
240 +
241 + driver_args = args if all_args is None else all_args[0]
242 + driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
243 +
244 # Start the driver worker after all the ray workers.
245 if not use_dummy_driver:
246 driver_worker_output = self.driver_worker.execute_method(
@@ -260,6 +267,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
267
268 return [driver_worker_output] + ray_worker_outputs
269
270 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
271 + """Wait for futures returned from _run_workers() with
272 + async_run_remote_workers_only to complete."""
273 + ray.get(parallel_worker_tasks)
274 +
275 def _compiled_ray_dag(self):
276 import pkg_resources
277 required_version = "2.9"
@@ -303,30 +315,18 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
315
316 def __init__(self, *args, **kwargs):
317 super().__init__(*args, **kwargs)
306 - self.driver_executor = make_async(self.driver_worker.execute_method)
318 + self.driver_exec_method = make_async(self.driver_worker.execute_method)
319
308 - async def _run_workers_async(
320 + async def _driver_execute_model_async(
321 self,
310 - method: str,
311 - *args,
312 - driver_args: Optional[Tuple[Any, ...]] = None,
313 - driver_kwargs: Optional[Dict[str, Any]] = None,
314 - **kwargs,
315 - ) -> Any:
316 - """Runs the given method on all workers."""
317 - coros = []
318 -
319 - if driver_args is None:
320 - driver_args = args
321 - if driver_kwargs is None:
322 - driver_kwargs = kwargs
323 -
324 - coros.append(
325 - self.driver_executor(method, *driver_args, **driver_kwargs))
326 -
327 - # Run the ray workers asynchronously.
328 - for worker in self.workers:
329 - coros.append(worker.execute_method.remote(method, *args, **kwargs))
330 -
331 - all_outputs = await asyncio.gather(*coros)
332 - return all_outputs
322 + execute_model_req: Optional[ExecuteModelRequest] = None
323 + ) -> List[SamplerOutput]:
324 + return await self.driver_exec_method("execute_model",
325 + execute_model_req)
326 +
327 + async def _start_worker_execution_loop(self):
328 + coros = [
329 + worker.execute_method.remote("start_worker_execution_loop")
330 + for worker in self.workers
331 + ]
332 + return await asyncio.gather(*coros)
vllm/spec_decode/ngram_worker.py
+3 -1
@@ -47,7 +47,9 @@ class NGramWorker(LoraNotSupportedWorkerBase):
47 # NGram don't need gpu sampler
48 pass
49
50 - def execute_model(self, execute_model_req: ExecuteModelRequest) -> None:
50 + def execute_model(
51 + self,
52 + execute_model_req: Optional[ExecuteModelRequest] = None) -> None:
53 """NGram doesn't depend on model execution, just pass this function"""
54 pass
55
vllm/spec_decode/spec_decode_worker.py
+61 -64
@@ -231,35 +231,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231 self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
232 num_cpu_blocks=num_cpu_blocks)
233
234 - def _broadcast_control_flow_decision(
235 - self,
236 - execute_model_req: Optional[ExecuteModelRequest] = None,
237 - disable_all_speculation: bool = False) -> Tuple[int, bool]:
238 - """Broadcast how many lookahead slots are scheduled for this step, and
239 - whether all speculation is disabled, to all non-driver workers.
240 -
241 - This is required as if the number of draft model runs changes
242 - dynamically, the non-driver workers won't know unless we perform a
243 - communication to inform then.
244 -
245 - Returns the broadcasted num_lookahead_slots and disable_all_speculation.
246 - """
247 -
248 - if self.rank == self._driver_rank:
249 - assert execute_model_req is not None
250 -
251 - broadcast_dict = dict(
252 - num_lookahead_slots=execute_model_req.num_lookahead_slots,
253 - disable_all_speculation=disable_all_speculation,
254 - )
255 - broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
256 - else:
257 - assert execute_model_req is None
258 - broadcast_dict = broadcast_tensor_dict(src=self._driver_rank)
259 -
260 - return (broadcast_dict["num_lookahead_slots"],
261 - broadcast_dict["disable_all_speculation"])
262 -
234 @torch.inference_mode()
235 def execute_model(
236 self,
@@ -267,39 +238,58 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
238 ) -> List[SamplerOutput]:
239 """Perform speculative decoding on the input batch.
240 """
241 + if self.rank != self._driver_rank:
242 + self._run_non_driver_rank()
243 + return []
244
271 - disable_all_speculation = False
272 - if self.rank == self._driver_rank:
273 - disable_all_speculation = self._should_disable_all_speculation(
274 - execute_model_req)
275 -
276 - (num_lookahead_slots,
277 - disable_all_speculation) = self._broadcast_control_flow_decision(
278 - execute_model_req, disable_all_speculation)
279 -
280 - if self.rank == self._driver_rank:
281 - assert execute_model_req is not None
282 - assert execute_model_req.seq_group_metadata_list is not None, (
283 - "speculative decoding requires non-None seq_group_metadata_list"
284 - )
285 -
286 - self._maybe_disable_speculative_tokens(
287 - disable_all_speculation,
288 - execute_model_req.seq_group_metadata_list)
289 -
290 - # If no spec tokens, call the proposer and scorer workers normally.
291 - # Used for prefill.
292 - if num_lookahead_slots == 0 or len(
293 - execute_model_req.seq_group_metadata_list) == 0:
294 - return self._run_no_spec(execute_model_req,
295 - skip_proposer=disable_all_speculation)
296 -
297 - return self._run_speculative_decoding_step(execute_model_req,
298 - num_lookahead_slots)
299 - else:
300 - self._run_non_driver_rank(num_lookahead_slots)
245 + if execute_model_req is None:
246 + # This signals that there's no more requests to process for now.
247 + # All workers are running infinite loop with broadcast_tensor_dict,
248 + # and it stops the loop when the driver broadcasts an empty input.
249 + # Send an empty input to notify all other workers to stop their
250 + # execution loop.
251 + broadcast_tensor_dict({}, src=0)
252 return []
253
254 + disable_all_speculation = self._should_disable_all_speculation(
255 + execute_model_req)
256 + num_lookahead_slots = execute_model_req.num_lookahead_slots
257 +
258 + # Broadcast how many lookahead slots are scheduled for this step, and
259 + # whether all speculation is disabled, to all non-driver workers.
260 +
261 + # This is required as if the number of draft model runs changes
262 + # dynamically, the non-driver workers won't know unless we perform a
263 + # communication to inform then.
264 + broadcast_dict = dict(
265 + num_lookahead_slots=num_lookahead_slots,
266 + disable_all_speculation=disable_all_speculation,
267 + )
268 + broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
269 +
270 + assert execute_model_req.seq_group_metadata_list is not None, (
271 + "speculative decoding requires non-None seq_group_metadata_list")
272 +
273 + self._maybe_disable_speculative_tokens(
274 + disable_all_speculation, execute_model_req.seq_group_metadata_list)
275 +
276 + # If no spec tokens, call the proposer and scorer workers normally.
277 + # Used for prefill.
278 + if num_lookahead_slots == 0 or len(
279 + execute_model_req.seq_group_metadata_list) == 0:
280 + return self._run_no_spec(execute_model_req,
281 + skip_proposer=disable_all_speculation)
282 +
283 + return self._run_speculative_decoding_step(execute_model_req,
284 + num_lookahead_slots)
285 +
286 + @torch.inference_mode()
287 + def start_worker_execution_loop(self) -> None:
288 + """Execute model loop to perform speculative decoding
289 + in parallel worker."""
290 + while self._run_non_driver_rank():
291 + pass
292 +
293 def _should_disable_all_speculation(
294 self, execute_model_req: ExecuteModelRequest) -> bool:
295 # When the batch size is too large, disable speculative decoding
@@ -346,13 +336,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
336 sampler_output.logprobs = None
337 return [sampler_output]
338
349 - def _run_non_driver_rank(self, num_lookahead_slots: int) -> None:
339 + def _run_non_driver_rank(self) -> bool:
340 """Run proposer and verifier model in non-driver workers. This is used
341 for both speculation cases (num_lookahead_slots>0) and non-speculation
342 cases (e.g. prefill).
343 +
344 + Returns True iff there are remaining sequences to process.
345 """
354 - # In non-driver workers the input is None
355 - execute_model_req = None
346 + assert self.rank != self._driver_rank
347 +
348 + data = broadcast_tensor_dict(src=self._driver_rank)
349 + if not data:
350 + return False
351 + num_lookahead_slots = data["num_lookahead_slots"]
352
353 # Even if num_lookahead_slots is zero, we want to run the proposer model
354 # as it may have KV.
@@ -360,9 +356,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
356 # We run the proposer once per lookahead slot. In the future we should
357 # delegate how many times it runs to the proposer.
358 for _ in range(max(num_lookahead_slots, 1)):
363 - self.proposer_worker.execute_model(execute_model_req)
359 + self.proposer_worker.execute_model()
360
365 - self.scorer_worker.execute_model(execute_model_req)
361 + self.scorer_worker.execute_model()
362 + return True
363
364 @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
365 def _run_speculative_decoding_step(
vllm/worker/embedding_model_runner.py
+3 -2
@@ -47,7 +47,7 @@ class EmbeddingModelRunner(ModelRunner):
47 @torch.inference_mode()
48 def execute_model(
49 self,
50 - seq_group_metadata_list: List[SequenceGroupMetadata],
50 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
51 kv_caches: List[torch.Tensor],
52 ) -> Optional[PoolerOutput]:
53 (input_tokens, input_positions, attn_metadata, pooling_metadata,
@@ -84,10 +84,11 @@ class EmbeddingModelRunner(ModelRunner):
84
85 def prepare_input_tensors(
86 self,
87 - seq_group_metadata_list: List[SequenceGroupMetadata],
87 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
88 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89 Set[LoRARequest], LoRAMapping, torch.Tensor]:
90 if self.is_driver_worker:
91 + assert seq_group_metadata_list is not None
92 # Prepare input tensors.
93 (
94 input_tokens,
vllm/worker/model_runner.py
+3 -2
@@ -609,10 +609,11 @@ class ModelRunner:
609
610 def prepare_input_tensors(
611 self,
612 - seq_group_metadata_list: List[SequenceGroupMetadata],
612 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
613 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
614 Set[LoRARequest], LoRAMapping, torch.Tensor]:
615 if self.is_driver_worker:
616 + assert seq_group_metadata_list is not None
617 # Prepare input tensors.
618 (
619 input_tokens,
@@ -676,7 +677,7 @@ class ModelRunner:
677 @torch.inference_mode()
678 def execute_model(
679 self,
679 - seq_group_metadata_list: List[SequenceGroupMetadata],
680 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
681 kv_caches: List[torch.Tensor],
682 ) -> Optional[SamplerOutput]:
683 (input_tokens, input_positions, attn_metadata, sampling_metadata,
vllm/worker/worker.py
+65 -38
@@ -226,48 +226,42 @@ class Worker(WorkerBase):
226 self,
227 execute_model_req: Optional[ExecuteModelRequest] = None
228 ) -> List[Union[SamplerOutput, PoolerOutput]]:
229 + if not self.is_driver_worker:
230 + self._execute_model_non_driver()
231 + return []
232
233 if execute_model_req is None:
231 - seq_group_metadata_list = None
232 - else:
233 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
234 + # This signals that there's no more requests to process for now.
235 + # All workers are running infinite loop with broadcast_tensor_dict,
236 + # and it stops the loop when the driver broadcasts an empty input.
237 + # Send an empty input to notify all other workers to stop their
238 + # execution loop.
239 + broadcast_tensor_dict({}, src=0)
240 + return []
241
235 - blocks_to_swap_in: torch.Tensor
236 - blocks_to_swap_out: torch.Tensor
237 - blocks_to_copy: torch.Tensor
238 - if self.is_driver_worker:
239 - assert seq_group_metadata_list is not None
240 - assert execute_model_req is not None
241 - num_seq_groups = len(seq_group_metadata_list)
242 - # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
243 - # they contain parameters to launch cudamemcpyasync.
244 - blocks_to_swap_in = torch.tensor(
245 - execute_model_req.blocks_to_swap_in,
246 - device="cpu",
247 - dtype=torch.int64).view(-1, 2)
248 - blocks_to_swap_out = torch.tensor(
249 - execute_model_req.blocks_to_swap_out,
250 - device="cpu",
251 - dtype=torch.int64).view(-1, 2)
252 - # `blocks_to_copy` is a gpu tensor. The src and tgt of
253 - # blocks to copy are in the same device, and `blocks_to_copy`
254 - # can be used directly within cuda kernels.
255 - blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
256 - device=self.device,
242 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
243 + num_seq_groups = len(seq_group_metadata_list)
244 + # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
245 + # they contain parameters to launch cudamemcpyasync.
246 + blocks_to_swap_in = torch.tensor(execute_model_req.blocks_to_swap_in,
247 + device="cpu",
248 + dtype=torch.int64).view(-1, 2)
249 + blocks_to_swap_out = torch.tensor(execute_model_req.blocks_to_swap_out,
250 + device="cpu",
251 dtype=torch.int64).view(-1, 2)
258 - data: Dict[str, Any] = {
259 - "num_seq_groups": num_seq_groups,
260 - "blocks_to_swap_in": blocks_to_swap_in,
261 - "blocks_to_swap_out": blocks_to_swap_out,
262 - "blocks_to_copy": blocks_to_copy,
263 - }
264 - broadcast_tensor_dict(data, src=0)
265 - else:
266 - data = broadcast_tensor_dict(src=0)
267 - num_seq_groups = data["num_seq_groups"]
268 - blocks_to_swap_in = data["blocks_to_swap_in"]
269 - blocks_to_swap_out = data["blocks_to_swap_out"]
270 - blocks_to_copy = data["blocks_to_copy"]
252 + # `blocks_to_copy` is a gpu tensor. The src and tgt of
253 + # blocks to copy are in the same device, and `blocks_to_copy`
254 + # can be used directly within cuda kernels.
255 + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
256 + device=self.device,
257 + dtype=torch.int64).view(-1, 2)
258 + data: Dict[str, Any] = {
259 + "num_seq_groups": num_seq_groups,
260 + "blocks_to_swap_in": blocks_to_swap_in,
261 + "blocks_to_swap_out": blocks_to_swap_out,
262 + "blocks_to_copy": blocks_to_copy,
263 + }
264 + broadcast_tensor_dict(data, src=0)
265
266 self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
267
@@ -282,6 +276,39 @@ class Worker(WorkerBase):
276 # to conform to interface.
277 return [output]
278
279 + @torch.inference_mode()
280 + def start_worker_execution_loop(self) -> None:
281 + """Execute model loop in parallel worker.
282 +
283 + You can stop the loop by executing a driver worker with an empty output.
284 + See `stop_remote_worker_execution_loop` for more details.
285 + """
286 + while self._execute_model_non_driver():
287 + pass
288 +
289 + def _execute_model_non_driver(self) -> bool:
290 + """Execute model in parallel worker.
291 +
292 + Returns True iff there are remaining sequences to process.
293 + """
294 + assert not self.is_driver_worker
295 + data = broadcast_tensor_dict(src=0)
296 + if not data:
297 + return False
298 +
299 + num_seq_groups = data.get("num_seq_groups", 0)
300 + blocks_to_swap_in = data.get("blocks_to_swap_in")
301 + blocks_to_swap_out = data.get("blocks_to_swap_out")
302 + blocks_to_copy = data.get("blocks_to_copy")
303 + self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
304 +
305 + # If there is no input, we don't need to execute the model.
306 + if num_seq_groups == 0:
307 + return False
308 +
309 + self.model_runner.execute_model(None, self.gpu_cache)
310 + return True
311 +
312 def add_lora(self, lora_request: LoRARequest) -> bool:
313 return self.model_runner.add_lora(lora_request)
314
vllm/worker/worker_base.py
+4 -3
@@ -1,7 +1,7 @@
1 import importlib
2 import os
3 from abc import ABC, abstractmethod
4 -from typing import Dict, List, Set, Tuple
4 +from typing import Dict, List, Optional, Set, Tuple
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
@@ -48,8 +48,9 @@ class WorkerBase(ABC):
48
49 @abstractmethod
50 def execute_model(
51 - self,
52 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51 + self,
52 + execute_model_req: Optional[ExecuteModelRequest] = None
53 + ) -> List[SamplerOutput]:
54 """Executes at least one model step on the given sequences, unless no
55 sequences are provided."""
56 raise NotImplementedError