A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Eliminate parallel worker per-step task scheduling overhead (#4894)
Nick Hill committed
May 22, 2024 at 14:17 UTC
eb6d3c264d0cd8e44dec16bca7947fbe96415ce9
12 files changed
+348
-209
vllm/engine/async_llm_engine.py
+9
-1
@@ -234,6 +234,14 @@ class _AsyncLLMEngine(LLMEngine):
234
# Log stats.
235
self.do_log_stats(scheduler_outputs, output)
236
237
+ if not request_outputs:
238
+ # Stop the execute model loop in parallel workers until there are
239
+ # more requests to process. This avoids waiting indefinitely in
240
+ # torch.distributed ops which may otherwise timeout, and unblocks
241
+ # the RPC thread in the workers so that they can process any other
242
+ # queued control plane messages, such as add/remove lora adapters.
243
+ await self.model_executor.stop_remote_worker_execution_loop_async()
244
+
245
return request_outputs
246
247
async def encode_request_async(
@@ -687,7 +695,7 @@ class AsyncLLMEngine:
695
multi_modal_data: Multi modal data per request.
696
697
Yields:
690
- The output `EmbeddingRequestOutput` objects from the LLMEngine
698
+ The output `EmbeddingRequestOutput` objects from the LLMEngine
699
for the request.
700
701
Details:
vllm/engine/llm_engine.py
+8
@@ -692,6 +692,14 @@ class LLMEngine:
692
# Log stats.
693
self.do_log_stats(scheduler_outputs, output)
694
695
+ if not request_outputs:
696
+ # Stop the execute model loop in parallel workers until there are
697
+ # more requests to process. This avoids waiting indefinitely in
698
+ # torch.distributed ops which may otherwise timeout, and unblocks
699
+ # the RPC thread in the workers so that they can process any other
700
+ # queued control plane messages, such as add/remove lora adapters.
701
+ self.model_executor.stop_remote_worker_execution_loop()
702
+
703
return request_outputs
704
705
def do_log_stats(
vllm/executor/distributed_gpu_executor.py
+97
-26
@@ -1,11 +1,12 @@
1
+import asyncio
2
from abc import abstractmethod
2
-from typing import Any, Dict, List, Optional, Set, Tuple
3
+from typing import Any, Awaitable, Dict, List, Optional, Set, Tuple, Union
4
5
from vllm.executor.executor_base import ExecutorAsyncBase
6
from vllm.executor.gpu_executor import GPUExecutor
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
8
-from vllm.sequence import SamplerOutput
9
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
10
11
logger = init_logger(__name__)
12
@@ -13,6 +14,16 @@ logger = init_logger(__name__)
14
class DistributedGPUExecutor(GPUExecutor):
15
"""Abstract superclass of multi-GPU executor implementations."""
16
17
+ def __init__(self, *args, **kwargs):
18
+ # This is non-None when the execute model loop is running
19
+ # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
20
+ self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
21
+ # Updated by implementations that require additional args to be passed
22
+ # to the _run_workers execute_model call
23
+ self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
24
+
25
+ super().__init__(*args, **kwargs)
26
+
27
def determine_num_available_blocks(self) -> Tuple[int, int]:
28
"""Determine the number of available KV blocks.
29
@@ -52,13 +63,28 @@ class DistributedGPUExecutor(GPUExecutor):
63
num_gpu_blocks=num_gpu_blocks,
64
num_cpu_blocks=num_cpu_blocks)
65
55
- def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
56
- all_outputs = self._run_workers("execute_model",
57
- driver_args=args,
58
- driver_kwargs=kwargs)
66
+ def execute_model(
67
+ self,
68
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
69
+ if self.parallel_worker_tasks is None:
70
+ self.parallel_worker_tasks = self._run_workers(
71
+ "start_worker_execution_loop",
72
+ async_run_remote_workers_only=True,
73
+ **self.extra_execute_model_run_workers_kwargs)
74
75
# Only the driver worker returns the sampling results.
61
- return all_outputs[0]
76
+ return self._driver_execute_model(execute_model_req)
77
+
78
+ def stop_remote_worker_execution_loop(self) -> None:
79
+ if self.parallel_worker_tasks is None:
80
+ return
81
+
82
+ self._driver_execute_model()
83
+ parallel_worker_tasks = self.parallel_worker_tasks
84
+ self.parallel_worker_tasks = None
85
+ # Ensure that workers exit model loop cleanly
86
+ # (this will raise otherwise)
87
+ self._wait_for_tasks_completion(parallel_worker_tasks)
88
89
def add_lora(self, lora_request: LoRARequest) -> bool:
90
assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
@@ -88,39 +114,84 @@ class DistributedGPUExecutor(GPUExecutor):
114
pattern=pattern,
115
max_size=max_size)
116
117
+ @abstractmethod
118
+ def _driver_execute_model(
119
+ self,
120
+ execute_model_req: Optional[ExecuteModelRequest] = None
121
+ ) -> List[SamplerOutput]:
122
+ """Run execute_model in the driver worker.
123
+
124
+ Passing None will cause the driver to stop the model execution
125
+ loop running in each of the remote workers.
126
+ """
127
+ raise NotImplementedError
128
+
129
@abstractmethod
130
def _run_workers(
131
self,
132
method: str,
133
*args,
96
- driver_args: Optional[Tuple[Any, ...]] = None,
97
- driver_kwargs: Optional[Dict[str, Any]] = None,
134
+ async_run_remote_workers_only: bool = False,
135
max_concurrent_workers: Optional[int] = None,
136
**kwargs,
137
) -> Any:
101
- """Runs the given method on all workers."""
138
+ """Runs the given method on all workers.
139
+
140
+ Args:
141
+ async_run_remote_workers_only: If True the method will be run only
142
+ in the remote workers, not the driver worker. It will also be
143
+ run asynchronously and return a list of futures rather than
144
+ blocking on the results.
145
+ """
146
+ raise NotImplementedError
147
+
148
+ @abstractmethod
149
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
150
+ """Wait for futures returned from _run_workers() with
151
+ async_run_remote_workers_only to complete."""
152
raise NotImplementedError
153
154
155
class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
156
157
+ async def execute_model_async(
158
+ self,
159
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
160
+ if self.parallel_worker_tasks is None:
161
+ # Start model execution loop running in the parallel workers
162
+ self.parallel_worker_tasks = asyncio.create_task(
163
+ self._start_worker_execution_loop())
164
+
165
+ # Only the driver worker returns the sampling results.
166
+ return await self._driver_execute_model_async(execute_model_req)
167
+
168
+ async def stop_remote_worker_execution_loop_async(self) -> None:
169
+ if self.parallel_worker_tasks is None:
170
+ return
171
+
172
+ await self._driver_execute_model_async()
173
+ parallel_worker_tasks = self.parallel_worker_tasks
174
+ self.parallel_worker_tasks = None
175
+ # Ensure that workers exit model loop cleanly
176
+ # (this will raise otherwise)
177
+ await parallel_worker_tasks
178
+
179
@abstractmethod
108
- async def _run_workers_async(
180
+ async def _driver_execute_model_async(
181
self,
110
- method: str,
111
- *args,
112
- driver_args: Optional[Tuple[Any, ...]] = None,
113
- driver_kwargs: Optional[Dict[str, Any]] = None,
114
- **kwargs,
115
- ) -> Any:
116
- """Runs the given method on all workers."""
117
- raise NotImplementedError
182
+ execute_model_req: Optional[ExecuteModelRequest] = None
183
+ ) -> List[SamplerOutput]:
184
+ """Execute the model asynchronously in the driver worker.
185
119
- async def execute_model_async(self, *args,
120
- **kwargs) -> List[SamplerOutput]:
121
- all_outputs = await self._run_workers_async("execute_model",
122
- driver_args=args,
123
- driver_kwargs=kwargs)
186
+ Passing None will cause the driver to stop the model execution
187
+ loop running in each of the remote workers.
188
+ """
189
+ raise NotImplementedError
190
125
- # Only the driver worker returns the sampling results.
126
- return all_outputs[0]
191
+ @abstractmethod
192
+ async def _start_worker_execution_loop(self):
193
+ """Run execution loop on all workers. It guarantees all workers run
194
+ the loop or None of them is running the loop. Loop can be stopped by
195
+ `stop_remote_worker_execution_loop`.
196
+ The API is idempotent (guarantee only 1 loop run at any moment)."""
197
+ raise NotImplementedError
vllm/executor/executor_base.py
+8
@@ -74,6 +74,10 @@ class ExecutorBase(ABC):
74
"""Executes at least one model step on the given sequences."""
75
raise NotImplementedError
76
77
+ def stop_remote_worker_execution_loop(self) -> None:
78
+ """Releases parallel workers from model loop."""
79
+ return
80
+
81
@abstractmethod
82
def add_lora(self, lora_request: LoRARequest) -> bool:
83
raise NotImplementedError
@@ -109,6 +113,10 @@ class ExecutorAsyncBase(ExecutorBase):
113
"""Executes one model step on the given sequences."""
114
raise NotImplementedError
115
116
+ async def stop_remote_worker_execution_loop_async(self) -> None:
117
+ """Releases parallel workers from model loop."""
118
+ return
119
+
120
async def check_health_async(self) -> None:
121
"""Checks if the executor is healthy. If not, it should raise an
122
exception."""
vllm/executor/multiproc_gpu_executor.py
+44
-29
@@ -1,13 +1,14 @@
1
import asyncio
2
import os
3
from functools import partial
4
-from typing import Any, Dict, Optional, Tuple
4
+from typing import Any, List, Optional
5
6
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
7
DistributedGPUExecutor, DistributedGPUExecutorAsync)
8
from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9
ResultHandler, WorkerMonitor)
10
from vllm.logger import init_logger
11
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
get_vllm_instance_id, make_async)
14
@@ -71,16 +72,34 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
72
None)) is not None:
73
worker_monitor.close()
74
75
+ def _driver_execute_model(
76
+ self,
77
+ execute_model_req: Optional[ExecuteModelRequest] = None
78
+ ) -> List[SamplerOutput]:
79
+ """Run execute_model in the driver worker.
80
+
81
+ Passing None will cause the driver to stop the model execution
82
+ loop running in each of the remote workers.
83
+ """
84
+ return self.driver_worker.execute_model(
85
+ execute_model_req=execute_model_req)
86
+
87
def _run_workers(
88
self,
89
method: str,
90
*args,
78
- driver_args: Optional[Tuple[Any, ...]] = None,
79
- driver_kwargs: Optional[Dict[str, Any]] = None,
91
+ async_run_remote_workers_only: bool = False,
92
max_concurrent_workers: Optional[int] = None,
93
**kwargs,
94
) -> Any:
83
- """Runs the given method on all workers."""
95
+ """Runs the given method on all workers.
96
+
97
+ Args:
98
+ async_run_remote_workers_only: If True the method will be run only
99
+ in the remote workers, not the driver worker. It will also be
100
+ run asynchronously and return a list of futures rather than
101
+ blocking on the results.
102
+ """
103
104
if max_concurrent_workers:
105
raise NotImplementedError(
@@ -92,15 +111,12 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
111
for worker in self.workers
112
]
113
95
- if driver_args is None:
96
- driver_args = args
97
- if driver_kwargs is None:
98
- driver_kwargs = kwargs
114
+ if async_run_remote_workers_only:
115
+ # Just return futures
116
+ return worker_outputs
117
100
- # Start the driver worker after all the ray workers.
118
driver_worker_method = getattr(self.driver_worker, method)
102
- driver_worker_output = driver_worker_method(*driver_args,
103
- **driver_kwargs)
119
+ driver_worker_output = driver_worker_method(*args, **kwargs)
120
121
# Get the results of the workers.
122
return [driver_worker_output
@@ -111,30 +127,29 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
127
if not self.worker_monitor.is_alive():
128
raise RuntimeError("Worker processes are not running")
129
130
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
131
+ """Wait for futures returned from _run_workers() with
132
+ async_run_remote_workers_only to complete."""
133
+ for result in parallel_worker_tasks:
134
+ result.get()
135
+
136
137
class MultiprocessingGPUExecutorAsync(MultiprocessingGPUExecutor,
138
DistributedGPUExecutorAsync):
139
118
- async def _run_workers_async(
119
- self,
120
- method: str,
121
- *args,
122
- driver_args: Optional[Tuple[Any, ...]] = None,
123
- driver_kwargs: Optional[Dict[str, Any]] = None,
124
- **kwargs,
125
- ) -> Any:
126
- """Runs the given method on all workers."""
127
- if driver_args is None:
128
- driver_args = args
129
- if driver_kwargs is None:
130
- driver_kwargs = kwargs
140
+ def __init__(self, *args, **kwargs):
141
+ super().__init__(*args, **kwargs)
142
+ self.driver_exec_model = make_async(self.driver_worker.execute_model)
143
132
- driver_executor = make_async(getattr(self.driver_worker, method))
144
+ async def _driver_execute_model_async(
145
+ self,
146
+ execute_model_req: Optional[ExecuteModelRequest] = None
147
+ ) -> List[SamplerOutput]:
148
+ return await self.driver_exec_model(execute_model_req)
149
134
- # Run all the workers asynchronously.
135
- coros = [driver_executor(*driver_args, **driver_kwargs)] + [
136
- worker.execute_method_async(method, *args, **kwargs)
150
+ async def _start_worker_execution_loop(self):
151
+ coros = [
152
+ worker.execute_method_async("start_worker_execution_loop")
153
for worker in self.workers
154
]
139
-
155
return await asyncio.gather(*coros)
vllm/executor/ray_gpu_executor.py
+43
-43
@@ -42,6 +42,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
42
self.forward_dag = None
43
if USE_RAY_COMPILED_DAG:
44
self.forward_dag = self._compiled_ray_dag()
45
+ self.extra_execute_model_run_workers_kwargs[
46
+ "use_ray_compiled_dag"] = True
47
48
def _configure_ray_workers_use_nsight(self,
49
ray_remote_kwargs) -> Dict[str, Any]:
@@ -171,23 +173,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
173
max_concurrent_workers=self.parallel_config.
174
max_parallel_loading_workers)
175
174
- def execute_model(
175
- self,
176
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
177
- all_outputs = self._run_workers(
178
- "execute_model",
179
- driver_kwargs={"execute_model_req": execute_model_req},
180
- use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
176
+ def _driver_execute_model(
177
+ self,
178
+ execute_model_req: Optional[ExecuteModelRequest] = None
179
+ ) -> List[SamplerOutput]:
180
+ """Run execute_model in the driver worker.
181
182
- # Only the driver worker returns the sampling results.
183
- return all_outputs[0]
182
+ Passing None will cause the driver to stop the model execution
183
+ loop running in each of the remote workers.
184
+ """
185
+ return self.driver_worker.execute_method("execute_model",
186
+ execute_model_req)
187
188
def _run_workers(
189
self,
190
method: str,
191
*args,
189
- driver_args: Optional[Tuple[Any, ...]] = None,
190
- driver_kwargs: Optional[Dict[str, Any]] = None,
192
+ async_run_remote_workers_only: bool = False,
193
all_args: Optional[List[Tuple[Any, ...]]] = None,
194
all_kwargs: Optional[List[Dict[str, Any]]] = None,
195
use_dummy_driver: bool = False,
@@ -198,9 +200,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
200
"""Runs the given method on all workers. Can be used in the following
201
ways:
202
203
+ - async_run_remote_workers_only: If True the method will be run only
204
+ in the remote workers, not the driver worker. It will also be
205
+ run asynchronously and return a list of futures rather than blocking
206
+ on the results.
207
- args/kwargs: All workers share the same args/kwargs
202
- - args/kwargs and driver_args/driver_kwargs: Driver worker has
203
- different args
208
- all_args/all_kwargs: args/kwargs for each worker are specified
209
individually
210
"""
@@ -209,11 +213,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
213
raise NotImplementedError(
214
"max_concurrent_workers is not supported yet.")
215
212
- if driver_args is None:
213
- driver_args = args if all_args is None else all_args[0]
214
- if driver_kwargs is None:
215
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
216
-
216
count = len(self.workers)
217
all_worker_args = repeat(args, count) if all_args is None \
218
else islice(all_args, 1, None)
@@ -225,6 +224,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
224
# input. TODO(sang): Fix it.
225
assert self.forward_dag is not None
226
output_channels = self.forward_dag.execute(1)
227
+ ray_worker_outputs = []
228
else:
229
# Start the ray workers first.
230
ray_worker_outputs = [
@@ -234,6 +234,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
234
) in zip(self.workers, all_worker_args, all_worker_kwargs)
235
]
236
237
+ if async_run_remote_workers_only:
238
+ # Just return futures
239
+ return ray_worker_outputs
240
+
241
+ driver_args = args if all_args is None else all_args[0]
242
+ driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
243
+
244
# Start the driver worker after all the ray workers.
245
if not use_dummy_driver:
246
driver_worker_output = self.driver_worker.execute_method(
@@ -260,6 +267,11 @@ class RayGPUExecutor(DistributedGPUExecutor):
267
268
return [driver_worker_output] + ray_worker_outputs
269
270
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
271
+ """Wait for futures returned from _run_workers() with
272
+ async_run_remote_workers_only to complete."""
273
+ ray.get(parallel_worker_tasks)
274
+
275
def _compiled_ray_dag(self):
276
import pkg_resources
277
required_version = "2.9"
@@ -303,30 +315,18 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
315
316
def __init__(self, *args, **kwargs):
317
super().__init__(*args, **kwargs)
306
- self.driver_executor = make_async(self.driver_worker.execute_method)
318
+ self.driver_exec_method = make_async(self.driver_worker.execute_method)
319
308
- async def _run_workers_async(
320
+ async def _driver_execute_model_async(
321
self,
310
- method: str,
311
- *args,
312
- driver_args: Optional[Tuple[Any, ...]] = None,
313
- driver_kwargs: Optional[Dict[str, Any]] = None,
314
- **kwargs,
315
- ) -> Any:
316
- """Runs the given method on all workers."""
317
- coros = []
318
-
319
- if driver_args is None:
320
- driver_args = args
321
- if driver_kwargs is None:
322
- driver_kwargs = kwargs
323
-
324
- coros.append(
325
- self.driver_executor(method, *driver_args, **driver_kwargs))
326
-
327
- # Run the ray workers asynchronously.
328
- for worker in self.workers:
329
- coros.append(worker.execute_method.remote(method, *args, **kwargs))
330
-
331
- all_outputs = await asyncio.gather(*coros)
332
- return all_outputs
322
+ execute_model_req: Optional[ExecuteModelRequest] = None
323
+ ) -> List[SamplerOutput]:
324
+ return await self.driver_exec_method("execute_model",
325
+ execute_model_req)
326
+
327
+ async def _start_worker_execution_loop(self):
328
+ coros = [
329
+ worker.execute_method.remote("start_worker_execution_loop")
330
+ for worker in self.workers
331
+ ]
332
+ return await asyncio.gather(*coros)
vllm/spec_decode/ngram_worker.py
+3
-1
@@ -47,7 +47,9 @@ class NGramWorker(LoraNotSupportedWorkerBase):
47
# NGram don't need gpu sampler
48
pass
49
50
- def execute_model(self, execute_model_req: ExecuteModelRequest) -> None:
50
+ def execute_model(
51
+ self,
52
+ execute_model_req: Optional[ExecuteModelRequest] = None) -> None:
53
"""NGram doesn't depend on model execution, just pass this function"""
54
pass
55
vllm/spec_decode/spec_decode_worker.py
+61
-64
@@ -231,35 +231,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
231
self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
232
num_cpu_blocks=num_cpu_blocks)
233
234
- def _broadcast_control_flow_decision(
235
- self,
236
- execute_model_req: Optional[ExecuteModelRequest] = None,
237
- disable_all_speculation: bool = False) -> Tuple[int, bool]:
238
- """Broadcast how many lookahead slots are scheduled for this step, and
239
- whether all speculation is disabled, to all non-driver workers.
240
-
241
- This is required as if the number of draft model runs changes
242
- dynamically, the non-driver workers won't know unless we perform a
243
- communication to inform then.
244
-
245
- Returns the broadcasted num_lookahead_slots and disable_all_speculation.
246
- """
247
-
248
- if self.rank == self._driver_rank:
249
- assert execute_model_req is not None
250
-
251
- broadcast_dict = dict(
252
- num_lookahead_slots=execute_model_req.num_lookahead_slots,
253
- disable_all_speculation=disable_all_speculation,
254
- )
255
- broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
256
- else:
257
- assert execute_model_req is None
258
- broadcast_dict = broadcast_tensor_dict(src=self._driver_rank)
259
-
260
- return (broadcast_dict["num_lookahead_slots"],
261
- broadcast_dict["disable_all_speculation"])
262
-
234
@torch.inference_mode()
235
def execute_model(
236
self,
@@ -267,39 +238,58 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
238
) -> List[SamplerOutput]:
239
"""Perform speculative decoding on the input batch.
240
"""
241
+ if self.rank != self._driver_rank:
242
+ self._run_non_driver_rank()
243
+ return []
244
271
- disable_all_speculation = False
272
- if self.rank == self._driver_rank:
273
- disable_all_speculation = self._should_disable_all_speculation(
274
- execute_model_req)
275
-
276
- (num_lookahead_slots,
277
- disable_all_speculation) = self._broadcast_control_flow_decision(
278
- execute_model_req, disable_all_speculation)
279
-
280
- if self.rank == self._driver_rank:
281
- assert execute_model_req is not None
282
- assert execute_model_req.seq_group_metadata_list is not None, (
283
- "speculative decoding requires non-None seq_group_metadata_list"
284
- )
285
-
286
- self._maybe_disable_speculative_tokens(
287
- disable_all_speculation,
288
- execute_model_req.seq_group_metadata_list)
289
-
290
- # If no spec tokens, call the proposer and scorer workers normally.
291
- # Used for prefill.
292
- if num_lookahead_slots == 0 or len(
293
- execute_model_req.seq_group_metadata_list) == 0:
294
- return self._run_no_spec(execute_model_req,
295
- skip_proposer=disable_all_speculation)
296
-
297
- return self._run_speculative_decoding_step(execute_model_req,
298
- num_lookahead_slots)
299
- else:
300
- self._run_non_driver_rank(num_lookahead_slots)
245
+ if execute_model_req is None:
246
+ # This signals that there's no more requests to process for now.
247
+ # All workers are running infinite loop with broadcast_tensor_dict,
248
+ # and it stops the loop when the driver broadcasts an empty input.
249
+ # Send an empty input to notify all other workers to stop their
250
+ # execution loop.
251
+ broadcast_tensor_dict({}, src=0)
252
return []
253
254
+ disable_all_speculation = self._should_disable_all_speculation(
255
+ execute_model_req)
256
+ num_lookahead_slots = execute_model_req.num_lookahead_slots
257
+
258
+ # Broadcast how many lookahead slots are scheduled for this step, and
259
+ # whether all speculation is disabled, to all non-driver workers.
260
+
261
+ # This is required as if the number of draft model runs changes
262
+ # dynamically, the non-driver workers won't know unless we perform a
263
+ # communication to inform then.
264
+ broadcast_dict = dict(
265
+ num_lookahead_slots=num_lookahead_slots,
266
+ disable_all_speculation=disable_all_speculation,
267
+ )
268
+ broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
269
+
270
+ assert execute_model_req.seq_group_metadata_list is not None, (
271
+ "speculative decoding requires non-None seq_group_metadata_list")
272
+
273
+ self._maybe_disable_speculative_tokens(
274
+ disable_all_speculation, execute_model_req.seq_group_metadata_list)
275
+
276
+ # If no spec tokens, call the proposer and scorer workers normally.
277
+ # Used for prefill.
278
+ if num_lookahead_slots == 0 or len(
279
+ execute_model_req.seq_group_metadata_list) == 0:
280
+ return self._run_no_spec(execute_model_req,
281
+ skip_proposer=disable_all_speculation)
282
+
283
+ return self._run_speculative_decoding_step(execute_model_req,
284
+ num_lookahead_slots)
285
+
286
+ @torch.inference_mode()
287
+ def start_worker_execution_loop(self) -> None:
288
+ """Execute model loop to perform speculative decoding
289
+ in parallel worker."""
290
+ while self._run_non_driver_rank():
291
+ pass
292
+
293
def _should_disable_all_speculation(
294
self, execute_model_req: ExecuteModelRequest) -> bool:
295
# When the batch size is too large, disable speculative decoding
@@ -346,13 +336,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
336
sampler_output.logprobs = None
337
return [sampler_output]
338
349
- def _run_non_driver_rank(self, num_lookahead_slots: int) -> None:
339
+ def _run_non_driver_rank(self) -> bool:
340
"""Run proposer and verifier model in non-driver workers. This is used
341
for both speculation cases (num_lookahead_slots>0) and non-speculation
342
cases (e.g. prefill).
343
+
344
+ Returns True iff there are remaining sequences to process.
345
"""
354
- # In non-driver workers the input is None
355
- execute_model_req = None
346
+ assert self.rank != self._driver_rank
347
+
348
+ data = broadcast_tensor_dict(src=self._driver_rank)
349
+ if not data:
350
+ return False
351
+ num_lookahead_slots = data["num_lookahead_slots"]
352
353
# Even if num_lookahead_slots is zero, we want to run the proposer model
354
# as it may have KV.
@@ -360,9 +356,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
356
# We run the proposer once per lookahead slot. In the future we should
357
# delegate how many times it runs to the proposer.
358
for _ in range(max(num_lookahead_slots, 1)):
363
- self.proposer_worker.execute_model(execute_model_req)
359
+ self.proposer_worker.execute_model()
360
365
- self.scorer_worker.execute_model(execute_model_req)
361
+ self.scorer_worker.execute_model()
362
+ return True
363
364
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
365
def _run_speculative_decoding_step(
vllm/worker/embedding_model_runner.py
+3
-2
@@ -47,7 +47,7 @@ class EmbeddingModelRunner(ModelRunner):
47
@torch.inference_mode()
48
def execute_model(
49
self,
50
- seq_group_metadata_list: List[SequenceGroupMetadata],
50
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
51
kv_caches: List[torch.Tensor],
52
) -> Optional[PoolerOutput]:
53
(input_tokens, input_positions, attn_metadata, pooling_metadata,
@@ -84,10 +84,11 @@ class EmbeddingModelRunner(ModelRunner):
84
85
def prepare_input_tensors(
86
self,
87
- seq_group_metadata_list: List[SequenceGroupMetadata],
87
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
88
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89
Set[LoRARequest], LoRAMapping, torch.Tensor]:
90
if self.is_driver_worker:
91
+ assert seq_group_metadata_list is not None
92
# Prepare input tensors.
93
(
94
input_tokens,
vllm/worker/model_runner.py
+3
-2
@@ -609,10 +609,11 @@ class ModelRunner:
609
610
def prepare_input_tensors(
611
self,
612
- seq_group_metadata_list: List[SequenceGroupMetadata],
612
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
613
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
614
Set[LoRARequest], LoRAMapping, torch.Tensor]:
615
if self.is_driver_worker:
616
+ assert seq_group_metadata_list is not None
617
# Prepare input tensors.
618
(
619
input_tokens,
@@ -676,7 +677,7 @@ class ModelRunner:
677
@torch.inference_mode()
678
def execute_model(
679
self,
679
- seq_group_metadata_list: List[SequenceGroupMetadata],
680
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
681
kv_caches: List[torch.Tensor],
682
) -> Optional[SamplerOutput]:
683
(input_tokens, input_positions, attn_metadata, sampling_metadata,
vllm/worker/worker.py
+65
-38
@@ -226,48 +226,42 @@ class Worker(WorkerBase):
226
self,
227
execute_model_req: Optional[ExecuteModelRequest] = None
228
) -> List[Union[SamplerOutput, PoolerOutput]]:
229
+ if not self.is_driver_worker:
230
+ self._execute_model_non_driver()
231
+ return []
232
233
if execute_model_req is None:
231
- seq_group_metadata_list = None
232
- else:
233
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
234
+ # This signals that there's no more requests to process for now.
235
+ # All workers are running infinite loop with broadcast_tensor_dict,
236
+ # and it stops the loop when the driver broadcasts an empty input.
237
+ # Send an empty input to notify all other workers to stop their
238
+ # execution loop.
239
+ broadcast_tensor_dict({}, src=0)
240
+ return []
241
235
- blocks_to_swap_in: torch.Tensor
236
- blocks_to_swap_out: torch.Tensor
237
- blocks_to_copy: torch.Tensor
238
- if self.is_driver_worker:
239
- assert seq_group_metadata_list is not None
240
- assert execute_model_req is not None
241
- num_seq_groups = len(seq_group_metadata_list)
242
- # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
243
- # they contain parameters to launch cudamemcpyasync.
244
- blocks_to_swap_in = torch.tensor(
245
- execute_model_req.blocks_to_swap_in,
246
- device="cpu",
247
- dtype=torch.int64).view(-1, 2)
248
- blocks_to_swap_out = torch.tensor(
249
- execute_model_req.blocks_to_swap_out,
250
- device="cpu",
251
- dtype=torch.int64).view(-1, 2)
252
- # `blocks_to_copy` is a gpu tensor. The src and tgt of
253
- # blocks to copy are in the same device, and `blocks_to_copy`
254
- # can be used directly within cuda kernels.
255
- blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
256
- device=self.device,
242
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
243
+ num_seq_groups = len(seq_group_metadata_list)
244
+ # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
245
+ # they contain parameters to launch cudamemcpyasync.
246
+ blocks_to_swap_in = torch.tensor(execute_model_req.blocks_to_swap_in,
247
+ device="cpu",
248
+ dtype=torch.int64).view(-1, 2)
249
+ blocks_to_swap_out = torch.tensor(execute_model_req.blocks_to_swap_out,
250
+ device="cpu",
251
dtype=torch.int64).view(-1, 2)
258
- data: Dict[str, Any] = {
259
- "num_seq_groups": num_seq_groups,
260
- "blocks_to_swap_in": blocks_to_swap_in,
261
- "blocks_to_swap_out": blocks_to_swap_out,
262
- "blocks_to_copy": blocks_to_copy,
263
- }
264
- broadcast_tensor_dict(data, src=0)
265
- else:
266
- data = broadcast_tensor_dict(src=0)
267
- num_seq_groups = data["num_seq_groups"]
268
- blocks_to_swap_in = data["blocks_to_swap_in"]
269
- blocks_to_swap_out = data["blocks_to_swap_out"]
270
- blocks_to_copy = data["blocks_to_copy"]
252
+ # `blocks_to_copy` is a gpu tensor. The src and tgt of
253
+ # blocks to copy are in the same device, and `blocks_to_copy`
254
+ # can be used directly within cuda kernels.
255
+ blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
256
+ device=self.device,
257
+ dtype=torch.int64).view(-1, 2)
258
+ data: Dict[str, Any] = {
259
+ "num_seq_groups": num_seq_groups,
260
+ "blocks_to_swap_in": blocks_to_swap_in,
261
+ "blocks_to_swap_out": blocks_to_swap_out,
262
+ "blocks_to_copy": blocks_to_copy,
263
+ }
264
+ broadcast_tensor_dict(data, src=0)
265
266
self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
267
@@ -282,6 +276,39 @@ class Worker(WorkerBase):
276
# to conform to interface.
277
return [output]
278
279
+ @torch.inference_mode()
280
+ def start_worker_execution_loop(self) -> None:
281
+ """Execute model loop in parallel worker.
282
+
283
+ You can stop the loop by executing a driver worker with an empty output.
284
+ See `stop_remote_worker_execution_loop` for more details.
285
+ """
286
+ while self._execute_model_non_driver():
287
+ pass
288
+
289
+ def _execute_model_non_driver(self) -> bool:
290
+ """Execute model in parallel worker.
291
+
292
+ Returns True iff there are remaining sequences to process.
293
+ """
294
+ assert not self.is_driver_worker
295
+ data = broadcast_tensor_dict(src=0)
296
+ if not data:
297
+ return False
298
+
299
+ num_seq_groups = data.get("num_seq_groups", 0)
300
+ blocks_to_swap_in = data.get("blocks_to_swap_in")
301
+ blocks_to_swap_out = data.get("blocks_to_swap_out")
302
+ blocks_to_copy = data.get("blocks_to_copy")
303
+ self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
304
+
305
+ # If there is no input, we don't need to execute the model.
306
+ if num_seq_groups == 0:
307
+ return False
308
+
309
+ self.model_runner.execute_model(None, self.gpu_cache)
310
+ return True
311
+
312
def add_lora(self, lora_request: LoRARequest) -> bool:
313
return self.model_runner.add_lora(lora_request)
314
vllm/worker/worker_base.py
+4
-3
@@ -1,7 +1,7 @@
1
import importlib
2
import os
3
from abc import ABC, abstractmethod
4
-from typing import Dict, List, Set, Tuple
4
+from typing import Dict, List, Optional, Set, Tuple
5
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
@@ -48,8 +48,9 @@ class WorkerBase(ABC):
48
49
@abstractmethod
50
def execute_model(
51
- self,
52
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51
+ self,
52
+ execute_model_req: Optional[ExecuteModelRequest] = None
53
+ ) -> List[SamplerOutput]:
54
"""Executes at least one model step on the given sequences, unless no
55
sequences are provided."""
56
raise NotImplementedError