A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Combine async postprocessor and multi-step (#7921)
Alexander Matveev committed
Aug 29, 2024 at 14:18 UTC
3f60f2244e3ffec6198d7a41765918d1efd3bb96
8 files changed
+215
-65
tests/multi_step/test_correctness_async_llm.py
+6
-4
@@ -47,10 +47,12 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
47
@pytest.mark.parametrize("eager_mode", [False, True])
48
@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
49
@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
50
+@pytest.mark.parametrize("is_async", [False, True])
51
@pytest.mark.asyncio
52
async def test_multi_step(example_prompts, model: str, tp_size: int,
53
pp_size: int, eager_mode: int,
53
- num_scheduler_steps: int, num_prompts: int):
54
+ num_scheduler_steps: int, num_prompts: int,
55
+ is_async: bool):
56
57
prompts = example_prompts
58
if len(prompts) < num_prompts:
@@ -62,9 +64,9 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
64
ms_server_args = DEFAULT_SERVER_ARGS + \
65
["--num-scheduler-steps", f"{num_scheduler_steps}"]
66
65
- # Disable output proc callback as its not supported
66
- # with multi-step right now
67
- ms_server_args += ["--disable-async-output-proc"]
67
+ if not is_async:
68
+ ms_server_args += ["--disable-async-output-proc"]
69
+
70
if eager_mode:
71
ms_server_args.append("--enforce-eager")
72
vllm/core/scheduler.py
+1
-4
@@ -1107,10 +1107,7 @@ class Scheduler:
1107
if not self.cache_config.enable_prefix_caching:
1108
common_computed_block_nums = []
1109
1110
- # TODO: Combine multi-step and async postprocessor
1111
- allow_async_output_proc: bool = (
1112
- self.use_async_output_proc
1113
- and not self.scheduler_config.is_multi_step)
1110
+ allow_async_output_proc: bool = self.use_async_output_proc
1111
1112
# Create input data structures.
1113
seq_group_metadata_list: List[SequenceGroupMetadata] = []
vllm/engine/async_llm_engine.py
+44
-21
@@ -279,6 +279,10 @@ class _AsyncLLMEngine(LLMEngine):
279
scheduler_outputs = cached_outputs.scheduler_outputs
280
allow_async_output_proc = cached_outputs.allow_async_output_proc
281
282
+ # Detect async + multi-step
283
+ use_async_and_multi_step = (self.scheduler_config.is_multi_step
284
+ and allow_async_output_proc)
285
+
286
ctx = self.scheduler_contexts[virtual_engine]
287
288
# skip the scheduler if there are any remaining steps in the seq groups.
@@ -289,17 +293,27 @@ class _AsyncLLMEngine(LLMEngine):
293
# Clear outputs on scheduler iteration start
294
ctx.request_outputs.clear()
295
296
+ # Schedule iteration
297
(seq_group_metadata_list, scheduler_outputs,
298
allow_async_output_proc
299
) = self.scheduler[virtual_engine].schedule()
300
296
- # If current scheduler iteration has no async postprocessor,
297
- # then we need first to drain the pending async postprocessor
298
- # before moving forward
301
+ # Detect async + multi-step
302
+ use_async_and_multi_step = (self.scheduler_config.is_multi_step
303
+ and allow_async_output_proc)
304
+
305
+ # Maybe switch from async mode to sync mode
306
if not allow_async_output_proc and len(ctx.output_queue) > 0:
307
self._process_model_outputs(virtual_engine=virtual_engine,
308
is_async=True)
309
310
+ # For async + multi-step, init the queue
311
+ if use_async_and_multi_step:
312
+ assert len(ctx.output_queue) == 0
313
+ assert seq_group_metadata_list is not None
314
+ ctx.output_queue.append(
315
+ (None, seq_group_metadata_list, scheduler_outputs))
316
+
317
if (self.scheduler_config.is_multi_step
318
and scheduler_outputs.num_lookahead_slots > 0):
319
# cache the scheduler outputs for the next iteration if we have
@@ -311,9 +325,6 @@ class _AsyncLLMEngine(LLMEngine):
325
assert seq_group_metadata_list is not None
326
assert scheduler_outputs is not None
327
314
- assert not (self.scheduler_config.is_multi_step and \
315
- allow_async_output_proc)
316
-
328
if not scheduler_outputs.is_empty():
329
finished_requests_ids = self.scheduler[
330
virtual_engine].get_and_reset_finished_requests_ids()
@@ -339,8 +350,13 @@ class _AsyncLLMEngine(LLMEngine):
350
last_sampled_token_ids=last_sampled_token_ids)
351
352
if allow_async_output_proc:
342
- execute_model_req.async_callback = self.async_callback[
343
- virtual_engine]
353
+ async_callback = self.async_callback_multi_step[
354
+ virtual_engine] if use_async_and_multi_step \
355
+ else self.async_callback[virtual_engine]
356
+
357
+ execute_model_req.async_callback = async_callback
358
+ execute_model_req.use_async_and_multi_step = \
359
+ use_async_and_multi_step
360
361
# Execute the model.
362
output = await self.model_executor.execute_model_async(
@@ -350,7 +366,7 @@ class _AsyncLLMEngine(LLMEngine):
366
if self.scheduler_config.is_multi_step:
367
self._update_cached_scheduler_output(virtual_engine, output)
368
else:
353
- if len(ctx.output_queue) > 0:
369
+ if not use_async_and_multi_step and len(ctx.output_queue) > 0:
370
assert not self.scheduler_config.is_multi_step
371
self._process_model_outputs(virtual_engine=virtual_engine,
372
is_async=True)
@@ -362,22 +378,25 @@ class _AsyncLLMEngine(LLMEngine):
378
seq_group.finish_step()
379
380
if not self._has_remaining_steps(seq_group_metadata_list):
365
- # clear the cache if we have finished all the steps
381
+ # Clear the cache if we have finished all the steps
382
if self.scheduler_config.is_multi_step:
383
self.cached_scheduler_outputs[
384
virtual_engine] = SchedulerOutputState()
385
370
- # Cache results in engine
371
- ctx.output_queue.append(
372
- (output, seq_group_metadata_list, scheduler_outputs))
386
+ if use_async_and_multi_step:
387
+ # For async + multi-step, clear the queue
388
+ ctx.output_queue.clear()
389
+ else:
390
+ ctx.output_queue.append(
391
+ (output, seq_group_metadata_list, scheduler_outputs))
392
374
- if output and allow_async_output_proc:
375
- assert len(
376
- output
377
- ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
378
- self._advance_to_next_step(
379
- output[0], seq_group_metadata_list,
380
- scheduler_outputs.scheduled_seq_groups)
393
+ if output and allow_async_output_proc:
394
+ assert len(
395
+ output
396
+ ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
397
+ self._advance_to_next_step(
398
+ output[0], seq_group_metadata_list,
399
+ scheduler_outputs.scheduled_seq_groups)
400
401
if not allow_async_output_proc:
402
self._process_model_outputs(virtual_engine=virtual_engine,
@@ -390,7 +409,11 @@ class _AsyncLLMEngine(LLMEngine):
409
self.do_tracing(scheduler_outputs)
410
411
else:
393
- ctx.request_outputs = []
412
+ # Multi-step case
413
+ if use_async_and_multi_step:
414
+ return []
415
+ else:
416
+ ctx.request_outputs = []
417
418
if not self.has_unfinished_requests():
419
# Drain async postprocessor (if exists)
vllm/engine/llm_engine.py
+88
-26
@@ -91,7 +91,8 @@ class SchedulerOutputState:
91
92
@dataclass
93
class SchedulerContext:
94
- output_queue: Deque[Tuple[List[SamplerOutput], List[SequenceGroupMetadata],
94
+ output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
95
+ List[SequenceGroupMetadata],
96
SchedulerOutputs]] = field(
97
default_factory=lambda: deque())
98
@@ -432,6 +433,13 @@ class LLMEngine:
433
for v_id in range(self.parallel_config.pipeline_parallel_size)
434
]
435
436
+ self.async_callback_multi_step = [
437
+ functools.partial(self._process_model_outputs,
438
+ virtual_engine=v_id,
439
+ is_async=False)
440
+ for v_id in range(self.parallel_config.pipeline_parallel_size)
441
+ ]
442
+
443
def _initialize_kv_caches(self) -> None:
444
"""Initialize the KV cache in the worker(s).
445
@@ -1240,28 +1248,49 @@ class LLMEngine:
1248
1249
return
1250
1243
- def _process_model_outputs(self, virtual_engine: int,
1244
- is_async: bool) -> None:
1251
+ def _process_model_outputs(self,
1252
+ virtual_engine: int,
1253
+ is_async: bool,
1254
+ sampler_output: Optional[SamplerOutput] = None,
1255
+ is_last_output: bool = False) -> None:
1256
"""Apply the model output to the sequences in the scheduled seq groups.
1257
1258
virtual_engine: The engine id to operate on
1259
+
1260
is_async: Indicates whether this postprocessor runs in
1261
parallel with the GPU forward pass and is processing
1262
tokens from the previous step. If this is true, then
1263
no tokens need to be appended since it is already done
1264
externally (before the next schedule() call)
1265
1266
+ sampler_output: Used with multi-step execution to provide
1267
+ sampler_output of each step
1268
+ is_last_output: Used with multi-step execution to indicate
1269
+ the last step (of each multi-step group)
1270
+
1271
Returns RequestOutputs that can be returned to the client.
1272
"""
1273
now = time.time()
1274
1275
+ is_multi_step = sampler_output is not None
1276
+
1277
ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1278
1279
if len(ctx.output_queue) == 0:
1280
return None
1281
1263
- (outputs, seq_group_metadata_list,
1264
- scheduler_outputs) = ctx.output_queue.popleft()
1282
+ if is_multi_step:
1283
+ # Async + multi-step case
1284
+ (outputs, seq_group_metadata_list,
1285
+ scheduler_outputs) = ctx.output_queue[0]
1286
+ assert outputs is None
1287
+ outputs = [sampler_output]
1288
+ else:
1289
+ # Async standard case
1290
+ (outputs, seq_group_metadata_list,
1291
+ scheduler_outputs) = ctx.output_queue.popleft()
1292
+
1293
+ assert outputs is not None
1294
1295
# Sanity check
1296
assert len(seq_group_metadata_list) == len(
@@ -1320,7 +1349,11 @@ class LLMEngine:
1349
self.output_processor.process_outputs(seq_group, output,
1350
is_async)
1351
1323
- # Free the finished sequence groups.
1352
+ # For async + multi-step, free finished seqs and create outputs
1353
+ # only on the final step.
1354
+ if is_multi_step and not is_last_output:
1355
+ return
1356
+
1357
for scheduler in self.scheduler:
1358
scheduler.free_finished_seq_groups()
1359
@@ -1328,7 +1361,7 @@ class LLMEngine:
1361
for i, _ in enumerate(seq_group_metadata_list):
1362
scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1363
1331
- if i in finished_before:
1364
+ if not is_multi_step and i in finished_before:
1365
continue # Avoids double processing
1366
1367
seq_group = scheduled_seq_group.seq_group
@@ -1342,7 +1375,11 @@ class LLMEngine:
1375
request_output = RequestOutputFactory.create(seq_group)
1376
ctx.request_outputs.append(request_output)
1377
1345
- if is_async:
1378
+ # For async + multi-step, do stats only on the last output.
1379
+ # Otherwise, do stats if the execution is async
1380
+ do_stats = is_multi_step or is_async
1381
+
1382
+ if do_stats:
1383
# Log stats.
1384
self.do_log_stats(scheduler_outputs, outputs, finished_before)
1385
@@ -1437,7 +1474,7 @@ class LLMEngine:
1474
"as performance will be severely degraded otherwise.")
1475
1476
# For llm_engine, there is no pipeline parallel support, so the engine
1440
- # used is always 0
1477
+ # used is always 0.
1478
virtual_engine = 0
1479
1480
# These are cached outputs from previous iterations. None if on first
@@ -1447,6 +1484,10 @@ class LLMEngine:
1484
scheduler_outputs = cached_outputs.scheduler_outputs
1485
allow_async_output_proc = cached_outputs.allow_async_output_proc
1486
1487
+ # Detect async + multi-step
1488
+ use_async_and_multi_step = (self.scheduler_config.is_multi_step
1489
+ and allow_async_output_proc)
1490
+
1491
ctx = self.scheduler_contexts[virtual_engine]
1492
1493
# Skip the scheduler if there are any remaining steps in the seq groups.
@@ -1462,11 +1503,22 @@ class LLMEngine:
1503
allow_async_output_proc
1504
) = self.scheduler[virtual_engine].schedule()
1505
1506
+ # Detect async + multi-step
1507
+ use_async_and_multi_step = (self.scheduler_config.is_multi_step
1508
+ and allow_async_output_proc)
1509
+
1510
# Maybe switch from async mode to sync mode
1511
if not allow_async_output_proc and len(ctx.output_queue) > 0:
1512
self._process_model_outputs(virtual_engine=virtual_engine,
1513
is_async=True)
1514
1515
+ # For async + multi-step, init the queue
1516
+ if use_async_and_multi_step:
1517
+ assert len(ctx.output_queue) == 0
1518
+ assert seq_group_metadata_list is not None
1519
+ ctx.output_queue.append(
1520
+ (None, seq_group_metadata_list, scheduler_outputs))
1521
+
1522
if (self.scheduler_config.is_multi_step
1523
and scheduler_outputs.num_lookahead_slots > 0):
1524
# cache the scheduler outputs for the next iteration if we have
@@ -1478,9 +1530,6 @@ class LLMEngine:
1530
assert seq_group_metadata_list is not None
1531
assert scheduler_outputs is not None
1532
1481
- assert not (self.scheduler_config.is_multi_step and \
1482
- allow_async_output_proc)
1483
-
1533
if not scheduler_outputs.is_empty():
1534
finished_requests_ids = self.scheduler[
1535
virtual_engine].get_and_reset_finished_requests_ids()
@@ -1505,8 +1554,13 @@ class LLMEngine:
1554
last_sampled_token_ids=last_sampled_token_ids)
1555
1556
if allow_async_output_proc:
1508
- execute_model_req.async_callback = self.async_callback[
1509
- virtual_engine]
1557
+ async_callback = self.async_callback_multi_step[
1558
+ virtual_engine] if use_async_and_multi_step \
1559
+ else self.async_callback[virtual_engine]
1560
+
1561
+ execute_model_req.async_callback = async_callback
1562
+ execute_model_req.use_async_and_multi_step = \
1563
+ use_async_and_multi_step
1564
1565
output = self.model_executor.execute_model(
1566
execute_model_req=execute_model_req)
@@ -1518,7 +1572,7 @@ class LLMEngine:
1572
else:
1573
# Nothing scheduled => If there is pending async postprocessor,
1574
# then finish it here.
1521
- if len(ctx.output_queue) > 0:
1575
+ if not use_async_and_multi_step and len(ctx.output_queue) > 0:
1576
assert not self.scheduler_config.is_multi_step
1577
self._process_model_outputs(virtual_engine=virtual_engine,
1578
is_async=True)
@@ -1535,18 +1589,23 @@ class LLMEngine:
1589
if self.scheduler_config.is_multi_step:
1590
self.cached_scheduler_outputs[0] = SchedulerOutputState()
1591
1538
- # Add results to the output_queue
1539
- # (for async or non-async postprocessing)
1540
- ctx.output_queue.append(
1541
- (output, seq_group_metadata_list, scheduler_outputs))
1592
+ if use_async_and_multi_step:
1593
+ # For async + multi-step, clear the queue
1594
+ ctx.output_queue.clear()
1595
+ else:
1596
+ # Add results to the output_queue
1597
+ # (for async or non-async postprocessing)
1598
+ ctx.output_queue.append(
1599
+ (output, seq_group_metadata_list, scheduler_outputs))
1600
1543
- if output and allow_async_output_proc:
1544
- assert len(output) == 1, ("Multi step decoding does not work "
1545
- "with async output processing.")
1601
+ if output and allow_async_output_proc:
1602
+ assert len(output) == 1, (
1603
+ "Multi step decoding does not work "
1604
+ "with async output processing.")
1605
1547
- self._advance_to_next_step(
1548
- output[0], seq_group_metadata_list,
1549
- scheduler_outputs.scheduled_seq_groups)
1606
+ self._advance_to_next_step(
1607
+ output[0], seq_group_metadata_list,
1608
+ scheduler_outputs.scheduled_seq_groups)
1609
1610
# Check if need to run the usual non-async path
1611
if not allow_async_output_proc:
@@ -1560,7 +1619,10 @@ class LLMEngine:
1619
self.do_tracing(scheduler_outputs)
1620
else:
1621
# Multi-step case
1563
- ctx.request_outputs = []
1622
+ if use_async_and_multi_step:
1623
+ return []
1624
+ else:
1625
+ ctx.request_outputs = []
1626
1627
if not self.has_unfinished_requests():
1628
# Drain async postprocessor (if exists)
vllm/sequence.py
+3
-1
@@ -1295,6 +1295,7 @@ class ExecuteModelRequest(
1295
last_sampled_token_ids: Optional[torch.Tensor] = None
1296
# Async callback
1297
async_callback: Optional[Callable] = None
1298
+ use_async_and_multi_step: bool = False
1299
1300
@property
1301
def is_first_multi_step(self) -> bool:
@@ -1341,4 +1342,5 @@ class ExecuteModelRequest(
1342
finished_requests_ids=self.finished_requests_ids,
1343
last_sampled_token_ids=self.last_sampled_token_ids.clone()
1344
if self.last_sampled_token_ids is not None else None,
1344
- async_callback=self.async_callback)
1345
+ async_callback=self.async_callback,
1346
+ use_async_and_multi_step=self.use_async_and_multi_step)
vllm/worker/model_runner.py
+1
@@ -92,6 +92,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
92
finished_requests_ids: Optional[List[str]] = None
93
virtual_engine: int = 0
94
async_callback: Optional[Callable] = None
95
+ use_async_and_multi_step: bool = False
96
97
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
98
tensor_dict = {
vllm/worker/multi_step_model_runner.py
+64
-9
@@ -1,5 +1,7 @@
1
+import dataclasses
2
+import functools
3
from dataclasses import dataclass, field
2
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
4
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
5
6
try:
7
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -215,6 +217,46 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
217
)
218
return model_input
219
220
+ def _async_process_outputs(self, model_input: StatefulModelInput,
221
+ output_proc_callback: Callable):
222
+ # Proceed with pythonization and output_proc in order.
223
+ # Stop on the first one that fails to pythonize
224
+ cont = True
225
+ for model_output in model_input.cached_outputs:
226
+ if not model_output.pythonized:
227
+ model_output.maybe_pythonize(model_input, self._copy_stream,
228
+ self.pinned_sampled_token_ids)
229
+ if model_output.pythonized:
230
+ output_proc_callback(
231
+ sampler_output=model_output.sampler_output)
232
+ else:
233
+ cont = False
234
+
235
+ if not cont:
236
+ break
237
+
238
+ def _final_process_outputs(self, model_input: StatefulModelInput,
239
+ output_proc_callback: Optional[Callable]):
240
+ assert model_input.frozen_model_input is not None
241
+
242
+ outputs = []
243
+ for output_id in range(len(model_input.cached_outputs)):
244
+ is_last_output = output_id == len(model_input.cached_outputs) - 1
245
+
246
+ output = model_input.cached_outputs[output_id]
247
+ if not output.pythonized:
248
+ output.pythonize(model_input, self._copy_stream,
249
+ self.pinned_sampled_token_ids)
250
+
251
+ if model_input.frozen_model_input.use_async_and_multi_step:
252
+ assert output_proc_callback is not None
253
+ output_proc_callback(sampler_output=output.sampler_output,
254
+ is_last_output=is_last_output)
255
+
256
+ outputs.append(output.sampler_output)
257
+
258
+ return outputs
259
+
260
@torch.inference_mode()
261
def execute_model(
262
self,
@@ -271,6 +313,20 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
313
model_input = self._advance_step(
314
model_input, model_input.cached_outputs[-1].sampler_output)
315
316
+ output_proc_callback = None
317
+ if frozen_model_input.use_async_and_multi_step:
318
+ output_proc_callback = frozen_model_input.async_callback
319
+ assert output_proc_callback is not None
320
+ async_callback = functools.partial(
321
+ self._async_process_outputs,
322
+ model_input=model_input,
323
+ output_proc_callback=output_proc_callback)
324
+
325
+ frozen_model_input = dataclasses.replace( # type: ignore
326
+ model_input.frozen_model_input,
327
+ async_callback=async_callback)
328
+ assert frozen_model_input is not None
329
+
330
# Execute the model
331
output = self._base_model_runner.execute_model(frozen_model_input,
332
kv_caches,
@@ -301,9 +357,11 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
357
output[0].logprobs = None
358
# Pythonize the output if CPU is ahead and the previous step is
359
# ready.
304
- for model_output in model_input.cached_outputs:
305
- model_output.maybe_pythonize(model_input, self._copy_stream,
306
- self.pinned_sampled_token_ids)
360
+ if not frozen_model_input.use_async_and_multi_step:
361
+ for model_output in model_input.cached_outputs:
362
+ model_output.maybe_pythonize(model_input,
363
+ self._copy_stream,
364
+ self.pinned_sampled_token_ids)
365
366
model_input.current_step += 1
367
@@ -316,11 +374,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
374
375
# Pythonize the output and block if needed since it is the last step
376
if model_input.is_last_step:
319
- outputs = []
320
- for output in model_input.cached_outputs:
321
- output.pythonize(model_input, self._copy_stream,
322
- self.pinned_sampled_token_ids)
323
- outputs.append(output.sampler_output)
377
+ outputs = self._final_process_outputs(model_input,
378
+ output_proc_callback)
379
return outputs
380
381
# should be [SamplerOutput]
vllm/worker/multi_step_worker.py
+8
@@ -1,3 +1,4 @@
1
+import dataclasses
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple
4
@@ -61,6 +62,13 @@ class MultiStepWorker(Worker):
62
execute_model_req.seq_group_metadata_list,
63
execute_model_req.virtual_engine,
64
execute_model_req.finished_requests_ids))
65
+
66
+ if execute_model_req.async_callback:
67
+ model_input.frozen_model_input = dataclasses.replace( # type: ignore
68
+ model_input.frozen_model_input,
69
+ async_callback=execute_model_req.async_callback,
70
+ use_async_and_multi_step=execute_model_req.
71
+ use_async_and_multi_step)
72
else:
73
# on subsequent steps we reuse the worker input and model input
74
multi_step_state = self.multi_step_states[virtual_engine]