A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Optimize Async + Multi-step (#8050)
Alexander Matveev committed
Sep 3, 2024 at 14:50 UTC
6d646d08a2e0e73e83e313a5ae470c1f9e4f200e
8 files changed
+326
-248
tests/multi_step/test_correctness_async_llm.py
+2
-2
@@ -103,13 +103,13 @@ async def test_multi_step(
103
model,
104
server_args + distributed_args,
105
num_logprobs,
106
- max_wait_seconds=3 * 240)
106
+ max_wait_seconds=5 * 240)
107
test_completions = await completions_with_server_args(
108
prompts,
109
model,
110
ms_server_args + distributed_args,
111
num_logprobs,
112
- max_wait_seconds=3 * 240)
112
+ max_wait_seconds=5 * 240)
113
114
# Assert multi-step scheduling produces identical tokens
115
# to single-step scheduling.
vllm/engine/async_llm_engine.py
+54
-55
@@ -280,40 +280,27 @@ class _AsyncLLMEngine(LLMEngine):
280
scheduler_outputs = cached_outputs.scheduler_outputs
281
allow_async_output_proc = cached_outputs.allow_async_output_proc
282
283
- # Detect async + multi-step
284
- use_async_and_multi_step = (self.scheduler_config.is_multi_step
285
- and allow_async_output_proc)
286
-
283
ctx = self.scheduler_contexts[virtual_engine]
284
285
+ # Clear outputs for each new scheduler iteration
286
+ ctx.request_outputs.clear()
287
+
288
# skip the scheduler if there are any remaining steps in the seq groups.
289
# This ensures that the scheduler is only called again when the current
290
# batch has completed.
291
if not self._has_remaining_steps(seq_group_metadata_list):
292
294
- # Clear outputs on scheduler iteration start
295
- ctx.request_outputs.clear()
296
-
293
# Schedule iteration
294
(seq_group_metadata_list, scheduler_outputs,
295
allow_async_output_proc
296
) = self.scheduler[virtual_engine].schedule()
297
302
- # Detect async + multi-step
303
- use_async_and_multi_step = (self.scheduler_config.is_multi_step
304
- and allow_async_output_proc)
298
+ ctx.seq_group_metadata_list = seq_group_metadata_list
299
+ ctx.scheduler_outputs = scheduler_outputs
300
301
# Maybe switch from async mode to sync mode
302
if not allow_async_output_proc and len(ctx.output_queue) > 0:
308
- self._process_model_outputs(virtual_engine=virtual_engine,
309
- is_async=True)
310
-
311
- # For async + multi-step, init the queue
312
- if use_async_and_multi_step:
313
- assert len(ctx.output_queue) == 0
314
- assert seq_group_metadata_list is not None
315
- ctx.output_queue.append(
316
- (None, seq_group_metadata_list, scheduler_outputs))
303
+ self._process_model_outputs(ctx=ctx)
304
305
if (self.scheduler_config.is_multi_step
306
and scheduler_outputs.num_lookahead_slots > 0):
@@ -351,26 +338,20 @@ class _AsyncLLMEngine(LLMEngine):
338
last_sampled_token_ids=last_sampled_token_ids)
339
340
if allow_async_output_proc:
354
- async_callback = self.async_callback_multi_step[
355
- virtual_engine] if use_async_and_multi_step \
356
- else self.async_callback[virtual_engine]
357
-
358
- execute_model_req.async_callback = async_callback
359
- execute_model_req.use_async_and_multi_step = \
360
- use_async_and_multi_step
341
+ execute_model_req.async_callback = self.async_callbacks[
342
+ virtual_engine]
343
344
# Execute the model.
345
output = await self.model_executor.execute_model_async(
346
execute_model_req)
347
+
348
# we need to do this here so that last step's sampled_token_ids can
349
# be passed to the next iteration for PP.
350
if self.scheduler_config.is_multi_step:
351
self._update_cached_scheduler_output(virtual_engine, output)
352
else:
370
- if not use_async_and_multi_step and len(ctx.output_queue) > 0:
371
- assert not self.scheduler_config.is_multi_step
372
- self._process_model_outputs(virtual_engine=virtual_engine,
373
- is_async=True)
353
+ if len(ctx.output_queue) > 0:
354
+ self._process_model_outputs(ctx=ctx)
355
output = []
356
357
# Finish the current step for all the sequence groups.
@@ -384,24 +365,22 @@ class _AsyncLLMEngine(LLMEngine):
365
self.cached_scheduler_outputs[
366
virtual_engine] = SchedulerOutputState()
367
387
- if use_async_and_multi_step:
388
- # For async + multi-step, clear the queue
389
- ctx.output_queue.clear()
390
- else:
391
- ctx.output_queue.append(
392
- (output, seq_group_metadata_list, scheduler_outputs))
368
+ is_async = allow_async_output_proc
369
+ is_last_step = True
370
+ ctx.output_queue.append(
371
+ (output, seq_group_metadata_list, scheduler_outputs, is_async,
372
+ is_last_step))
373
394
- if output and allow_async_output_proc:
395
- assert len(
396
- output
397
- ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
398
- self._advance_to_next_step(
399
- output[0], seq_group_metadata_list,
400
- scheduler_outputs.scheduled_seq_groups)
374
+ if output and allow_async_output_proc:
375
+ assert len(
376
+ output
377
+ ) == 1, "Async postprocessor expects only a single output set"
378
+ self._advance_to_next_step(
379
+ output[0], seq_group_metadata_list,
380
+ scheduler_outputs.scheduled_seq_groups)
381
382
if not allow_async_output_proc:
403
- self._process_model_outputs(virtual_engine=virtual_engine,
404
- is_async=False)
383
+ self._process_model_outputs(ctx=ctx)
384
385
# Log stats.
386
self.do_log_stats(scheduler_outputs, output)
@@ -411,17 +390,12 @@ class _AsyncLLMEngine(LLMEngine):
390
391
else:
392
# Multi-step case
414
- if use_async_and_multi_step:
415
- return []
416
- else:
417
- ctx.request_outputs = []
393
+ return ctx.request_outputs
394
395
if not self.has_unfinished_requests():
396
# Drain async postprocessor (if exists)
397
if len(ctx.output_queue) > 0:
422
- assert not self.scheduler_config.is_multi_step
423
- self._process_model_outputs(virtual_engine=virtual_engine,
424
- is_async=True)
398
+ self._process_model_outputs(ctx=ctx)
399
assert len(ctx.output_queue) == 0
400
401
return ctx.request_outputs
@@ -640,6 +614,17 @@ class AsyncLLMEngine:
614
self.log_requests = log_requests
615
self.engine = self._init_engine(*args, **kwargs)
616
617
+ # This ensures quick processing of request outputs
618
+ # so the append to asyncio queues is not delayed,
619
+ # especially for multi-step.
620
+ #
621
+ # TODO: Currently, disabled for engine_use_ray, ask
622
+ # Cody/Will/Woosuk about this case.
623
+ self.use_process_request_outputs_callback = not self.engine_use_ray
624
+ if self.use_process_request_outputs_callback:
625
+ self.engine.process_request_outputs_callback = \
626
+ self.process_request_outputs
627
+
628
if self.engine_use_ray:
629
print_warning_once(
630
"DEPRECATED. `--engine-use-ray` is deprecated and will "
@@ -883,13 +868,27 @@ class AsyncLLMEngine:
868
request_outputs = await self.engine.step_async(virtual_engine)
869
870
# Put the outputs into the corresponding streams.
886
- finished = True
871
+ # If used as a callback, then already invoked inside
872
+ # LLMEngine's _process_model_outputs
873
+ if not self.use_process_request_outputs_callback:
874
+ all_finished = self.process_request_outputs(request_outputs)
875
+ else:
876
+ # For callback case, we only need to detect when all
877
+ # requests are finished
878
+ all_finished = all(request_output.finished
879
+ for request_output in request_outputs)
880
+
881
+ return not all_finished
882
+
883
+ def process_request_outputs(self, request_outputs) -> bool:
884
+ # Put the outputs into the corresponding streams.
885
+ all_finished = True
886
for request_output in request_outputs:
887
self._request_tracker.process_request_output(
888
request_output, verbose=self.log_requests)
890
- finished = finished and request_output.finished
889
+ all_finished = all_finished and request_output.finished
890
892
- return not finished
891
+ return all_finished
892
893
async def _engine_abort(self, request_ids: Iterable[str]):
894
if self.engine_use_ray:
vllm/engine/llm_engine.py
+97
-125
@@ -93,13 +93,14 @@ class SchedulerOutputState:
93
@dataclass
94
class SchedulerContext:
95
output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
96
- List[SequenceGroupMetadata],
97
- SchedulerOutputs]] = field(
98
- default_factory=lambda: deque())
99
-
96
+ List[SequenceGroupMetadata], SchedulerOutputs,
97
+ bool,
98
+ bool]] = field(default_factory=lambda: deque())
99
request_outputs: List[Union[RequestOutput,
100
EmbeddingRequestOutput]] = field(
101
default_factory=lambda: [])
102
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
103
+ scheduler_outputs: Optional[SchedulerOutputs] = None
104
105
106
class LLMEngine:
@@ -357,6 +358,26 @@ class LLMEngine:
358
# different process.
359
self.tokenizer.ping()
360
361
+ self.cached_scheduler_outputs = [
362
+ SchedulerOutputState()
363
+ for _ in range(self.parallel_config.pipeline_parallel_size)
364
+ ]
365
+
366
+ self.scheduler_contexts = [
367
+ SchedulerContext()
368
+ for _ in range(self.parallel_config.pipeline_parallel_size)
369
+ ]
370
+
371
+ self.async_callbacks = [
372
+ functools.partial(self._process_model_outputs,
373
+ ctx=self.scheduler_contexts[v_id])
374
+ for v_id in range(self.parallel_config.pipeline_parallel_size)
375
+ ]
376
+
377
+ # Currently used by AsyncLLMEngine to ensure quick append
378
+ # of request outputs to asyncio queues
379
+ self.process_request_outputs_callback = None
380
+
381
# Create the scheduler.
382
# NOTE: the cache_config here have been updated with the numbers of
383
# GPU and CPU blocks, which are profiled in the distributed executor.
@@ -364,9 +385,7 @@ class LLMEngine:
385
Scheduler(
386
scheduler_config, cache_config, lora_config,
387
parallel_config.pipeline_parallel_size,
367
- functools.partial(self._process_model_outputs,
368
- virtual_engine=v_id,
369
- is_async=True)
388
+ self.async_callbacks[v_id]
389
if model_config.use_async_output_proc else None)
390
for v_id in range(parallel_config.pipeline_parallel_size)
391
]
@@ -417,30 +436,6 @@ class LLMEngine:
436
),
437
))
438
420
- self.cached_scheduler_outputs = [
421
- SchedulerOutputState()
422
- for _ in range(self.parallel_config.pipeline_parallel_size)
423
- ]
424
-
425
- self.scheduler_contexts = [
426
- SchedulerContext()
427
- for _ in range(self.parallel_config.pipeline_parallel_size)
428
- ]
429
-
430
- self.async_callback = [
431
- functools.partial(self._process_model_outputs,
432
- virtual_engine=v_id,
433
- is_async=True)
434
- for v_id in range(self.parallel_config.pipeline_parallel_size)
435
- ]
436
-
437
- self.async_callback_multi_step = [
438
- functools.partial(self._process_model_outputs,
439
- virtual_engine=v_id,
440
- is_async=False)
441
- for v_id in range(self.parallel_config.pipeline_parallel_size)
442
- ]
443
-
439
def _initialize_kv_caches(self) -> None:
440
"""Initialize the KV cache in the worker(s).
441
@@ -1249,11 +1244,7 @@ class LLMEngine:
1244
1245
return
1246
1252
- def _process_model_outputs(self,
1253
- virtual_engine: int,
1254
- is_async: bool,
1255
- sampler_output: Optional[SamplerOutput] = None,
1256
- is_last_output: bool = False) -> None:
1247
+ def _process_model_outputs(self, ctx: SchedulerContext) -> None:
1248
"""Apply the model output to the sequences in the scheduled seq groups.
1249
1250
virtual_engine: The engine id to operate on
@@ -1273,24 +1264,12 @@ class LLMEngine:
1264
"""
1265
now = time.time()
1266
1276
- is_multi_step = sampler_output is not None
1277
-
1278
- ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1279
-
1267
if len(ctx.output_queue) == 0:
1268
return None
1269
1283
- if is_multi_step:
1284
- # Async + multi-step case
1285
- (outputs, seq_group_metadata_list,
1286
- scheduler_outputs) = ctx.output_queue[0]
1287
- assert outputs is None
1288
- outputs = [sampler_output]
1289
- else:
1290
- # Async standard case
1291
- (outputs, seq_group_metadata_list,
1292
- scheduler_outputs) = ctx.output_queue.popleft()
1293
-
1270
+ # Get pending async postprocessor
1271
+ (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1272
+ is_last_step) = ctx.output_queue.popleft()
1273
assert outputs is not None
1274
1275
# Sanity check
@@ -1306,6 +1285,7 @@ class LLMEngine:
1285
outputs_by_sequence_group = outputs
1286
1287
finished_before: List[int] = []
1288
+ finished_now: List[int] = []
1289
for i, seq_group_meta in enumerate(seq_group_metadata_list):
1290
scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1291
@@ -1343,26 +1323,44 @@ class LLMEngine:
1323
1324
if self.model_config.embedding_mode:
1325
self._process_sequence_group_outputs(seq_group, output)
1346
- continue
1326
+ else:
1327
+ self.output_processor.process_prompt_logprob(seq_group, output)
1328
+ if seq_group_meta.do_sample:
1329
+ self.output_processor.process_outputs(
1330
+ seq_group, output, is_async)
1331
1348
- self.output_processor.process_prompt_logprob(seq_group, output)
1349
- if seq_group_meta.do_sample:
1350
- self.output_processor.process_outputs(seq_group, output,
1351
- is_async)
1332
+ if seq_group.is_finished():
1333
+ finished_now.append(i)
1334
1353
- # For async + multi-step, free finished seqs and create outputs
1354
- # only on the final step.
1355
- if is_multi_step and not is_last_output:
1356
- return
1335
+ # Generate outputs for the requests that finished this iteration
1336
+ for i in finished_now:
1337
+ scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1338
1358
- for scheduler in self.scheduler:
1359
- scheduler.free_finished_seq_groups()
1339
+ seq_group = scheduled_seq_group.seq_group
1340
+ seq_group.maybe_set_first_token_time(now)
1341
+ request_output = RequestOutputFactory.create(seq_group)
1342
+ ctx.request_outputs.append(request_output)
1343
1361
- # Create the outputs.
1362
- for i, _ in enumerate(seq_group_metadata_list):
1363
- scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1344
+ # Free currently finished requests
1345
+ if finished_now:
1346
+ for scheduler in self.scheduler:
1347
+ scheduler.free_finished_seq_groups()
1348
+
1349
+ # For multi-step, do not create outputs each iteration
1350
+ if not is_last_step:
1351
+ # Immediately process request outputs here (if callback is given)
1352
+ if (finished_now
1353
+ and self.process_request_outputs_callback is not None):
1354
+ self.process_request_outputs_callback(ctx.request_outputs)
1355
+ return
1356
+
1357
+ # Create the outputs
1358
+ # Note: scheduled_seq_groups and seq_group_metadata_list
1359
+ # must match with the indices
1360
+ for i, scheduled_seq_group in enumerate(
1361
+ scheduler_outputs.scheduled_seq_groups):
1362
1365
- if not is_multi_step and i in finished_before:
1363
+ if i in finished_before or i in finished_now:
1364
continue # Avoids double processing
1365
1366
seq_group = scheduled_seq_group.seq_group
@@ -1376,11 +1374,15 @@ class LLMEngine:
1374
request_output = RequestOutputFactory.create(seq_group)
1375
ctx.request_outputs.append(request_output)
1376
1379
- # For async + multi-step, do stats only on the last output.
1380
- # Otherwise, do stats if the execution is async
1381
- do_stats = is_multi_step or is_async
1377
+ # Immediately process request outputs here (if callback is given)
1378
+ if (ctx.request_outputs
1379
+ and self.process_request_outputs_callback is not None):
1380
+ self.process_request_outputs_callback(ctx.request_outputs)
1381
1383
- if do_stats:
1382
+ # For async case, we need to record the stats here.
1383
+ # For non-async case, the stats are done in the
1384
+ # LLMEngine/AsyncLLMEngine directly
1385
+ if is_async:
1386
# Log stats.
1387
self.do_log_stats(scheduler_outputs, outputs, finished_before)
1388
@@ -1485,40 +1487,26 @@ class LLMEngine:
1487
scheduler_outputs = cached_outputs.scheduler_outputs
1488
allow_async_output_proc = cached_outputs.allow_async_output_proc
1489
1488
- # Detect async + multi-step
1489
- use_async_and_multi_step = (self.scheduler_config.is_multi_step
1490
- and allow_async_output_proc)
1491
-
1490
ctx = self.scheduler_contexts[virtual_engine]
1491
1492
+ # Clear outputs for each new scheduler iteration
1493
+ ctx.request_outputs.clear()
1494
+
1495
# Skip the scheduler if there are any remaining steps in the seq groups.
1496
# This ensures that the scheduler is only called again when the current
1497
# batch has completed.
1498
if not self._has_remaining_steps(seq_group_metadata_list):
1498
-
1499
- # Clear outputs on scheduler iteration start
1500
- ctx.request_outputs.clear()
1501
-
1499
# Schedule iteration
1500
(seq_group_metadata_list, scheduler_outputs,
1501
allow_async_output_proc
1502
) = self.scheduler[virtual_engine].schedule()
1503
1507
- # Detect async + multi-step
1508
- use_async_and_multi_step = (self.scheduler_config.is_multi_step
1509
- and allow_async_output_proc)
1504
+ ctx.seq_group_metadata_list = seq_group_metadata_list
1505
+ ctx.scheduler_outputs = scheduler_outputs
1506
1507
# Maybe switch from async mode to sync mode
1508
if not allow_async_output_proc and len(ctx.output_queue) > 0:
1513
- self._process_model_outputs(virtual_engine=virtual_engine,
1514
- is_async=True)
1515
-
1516
- # For async + multi-step, init the queue
1517
- if use_async_and_multi_step:
1518
- assert len(ctx.output_queue) == 0
1519
- assert seq_group_metadata_list is not None
1520
- ctx.output_queue.append(
1521
- (None, seq_group_metadata_list, scheduler_outputs))
1509
+ self._process_model_outputs(ctx=ctx)
1510
1511
if (self.scheduler_config.is_multi_step
1512
and scheduler_outputs.num_lookahead_slots > 0):
@@ -1555,13 +1543,8 @@ class LLMEngine:
1543
last_sampled_token_ids=last_sampled_token_ids)
1544
1545
if allow_async_output_proc:
1558
- async_callback = self.async_callback_multi_step[
1559
- virtual_engine] if use_async_and_multi_step \
1560
- else self.async_callback[virtual_engine]
1561
-
1562
- execute_model_req.async_callback = async_callback
1563
- execute_model_req.use_async_and_multi_step = \
1564
- use_async_and_multi_step
1546
+ execute_model_req.async_callback = self.async_callbacks[
1547
+ virtual_engine]
1548
1549
output = self.model_executor.execute_model(
1550
execute_model_req=execute_model_req)
@@ -1573,10 +1556,8 @@ class LLMEngine:
1556
else:
1557
# Nothing scheduled => If there is pending async postprocessor,
1558
# then finish it here.
1576
- if not use_async_and_multi_step and len(ctx.output_queue) > 0:
1577
- assert not self.scheduler_config.is_multi_step
1578
- self._process_model_outputs(virtual_engine=virtual_engine,
1579
- is_async=True)
1559
+ if len(ctx.output_queue) > 0:
1560
+ self._process_model_outputs(ctx=ctx)
1561
# No outputs in this case
1562
output = []
1563
@@ -1590,28 +1571,24 @@ class LLMEngine:
1571
if self.scheduler_config.is_multi_step:
1572
self.cached_scheduler_outputs[0] = SchedulerOutputState()
1573
1593
- if use_async_and_multi_step:
1594
- # For async + multi-step, clear the queue
1595
- ctx.output_queue.clear()
1596
- else:
1597
- # Add results to the output_queue
1598
- # (for async or non-async postprocessing)
1599
- ctx.output_queue.append(
1600
- (output, seq_group_metadata_list, scheduler_outputs))
1574
+ # Add results to the output_queue
1575
+ is_async = allow_async_output_proc
1576
+ is_last_step = True
1577
+ ctx.output_queue.append(
1578
+ (output, seq_group_metadata_list, scheduler_outputs, is_async,
1579
+ is_last_step))
1580
1602
- if output and allow_async_output_proc:
1603
- assert len(output) == 1, (
1604
- "Multi step decoding does not work "
1605
- "with async output processing.")
1581
+ if output and allow_async_output_proc:
1582
+ assert len(output) == 1, (
1583
+ "Async postprocessor expects only a single output set")
1584
1607
- self._advance_to_next_step(
1608
- output[0], seq_group_metadata_list,
1609
- scheduler_outputs.scheduled_seq_groups)
1585
+ self._advance_to_next_step(
1586
+ output[0], seq_group_metadata_list,
1587
+ scheduler_outputs.scheduled_seq_groups)
1588
1589
# Check if need to run the usual non-async path
1590
if not allow_async_output_proc:
1613
- self._process_model_outputs(virtual_engine=virtual_engine,
1614
- is_async=False)
1591
+ self._process_model_outputs(ctx=ctx)
1592
1593
# Log stats.
1594
self.do_log_stats(scheduler_outputs, output)
@@ -1620,17 +1597,12 @@ class LLMEngine:
1597
self.do_tracing(scheduler_outputs)
1598
else:
1599
# Multi-step case
1623
- if use_async_and_multi_step:
1624
- return []
1625
- else:
1626
- ctx.request_outputs = []
1600
+ return ctx.request_outputs
1601
1602
if not self.has_unfinished_requests():
1603
# Drain async postprocessor (if exists)
1604
if len(ctx.output_queue) > 0:
1631
- assert not self.scheduler_config.is_multi_step
1632
- self._process_model_outputs(virtual_engine=virtual_engine,
1633
- is_async=True)
1605
+ self._process_model_outputs(ctx=ctx)
1606
assert len(ctx.output_queue) == 0
1607
1608
# Stop the execute model loop in parallel workers until there are
vllm/engine/output_processor/multi_step.py
+36
-26
@@ -85,9 +85,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
85
no tokens need to be appended since it is already done
86
externally (before the next schedule() call)
87
"""
88
- # TODO: Add support for async if necessary
89
- assert not is_async
90
-
88
# Sequences can be in RUNNING or FINISHED_ABORTED state
89
# once scheduled, as a sequence is moved to FINSIHED_ABORTED
90
# if a client disconnects from the api server.
@@ -101,19 +98,41 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
98
"Beam search not supported in multi-step decoding.")
99
seq = seqs[0]
100
104
- # Since there's only one sequence per sequence group, we can take the
105
- # first sample.
106
- samples = [output.samples[0] for output in outputs]
107
-
108
- # -1 means the output token is not valid (eg. due to spec decode
109
- # rejecting tokens).
110
- valid_samples = [
111
- sample for sample in samples if sample.output_token != -1
112
- ]
113
- assert valid_samples
114
-
115
- self._process_seq_outputs(seq, valid_samples,
116
- sequence_group.sampling_params)
101
+ if is_async:
102
+ # Async case: We process tokens one by one. Here, we know the token
103
+ # was already appended, so we only need to do the rest of the
104
+ # postprocessor: Detokenization + stopping logic
105
+ self._process_decode_and_stop(seq, sequence_group.sampling_params)
106
+ else:
107
+ # Standard multi-step case
108
+
109
+ # Since there's only one sequence per sequence group,
110
+ # we can take the first sample.
111
+ samples = [output.samples[0] for output in outputs]
112
+
113
+ # -1 means the output token is not valid (eg. due to spec decode
114
+ # rejecting tokens).
115
+ valid_samples = [
116
+ sample for sample in samples if sample.output_token != -1
117
+ ]
118
+ assert valid_samples
119
+
120
+ self._process_seq_outputs(seq, valid_samples,
121
+ sequence_group.sampling_params)
122
+
123
+ def _process_decode_and_stop(self, seq: Sequence,
124
+ sampling_params: SamplingParams) -> None:
125
+ new_char_count = 0
126
+ if sampling_params.detokenize:
127
+ new_char_count = self.detokenizer.decode_sequence_inplace(
128
+ seq, sampling_params)
129
+
130
+ # TODO(sang): Support lora.
131
+ self.stop_checker.maybe_stop_sequence(
132
+ seq,
133
+ new_char_count=new_char_count,
134
+ sampling_params=sampling_params,
135
+ )
136
137
def _process_seq_outputs(self, seq: Sequence,
138
valid_samples: List[SequenceOutput],
@@ -151,16 +170,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
170
logprobs=output_logprob,
171
)
172
154
- new_char_count = 0
155
- if sampling_params.detokenize:
156
- new_char_count = self.detokenizer.decode_sequence_inplace(
157
- seq, sampling_params)
173
+ self._process_decode_and_stop(seq, sampling_params)
174
159
- # TODO(sang): Support lora.
160
- self.stop_checker.maybe_stop_sequence(
161
- seq,
162
- new_char_count=new_char_count,
163
- sampling_params=sampling_params,
164
- )
175
if seq.is_finished():
176
break
vllm/sequence.py
+1
-3
@@ -1225,7 +1225,6 @@ class ExecuteModelRequest(
1225
last_sampled_token_ids: Optional[torch.Tensor] = None
1226
# Async callback
1227
async_callback: Optional[Callable] = None
1228
- use_async_and_multi_step: bool = False
1228
1229
@property
1230
def is_first_multi_step(self) -> bool:
@@ -1272,5 +1271,4 @@ class ExecuteModelRequest(
1271
finished_requests_ids=self.finished_requests_ids,
1272
last_sampled_token_ids=self.last_sampled_token_ids.clone()
1273
if self.last_sampled_token_ids is not None else None,
1275
- async_callback=self.async_callback,
1276
- use_async_and_multi_step=self.use_async_and_multi_step)
1274
+ async_callback=self.async_callback)
vllm/worker/model_runner.py
+3
-1
@@ -21,6 +21,7 @@ from vllm.attention.backends.utils import CommonAttentionState
21
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
22
ModelConfig, ObservabilityConfig, ParallelConfig,
23
PromptAdapterConfig, SchedulerConfig)
24
+from vllm.core.scheduler import SchedulerOutputs
25
from vllm.distributed import get_pp_group
26
from vllm.distributed.parallel_state import graph_capture
27
from vllm.inputs import INPUT_REGISTRY, InputRegistry
@@ -96,7 +97,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
97
finished_requests_ids: Optional[List[str]] = None
98
virtual_engine: int = 0
99
async_callback: Optional[Callable] = None
99
- use_async_and_multi_step: bool = False
100
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
101
+ scheduler_outputs: Optional[SchedulerOutputs] = None
102
103
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
104
tensor_dict = {
vllm/worker/multi_step_model_runner.py
+132
-33
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
22
get_pythonized_sample_results)
23
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
24
Logprob, SequenceGroupMetadata, SequenceOutput)
25
+from vllm.utils import PyObjectCache
26
from vllm.worker.model_runner import (GPUModelRunnerBase,
27
ModelInputForGPUWithSamplingMetadata)
28
from vllm.worker.model_runner_base import (
@@ -37,6 +38,29 @@ if TYPE_CHECKING:
38
logger = init_logger(__name__)
39
40
41
+def seq_output_builder():
42
+ return SequenceOutput(
43
+ 0, 0,
44
+ {0: Logprob(logprob=float('inf'), rank=None, decoded_token=None)})
45
+
46
+
47
+def completion_seq_group_output_builder():
48
+ return CompletionSequenceGroupOutput([], None)
49
+
50
+
51
+# Used by pythonization to reduce python object allocations
52
+class PythonizationCache:
53
+
54
+ def __init__(self):
55
+ self.cached_seq_output = PyObjectCache(seq_output_builder)
56
+ self.cached_completion_seq_group_output = PyObjectCache(
57
+ completion_seq_group_output_builder)
58
+
59
+ def reset(self):
60
+ self.cached_seq_output.reset()
61
+ self.cached_completion_seq_group_output.reset()
62
+
63
+
64
@dataclass
65
class ModelOutput:
66
"""The output of a single model forward pass.
@@ -59,6 +83,7 @@ class ModelOutput:
83
pythonized: bool = False
84
# On-device tensor containing the logprobs of each token.
85
logprobs: Optional["torch.Tensor"] = None
86
+ pythonization_cache: Optional[PythonizationCache] = None
87
88
def pythonize(self, input_metadata: "StatefulModelInput",
89
copy_stream: torch.cuda.Stream,
@@ -97,7 +122,8 @@ class ModelOutput:
122
with torch.cuda.stream(copy_stream):
123
_pythonize_sampler_output(input_metadata, self.sampler_output,
124
pinned_sampled_token_buffer,
100
- self.sampled_token_ids, self.logprobs)
125
+ self.sampled_token_ids, self.logprobs,
126
+ self.pythonization_cache)
127
128
# Erase the logprobs GPU-side tensor.
129
# Note that although _pythonize_sampler_output() runs in its
@@ -209,6 +235,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
235
self._copy_stream = torch.cuda.Stream()
236
self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
237
238
+ self.pythonization_cache = PythonizationCache()
239
+
240
def make_model_input_from_broadcasted_tensor_dict(
241
self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
242
model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
@@ -237,14 +265,22 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
265
output_proc_callback: Callable):
266
# Proceed with pythonization and output_proc in order.
267
# Stop on the first one that fails to pythonize
268
+ output_proc_callback()
269
+
270
cont = True
271
for model_output in model_input.cached_outputs:
272
if not model_output.pythonized:
273
model_output.maybe_pythonize(model_input, self._copy_stream,
274
self.pinned_sampled_token_ids)
275
if model_output.pythonized:
246
- output_proc_callback(
247
- sampler_output=model_output.sampler_output)
276
+ ctx = output_proc_callback.keywords["ctx"]
277
+ is_async = False
278
+ is_last_step = False
279
+ ctx.output_queue.append(
280
+ ([model_output.sampler_output
281
+ ], ctx.seq_group_metadata_list,
282
+ ctx.scheduler_outputs, is_async, is_last_step))
283
+ output_proc_callback()
284
else:
285
cont = False
286
@@ -255,21 +291,46 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
291
output_proc_callback: Optional[Callable]):
292
assert model_input.frozen_model_input is not None
293
294
+ has_async_callback = output_proc_callback is not None
295
+
296
outputs = []
297
for output_id in range(len(model_input.cached_outputs)):
260
- is_last_output = output_id == len(model_input.cached_outputs) - 1
261
-
298
output = model_input.cached_outputs[output_id]
263
- if not output.pythonized:
299
+ is_last_step = output_id == len(model_input.cached_outputs) - 1
300
+
301
+ # For non-async case:
302
+ # -- We simply add the outputs
303
+ # For async case:
304
+ # -- Invoke callback, pythonize, add to callback queue and repeat
305
+ # -- For last output, just add to callback queue
306
+ if has_async_callback:
307
+ assert output_proc_callback is not None
308
+
309
+ # Invoke callback before pythonize (to overlap with GPU)
310
+ output_proc_callback()
311
+
312
+ # Pythonize
313
+ if not output.pythonized:
314
+ output.pythonize(model_input, self._copy_stream,
315
+ self.pinned_sampled_token_ids)
316
+
317
+ # For non last step, add to callback queue to chain
318
+ # callbacks=>pythonize pairs (for GPU overlap)
319
+ if not is_last_step:
320
+ ctx = output_proc_callback.keywords[ # type: ignore
321
+ "ctx"] # type: ignore
322
+ is_async = False
323
+ is_last_step = False
324
+ ctx.output_queue.append(
325
+ ([output.sampler_output
326
+ ], ctx.seq_group_metadata_list,
327
+ ctx.scheduler_outputs, is_async, is_last_step))
328
+ else:
329
+ outputs.append(output.sampler_output)
330
+ else:
331
output.pythonize(model_input, self._copy_stream,
332
self.pinned_sampled_token_ids)
266
-
267
- if model_input.frozen_model_input.use_async_and_multi_step:
268
- assert output_proc_callback is not None
269
- output_proc_callback(sampler_output=output.sampler_output,
270
- is_last_output=is_last_output)
271
-
272
- outputs.append(output.sampler_output)
333
+ outputs.append(output.sampler_output)
334
335
return outputs
336
@@ -330,7 +391,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
391
model_input, model_input.cached_outputs[-1].sampler_output)
392
393
output_proc_callback = None
333
- if frozen_model_input.use_async_and_multi_step:
394
+ if frozen_model_input.async_callback is not None:
395
output_proc_callback = frozen_model_input.async_callback
396
assert output_proc_callback is not None
397
async_callback = functools.partial(
@@ -367,7 +428,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
428
model_input.cached_outputs.append(
429
ModelOutput(output[0], output_ready_event,
430
output[0].sampled_token_ids, False,
370
- output[0].logprobs))
431
+ output[0].logprobs, self.pythonization_cache))
432
433
# These GPU tensors are not required by multi-step;
434
# erase them to ensure they are not pythonized or
@@ -378,7 +439,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
439
440
# Pythonize the output if CPU is ahead and the previous step is
441
# ready.
381
- if not frozen_model_input.use_async_and_multi_step:
442
+ if frozen_model_input.async_callback is None:
443
for model_output in model_input.cached_outputs:
444
model_output.maybe_pythonize(model_input,
445
self._copy_stream,
@@ -397,6 +458,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
458
if model_input.is_last_step:
459
outputs = self._final_process_outputs(model_input,
460
output_proc_callback)
461
+ self.pythonization_cache.reset()
462
return outputs
463
464
# should be [SamplerOutput]
@@ -537,6 +599,7 @@ def _pythonize_sampler_output(
599
pinned_sampled_token_buffer: torch.Tensor,
600
sampled_token_ids: torch.Tensor,
601
logprobs_tensor: Optional[torch.Tensor],
602
+ cache: Optional[PythonizationCache],
603
) -> None:
604
""" This function is only called when the output tensors are ready.
605
See :class:`ModelOutput`.
@@ -597,6 +660,9 @@ def _pythonize_sampler_output(
660
661
for sgdx, (seq_group,
662
sample_result) in enumerate(zip(seq_groups, samples_list)):
663
+ if seq_group.sampling_params.logits_processors:
664
+ assert len(seq_group.sampling_params.logits_processors) == 0, (
665
+ "Logits Processors are not supported in multi-step decoding")
666
667
if do_pythonize_logprobs:
668
assert prompt_logprobs is not None
@@ -621,23 +687,56 @@ def _pythonize_sampler_output(
687
seq_ids = seq_group.seq_ids
688
next_token_ids = sample_result
689
parent_ids = [0]
624
- seq_outputs: List[SequenceOutput] = []
625
- if seq_group.sampling_params.logits_processors:
626
- assert len(seq_group.sampling_params.logits_processors) == 0, (
627
- "Logits Processors are not supported in multi-step decoding")
690
+
691
+ if cache is not None:
692
+ completion_seq_group_output: CompletionSequenceGroupOutput = \
693
+ cache.cached_completion_seq_group_output.get_object()
694
+ completion_seq_group_output.samples.clear()
695
+ seq_outputs: List[
696
+ SequenceOutput] = completion_seq_group_output.samples
697
+ else:
698
+ seq_outputs = []
699
+
700
for tdx, (parent_id,
701
next_token_id) in enumerate(zip(parent_ids, next_token_ids)):
630
- seq_outputs.append(
631
- SequenceOutput(seq_ids[parent_id], next_token_id,
632
- (group_sample_logprobs[tdx]
633
- if logprobs_are_requested else {
634
- next_token_id:
635
- Logprob(logprob=float('inf'),
636
- rank=None,
637
- decoded_token=None)
638
- })))
639
- output.outputs.append(
640
- CompletionSequenceGroupOutput(
641
- seq_outputs,
642
- (group_prompt_logprobs if logprobs_are_requested else None)))
702
+ if cache is not None:
703
+ seq_output: SequenceOutput = cache.cached_seq_output.get_object(
704
+ )
705
+ seq_output.parent_seq_id = seq_ids[parent_id]
706
+ seq_output.output_token = next_token_id
707
+
708
+ if logprobs_are_requested:
709
+ seq_output.logprobs = group_sample_logprobs[tdx]
710
+ else:
711
+ logprobs = next(iter(seq_output.logprobs.values()))
712
+ seq_output.logprobs.clear()
713
+
714
+ logprobs.logprob = float('inf')
715
+ logprobs.rank = None
716
+ logprobs.decoded_token = None
717
+
718
+ seq_output.logprobs[next_token_id] = logprobs
719
+
720
+ seq_outputs.append(seq_output)
721
+
722
+ else:
723
+ seq_outputs.append(
724
+ SequenceOutput(seq_ids[parent_id], next_token_id,
725
+ (group_sample_logprobs[tdx]
726
+ if logprobs_are_requested else {
727
+ next_token_id:
728
+ Logprob(logprob=float('inf'),
729
+ rank=None,
730
+ decoded_token=None)
731
+ })))
732
+ if cache is not None:
733
+ completion_seq_group_output.prompt_logprobs = \
734
+ group_prompt_logprobs if logprobs_are_requested else None
735
+ output.outputs.append(completion_seq_group_output)
736
+ else:
737
+ output.outputs.append(
738
+ CompletionSequenceGroupOutput(
739
+ seq_outputs, (group_prompt_logprobs
740
+ if logprobs_are_requested else None)))
741
+
742
assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py
+1
-3
@@ -67,9 +67,7 @@ class MultiStepWorker(Worker):
67
if execute_model_req.async_callback:
68
model_input.frozen_model_input = dataclasses.replace( # type: ignore
69
model_input.frozen_model_input,
70
- async_callback=execute_model_req.async_callback,
71
- use_async_and_multi_step=execute_model_req.
72
- use_async_and_multi_step)
70
+ async_callback=execute_model_req.async_callback)
71
else:
72
# on subsequent steps we reuse the worker input and model input
73
multi_step_state = self.multi_step_states[virtual_engine]