A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix async postprocessor in case of preemption (#8267)
Alexander Matveev committed
Sep 8, 2024 at 00:01 UTC
4ef41b84766670c1bd8079f58d35bf32b5bcb3ab
4 files changed
+172
-114
vllm/core/scheduler.py
+47
-40
@@ -537,13 +537,6 @@ class Scheduler:
537
preempted: List[SequenceGroup] = ret.preempted
538
swapped_out: List[SequenceGroup] = ret.swapped_out
539
540
- # NOTE(woosuk): Preemption happens only when there is no available slot
541
- # to keep all the sequence groups in the RUNNING state.
542
-
543
- # Store original running requests for the case of async + preemption
544
- if self.use_async_output_proc:
545
- orig_running = self.running.copy()
546
-
540
running_queue = self.running
541
assert len(self._async_stopped) == 0
542
while running_queue:
@@ -552,6 +545,7 @@ class Scheduler:
545
seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
546
547
if num_running_tokens == 0:
548
+ # No budget => Stop
549
break
550
551
running_queue.popleft()
@@ -565,18 +559,8 @@ class Scheduler:
559
self._async_stopped.append(seq_group)
560
continue
561
568
- # With async postprocessor, when preemption kicks in, we need
569
- # first to drain the async postprocessor, so that all async
570
- # block_table freeing is applied before the preemption freeing
571
- # is applied.
572
- if self.use_async_output_proc and not self._can_append_slots(
573
- seq_group):
574
- tmp = self.running
575
- self.running = orig_running
576
- assert self.output_proc_callback is not None
577
- self.output_proc_callback()
578
- self.running = tmp
579
-
562
+ # NOTE(woosuk): Preemption happens only when there is no available
563
+ # slot to keep all the sequence groups in the RUNNING state.
564
while not self._can_append_slots(seq_group):
565
budget.subtract_num_batched_tokens(seq_group.request_id,
566
num_running_tokens)
@@ -588,24 +572,43 @@ class Scheduler:
572
and seq_group.lora_int_id in curr_loras):
573
curr_loras.remove(seq_group.lora_int_id)
574
575
+ # Determine victim sequence
576
+ cont_loop = True
577
if running_queue:
592
- # Preempt the lowest-priority sequence groups.
578
+ # Preempt the lowest-priority sequence group.
579
victim_seq_group = running_queue.pop()
580
+ else:
581
+ # No other sequence group can be preempted.
582
+ # Preempt the current sequence group.
583
+ # Note: This is also where we stop this loop
584
+ # (since there is nothing else to preempt)
585
+ victim_seq_group = seq_group
586
+ cont_loop = False
587
+
588
+ # With async postprocessor, before preempting a sequence
589
+ # we need to ensure it has no pending async postprocessor
590
+ do_preempt = True
591
+ if self.use_async_output_proc:
592
+ assert self.output_proc_callback is not None
593
+ self.output_proc_callback(
594
+ request_id=victim_seq_group.request_id)
595
+
596
+ # It may be that the async pending "victim_seq_group"
597
+ # becomes finished, in which case we simply free it.
598
+ if victim_seq_group.is_finished():
599
+ self._free_finished_seq_group(victim_seq_group)
600
+ do_preempt = False
601
+
602
+ # Do preemption
603
+ if do_preempt:
604
preempted_mode = self._preempt(victim_seq_group,
605
blocks_to_swap_out)
606
if preempted_mode == PreemptionMode.RECOMPUTE:
607
preempted.append(victim_seq_group)
608
else:
609
swapped_out.append(victim_seq_group)
600
- else:
601
- # No other sequence groups can be preempted.
602
- # Preempt the current sequence group.
603
- preempted_mode = self._preempt(seq_group,
604
- blocks_to_swap_out)
605
- if preempted_mode == PreemptionMode.RECOMPUTE:
606
- preempted.append(seq_group)
607
- else:
608
- swapped_out.append(seq_group)
610
+
611
+ if not cont_loop:
612
break
613
else:
614
self._append_slots(seq_group, blocks_to_copy)
@@ -1264,22 +1267,26 @@ class Scheduler:
1267
if seq.is_finished():
1268
self.free_seq(seq)
1269
1270
+ def _free_finished_seq_group(self, seq_group: SequenceGroup) -> None:
1271
+ if seq_group.is_finished():
1272
+ # Free cross-attention block table, if it exists
1273
+ self._free_seq_group_cross_attn_blocks(seq_group)
1274
+
1275
+ # Add the finished requests to the finished requests list.
1276
+ # This list will be used to update the Mamba cache in the
1277
+ # next step.
1278
+ self._finished_requests_ids.append(seq_group.request_id)
1279
+
1280
+ # Free finished seqs
1281
+ self._free_finished_seqs(seq_group)
1282
+
1283
def free_finished_seq_groups(self) -> None:
1284
remaining: Deque[SequenceGroup] = deque()
1285
for seq_group in self.running:
1270
- if seq_group.is_finished():
1271
- # Free cross-attention block table, if it exists
1272
- self._free_seq_group_cross_attn_blocks(seq_group)
1273
- # Add the finished requests to the finished requests list.
1274
- # This list will be used to update the Mamba cache in the
1275
- # next step.
1276
- self._finished_requests_ids.append(seq_group.request_id)
1277
- else:
1286
+ self._free_finished_seq_group(seq_group)
1287
+ if not seq_group.is_finished():
1288
remaining.append(seq_group)
1289
1280
- # Free finished seqs
1281
- self._free_finished_seqs(seq_group)
1282
-
1290
self.running = remaining
1291
1292
# Handle async stopped sequence groups
vllm/engine/async_llm_engine.py
+12
-12
@@ -342,17 +342,17 @@ class _AsyncLLMEngine(LLMEngine):
342
virtual_engine]
343
344
# Execute the model.
345
- output = await self.model_executor.execute_model_async(
345
+ outputs = await self.model_executor.execute_model_async(
346
execute_model_req)
347
348
# we need to do this here so that last step's sampled_token_ids can
349
# be passed to the next iteration for PP.
350
if self.scheduler_config.is_multi_step:
351
- self._update_cached_scheduler_output(virtual_engine, output)
351
+ self._update_cached_scheduler_output(virtual_engine, outputs)
352
else:
353
if len(ctx.output_queue) > 0:
354
self._process_model_outputs(ctx=ctx)
355
- output = []
355
+ outputs = []
356
357
# Finish the current step for all the sequence groups.
358
if self.scheduler_config.is_multi_step:
@@ -365,25 +365,25 @@ class _AsyncLLMEngine(LLMEngine):
365
self.cached_scheduler_outputs[
366
virtual_engine] = SchedulerOutputState()
367
368
- is_async = allow_async_output_proc
369
- is_last_step = True
370
- ctx.output_queue.append(
371
- (output, seq_group_metadata_list, scheduler_outputs, is_async,
372
- is_last_step))
368
+ ctx.append_output(outputs=outputs,
369
+ seq_group_metadata_list=seq_group_metadata_list,
370
+ scheduler_outputs=scheduler_outputs,
371
+ is_async=allow_async_output_proc,
372
+ is_last_step=True)
373
374
- if output and allow_async_output_proc:
374
+ if outputs and allow_async_output_proc:
375
assert len(
376
- output
376
+ outputs
377
) == 1, "Async postprocessor expects only a single output set"
378
self._advance_to_next_step(
379
- output[0], seq_group_metadata_list,
379
+ outputs[0], seq_group_metadata_list,
380
scheduler_outputs.scheduled_seq_groups)
381
382
if not allow_async_output_proc:
383
self._process_model_outputs(ctx=ctx)
384
385
# Log stats.
386
- self.do_log_stats(scheduler_outputs, output)
386
+ self.do_log_stats(scheduler_outputs, outputs)
387
388
# Tracing
389
self.do_tracing(scheduler_outputs)
vllm/engine/llm_engine.py
+99
-50
@@ -2,9 +2,9 @@ import functools
2
import time
3
from collections import deque
4
from contextlib import contextmanager
5
-from dataclasses import dataclass, field
5
+from dataclasses import dataclass
6
from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7
- Mapping, Optional)
7
+ Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
from typing import Set, Tuple, Type, Union
10
@@ -90,17 +90,36 @@ class SchedulerOutputState:
90
last_output: Optional[SamplerOutput] = None
91
92
93
-@dataclass
93
+class OutputData(NamedTuple):
94
+ outputs: List[SamplerOutput]
95
+ seq_group_metadata_list: List[SequenceGroupMetadata]
96
+ scheduler_outputs: SchedulerOutputs
97
+ is_async: bool
98
+ is_last_step: bool
99
+ skip: List[int]
100
+
101
+
102
class SchedulerContext:
95
- output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
96
- List[SequenceGroupMetadata], SchedulerOutputs,
97
- bool,
98
- bool]] = field(default_factory=lambda: deque())
99
- request_outputs: List[Union[RequestOutput,
100
- EmbeddingRequestOutput]] = field(
101
- default_factory=lambda: [])
102
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
103
- scheduler_outputs: Optional[SchedulerOutputs] = None
103
+
104
+ def __init__(self):
105
+ self.output_queue: Deque[OutputData] = deque()
106
+ self.request_outputs: List[Union[RequestOutput,
107
+ EmbeddingRequestOutput]] = []
108
+ self.seq_group_metadata_list: Optional[
109
+ List[SequenceGroupMetadata]] = None
110
+ self.scheduler_outputs: Optional[SchedulerOutputs] = None
111
+
112
+ def append_output(self, outputs: List[SamplerOutput],
113
+ seq_group_metadata_list: List[SequenceGroupMetadata],
114
+ scheduler_outputs: SchedulerOutputs, is_async: bool,
115
+ is_last_step: bool):
116
+ self.output_queue.append(
117
+ OutputData(outputs=outputs,
118
+ seq_group_metadata_list=seq_group_metadata_list,
119
+ scheduler_outputs=scheduler_outputs,
120
+ is_async=is_async,
121
+ is_last_step=is_last_step,
122
+ skip=[]))
123
124
125
class LLMEngine:
@@ -1246,23 +1265,15 @@ class LLMEngine:
1265
1266
return
1267
1249
- def _process_model_outputs(self, ctx: SchedulerContext) -> None:
1250
- """Apply the model output to the sequences in the scheduled seq groups.
1268
+ def _process_model_outputs(self,
1269
+ ctx: SchedulerContext,
1270
+ request_id: Optional[str] = None) -> None:
1271
+ """Apply the model output to the sequences in the scheduled seq groups
1272
+ and return responses.
1273
1252
- virtual_engine: The engine id to operate on
1274
+ ctx: The virtual engine context to work on
1275
+ request_id: If provided, then only this request is going to be processed
1276
1254
- is_async: Indicates whether this postprocessor runs in
1255
- parallel with the GPU forward pass and is processing
1256
- tokens from the previous step. If this is true, then
1257
- no tokens need to be appended since it is already done
1258
- externally (before the next schedule() call)
1259
-
1260
- sampler_output: Used with multi-step execution to provide
1261
- sampler_output of each step
1262
- is_last_output: Used with multi-step execution to indicate
1263
- the last step (of each multi-step group)
1264
-
1265
- Returns RequestOutputs that can be returned to the client.
1277
"""
1278
now = time.time()
1279
@@ -1270,9 +1281,14 @@ class LLMEngine:
1281
return None
1282
1283
# Get pending async postprocessor
1273
- (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1274
- is_last_step) = ctx.output_queue.popleft()
1275
- assert outputs is not None
1284
+ if request_id:
1285
+ # When we process only one request, no pop is required
1286
+ # (since later we will process all of the rest)
1287
+ (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1288
+ is_last_step, skip) = ctx.output_queue[0]
1289
+ else:
1290
+ (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1291
+ is_last_step, skip) = ctx.output_queue.popleft()
1292
1293
# Sanity check
1294
assert len(seq_group_metadata_list) == len(
@@ -1286,9 +1302,30 @@ class LLMEngine:
1302
else:
1303
outputs_by_sequence_group = outputs
1304
1305
+ # Determine the requests we need to operate on
1306
+ if request_id:
1307
+ indices = []
1308
+ for i, seq_group_meta in enumerate(seq_group_metadata_list):
1309
+ if seq_group_meta.request_id == request_id:
1310
+ assert i not in skip # Cannot be called twice
1311
+ indices.append(i)
1312
+ break
1313
+
1314
+ # If the request_id was not found, then it means that
1315
+ # this is a new request that has no pending async
1316
+ # postprocessor
1317
+ if not indices:
1318
+ return
1319
+ else:
1320
+ indices = range(len(seq_group_metadata_list)) # type: ignore
1321
+
1322
finished_before: List[int] = []
1323
finished_now: List[int] = []
1291
- for i, seq_group_meta in enumerate(seq_group_metadata_list):
1324
+ for i in indices:
1325
+ if i in skip:
1326
+ continue
1327
+
1328
+ seq_group_meta = seq_group_metadata_list[i]
1329
scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1330
1331
seq_group = scheduled_seq_group.seq_group
@@ -1343,6 +1380,18 @@ class LLMEngine:
1380
request_output = RequestOutputFactory.create(seq_group)
1381
ctx.request_outputs.append(request_output)
1382
1383
+ # When we process a single request, we skip it for the next time,
1384
+ # and invoke the request output callback (if there was final output)
1385
+ if request_id:
1386
+ assert len(indices) == 1
1387
+ skip.append(indices[0])
1388
+
1389
+ if (finished_now
1390
+ and self.process_request_outputs_callback is not None):
1391
+ self.process_request_outputs_callback(ctx.request_outputs)
1392
+ ctx.request_outputs.clear()
1393
+ return
1394
+
1395
# Free currently finished requests
1396
if finished_now:
1397
for scheduler in self.scheduler:
@@ -1354,17 +1403,16 @@ class LLMEngine:
1403
if (finished_now
1404
and self.process_request_outputs_callback is not None):
1405
self.process_request_outputs_callback(ctx.request_outputs)
1406
+ ctx.request_outputs.clear()
1407
return
1408
1409
# Create the outputs
1360
- # Note: scheduled_seq_groups and seq_group_metadata_list
1361
- # must match with the indices
1362
- for i, scheduled_seq_group in enumerate(
1363
- scheduler_outputs.scheduled_seq_groups):
1364
-
1365
- if i in finished_before or i in finished_now:
1410
+ for i in indices:
1411
+ if i in skip or i in finished_before or i in finished_now:
1412
continue # Avoids double processing
1413
1414
+ scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1415
+
1416
seq_group = scheduled_seq_group.seq_group
1417
seq_group.maybe_set_first_token_time(now)
1418
if (seq_group.is_finished()
@@ -1380,6 +1428,7 @@ class LLMEngine:
1428
if (ctx.request_outputs
1429
and self.process_request_outputs_callback is not None):
1430
self.process_request_outputs_callback(ctx.request_outputs)
1431
+ ctx.request_outputs.clear()
1432
1433
# For async case, we need to record the stats here.
1434
# For non-async case, the stats are done in the
@@ -1548,20 +1597,20 @@ class LLMEngine:
1597
execute_model_req.async_callback = self.async_callbacks[
1598
virtual_engine]
1599
1551
- output = self.model_executor.execute_model(
1600
+ outputs = self.model_executor.execute_model(
1601
execute_model_req=execute_model_req)
1602
1603
# We need to do this here so that last step's sampled_token_ids can
1604
# be passed to the next iteration for PP.
1605
if self.scheduler_config.is_multi_step:
1557
- self._update_cached_scheduler_output(virtual_engine, output)
1606
+ self._update_cached_scheduler_output(virtual_engine, outputs)
1607
else:
1608
# Nothing scheduled => If there is pending async postprocessor,
1609
# then finish it here.
1610
if len(ctx.output_queue) > 0:
1611
self._process_model_outputs(ctx=ctx)
1612
# No outputs in this case
1564
- output = []
1613
+ outputs = []
1614
1615
# Finish the current step for all the sequence groups.
1616
if self.scheduler_config.is_multi_step:
@@ -1574,18 +1623,18 @@ class LLMEngine:
1623
self.cached_scheduler_outputs[0] = SchedulerOutputState()
1624
1625
# Add results to the output_queue
1577
- is_async = allow_async_output_proc
1578
- is_last_step = True
1579
- ctx.output_queue.append(
1580
- (output, seq_group_metadata_list, scheduler_outputs, is_async,
1581
- is_last_step))
1582
-
1583
- if output and allow_async_output_proc:
1584
- assert len(output) == 1, (
1626
+ ctx.append_output(outputs=outputs,
1627
+ seq_group_metadata_list=seq_group_metadata_list,
1628
+ scheduler_outputs=scheduler_outputs,
1629
+ is_async=allow_async_output_proc,
1630
+ is_last_step=True)
1631
+
1632
+ if outputs and allow_async_output_proc:
1633
+ assert len(outputs) == 1, (
1634
"Async postprocessor expects only a single output set")
1635
1636
self._advance_to_next_step(
1588
- output[0], seq_group_metadata_list,
1637
+ outputs[0], seq_group_metadata_list,
1638
scheduler_outputs.scheduled_seq_groups)
1639
1640
# Check if need to run the usual non-async path
@@ -1593,7 +1642,7 @@ class LLMEngine:
1642
self._process_model_outputs(ctx=ctx)
1643
1644
# Log stats.
1596
- self.do_log_stats(scheduler_outputs, output)
1645
+ self.do_log_stats(scheduler_outputs, outputs)
1646
1647
# Tracing
1648
self.do_tracing(scheduler_outputs)
vllm/worker/multi_step_model_runner.py
+14
-12
@@ -274,12 +274,13 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
274
self.pinned_sampled_token_ids)
275
if model_output.pythonized:
276
ctx = output_proc_callback.keywords["ctx"]
277
- is_async = False
278
- is_last_step = False
279
- ctx.output_queue.append(
280
- ([model_output.sampler_output
281
- ], ctx.seq_group_metadata_list,
282
- ctx.scheduler_outputs, is_async, is_last_step))
277
+ ctx.append_output(
278
+ outputs=[model_output.sampler_output],
279
+ seq_group_metadata_list=ctx.seq_group_metadata_list,
280
+ scheduler_outputs=ctx.scheduler_outputs,
281
+ is_async=False,
282
+ is_last_step=False)
283
+
284
output_proc_callback()
285
else:
286
cont = False
@@ -319,12 +320,13 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
320
if not is_last_step:
321
ctx = output_proc_callback.keywords[ # type: ignore
322
"ctx"] # type: ignore
322
- is_async = False
323
- is_last_step = False
324
- ctx.output_queue.append(
325
- ([output.sampler_output
326
- ], ctx.seq_group_metadata_list,
327
- ctx.scheduler_outputs, is_async, is_last_step))
323
+ ctx.append_output(
324
+ outputs=[output.sampler_output],
325
+ seq_group_metadata_list=ctx.
326
+ seq_group_metadata_list,
327
+ scheduler_outputs=ctx.scheduler_outputs,
328
+ is_async=False,
329
+ is_last_step=False)
330
else:
331
outputs.append(output.sampler_output)
332
else: