@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ef41b8476
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix async postprocessor in case of preemption (#8267)

Alexander Matveev committed Sep 8, 2024 at 00:01 UTC 4ef41b84766670c1bd8079f58d35bf32b5bcb3ab
4 files changed +172 -114
vllm/core/scheduler.py
+47 -40
@@ -537,13 +537,6 @@ class Scheduler:
537 preempted: List[SequenceGroup] = ret.preempted
538 swapped_out: List[SequenceGroup] = ret.swapped_out
539
540 - # NOTE(woosuk): Preemption happens only when there is no available slot
541 - # to keep all the sequence groups in the RUNNING state.
542 -
543 - # Store original running requests for the case of async + preemption
544 - if self.use_async_output_proc:
545 - orig_running = self.running.copy()
546 -
540 running_queue = self.running
541 assert len(self._async_stopped) == 0
542 while running_queue:
@@ -552,6 +545,7 @@ class Scheduler:
545 seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
546
547 if num_running_tokens == 0:
548 + # No budget => Stop
549 break
550
551 running_queue.popleft()
@@ -565,18 +559,8 @@ class Scheduler:
559 self._async_stopped.append(seq_group)
560 continue
561
568 - # With async postprocessor, when preemption kicks in, we need
569 - # first to drain the async postprocessor, so that all async
570 - # block_table freeing is applied before the preemption freeing
571 - # is applied.
572 - if self.use_async_output_proc and not self._can_append_slots(
573 - seq_group):
574 - tmp = self.running
575 - self.running = orig_running
576 - assert self.output_proc_callback is not None
577 - self.output_proc_callback()
578 - self.running = tmp
579 -
562 + # NOTE(woosuk): Preemption happens only when there is no available
563 + # slot to keep all the sequence groups in the RUNNING state.
564 while not self._can_append_slots(seq_group):
565 budget.subtract_num_batched_tokens(seq_group.request_id,
566 num_running_tokens)
@@ -588,24 +572,43 @@ class Scheduler:
572 and seq_group.lora_int_id in curr_loras):
573 curr_loras.remove(seq_group.lora_int_id)
574
575 + # Determine victim sequence
576 + cont_loop = True
577 if running_queue:
592 - # Preempt the lowest-priority sequence groups.
578 + # Preempt the lowest-priority sequence group.
579 victim_seq_group = running_queue.pop()
580 + else:
581 + # No other sequence group can be preempted.
582 + # Preempt the current sequence group.
583 + # Note: This is also where we stop this loop
584 + # (since there is nothing else to preempt)
585 + victim_seq_group = seq_group
586 + cont_loop = False
587 +
588 + # With async postprocessor, before preempting a sequence
589 + # we need to ensure it has no pending async postprocessor
590 + do_preempt = True
591 + if self.use_async_output_proc:
592 + assert self.output_proc_callback is not None
593 + self.output_proc_callback(
594 + request_id=victim_seq_group.request_id)
595 +
596 + # It may be that the async pending "victim_seq_group"
597 + # becomes finished, in which case we simply free it.
598 + if victim_seq_group.is_finished():
599 + self._free_finished_seq_group(victim_seq_group)
600 + do_preempt = False
601 +
602 + # Do preemption
603 + if do_preempt:
604 preempted_mode = self._preempt(victim_seq_group,
605 blocks_to_swap_out)
606 if preempted_mode == PreemptionMode.RECOMPUTE:
607 preempted.append(victim_seq_group)
608 else:
609 swapped_out.append(victim_seq_group)
600 - else:
601 - # No other sequence groups can be preempted.
602 - # Preempt the current sequence group.
603 - preempted_mode = self._preempt(seq_group,
604 - blocks_to_swap_out)
605 - if preempted_mode == PreemptionMode.RECOMPUTE:
606 - preempted.append(seq_group)
607 - else:
608 - swapped_out.append(seq_group)
610 +
611 + if not cont_loop:
612 break
613 else:
614 self._append_slots(seq_group, blocks_to_copy)
@@ -1264,22 +1267,26 @@ class Scheduler:
1267 if seq.is_finished():
1268 self.free_seq(seq)
1269
1270 + def _free_finished_seq_group(self, seq_group: SequenceGroup) -> None:
1271 + if seq_group.is_finished():
1272 + # Free cross-attention block table, if it exists
1273 + self._free_seq_group_cross_attn_blocks(seq_group)
1274 +
1275 + # Add the finished requests to the finished requests list.
1276 + # This list will be used to update the Mamba cache in the
1277 + # next step.
1278 + self._finished_requests_ids.append(seq_group.request_id)
1279 +
1280 + # Free finished seqs
1281 + self._free_finished_seqs(seq_group)
1282 +
1283 def free_finished_seq_groups(self) -> None:
1284 remaining: Deque[SequenceGroup] = deque()
1285 for seq_group in self.running:
1270 - if seq_group.is_finished():
1271 - # Free cross-attention block table, if it exists
1272 - self._free_seq_group_cross_attn_blocks(seq_group)
1273 - # Add the finished requests to the finished requests list.
1274 - # This list will be used to update the Mamba cache in the
1275 - # next step.
1276 - self._finished_requests_ids.append(seq_group.request_id)
1277 - else:
1286 + self._free_finished_seq_group(seq_group)
1287 + if not seq_group.is_finished():
1288 remaining.append(seq_group)
1289
1280 - # Free finished seqs
1281 - self._free_finished_seqs(seq_group)
1282 -
1290 self.running = remaining
1291
1292 # Handle async stopped sequence groups
vllm/engine/async_llm_engine.py
+12 -12
@@ -342,17 +342,17 @@ class _AsyncLLMEngine(LLMEngine):
342 virtual_engine]
343
344 # Execute the model.
345 - output = await self.model_executor.execute_model_async(
345 + outputs = await self.model_executor.execute_model_async(
346 execute_model_req)
347
348 # we need to do this here so that last step's sampled_token_ids can
349 # be passed to the next iteration for PP.
350 if self.scheduler_config.is_multi_step:
351 - self._update_cached_scheduler_output(virtual_engine, output)
351 + self._update_cached_scheduler_output(virtual_engine, outputs)
352 else:
353 if len(ctx.output_queue) > 0:
354 self._process_model_outputs(ctx=ctx)
355 - output = []
355 + outputs = []
356
357 # Finish the current step for all the sequence groups.
358 if self.scheduler_config.is_multi_step:
@@ -365,25 +365,25 @@ class _AsyncLLMEngine(LLMEngine):
365 self.cached_scheduler_outputs[
366 virtual_engine] = SchedulerOutputState()
367
368 - is_async = allow_async_output_proc
369 - is_last_step = True
370 - ctx.output_queue.append(
371 - (output, seq_group_metadata_list, scheduler_outputs, is_async,
372 - is_last_step))
368 + ctx.append_output(outputs=outputs,
369 + seq_group_metadata_list=seq_group_metadata_list,
370 + scheduler_outputs=scheduler_outputs,
371 + is_async=allow_async_output_proc,
372 + is_last_step=True)
373
374 - if output and allow_async_output_proc:
374 + if outputs and allow_async_output_proc:
375 assert len(
376 - output
376 + outputs
377 ) == 1, "Async postprocessor expects only a single output set"
378 self._advance_to_next_step(
379 - output[0], seq_group_metadata_list,
379 + outputs[0], seq_group_metadata_list,
380 scheduler_outputs.scheduled_seq_groups)
381
382 if not allow_async_output_proc:
383 self._process_model_outputs(ctx=ctx)
384
385 # Log stats.
386 - self.do_log_stats(scheduler_outputs, output)
386 + self.do_log_stats(scheduler_outputs, outputs)
387
388 # Tracing
389 self.do_tracing(scheduler_outputs)
vllm/engine/llm_engine.py
+99 -50
@@ -2,9 +2,9 @@ import functools
2 import time
3 from collections import deque
4 from contextlib import contextmanager
5 -from dataclasses import dataclass, field
5 +from dataclasses import dataclass
6 from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7 - Mapping, Optional)
7 + Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 from typing import Set, Tuple, Type, Union
10
@@ -90,17 +90,36 @@ class SchedulerOutputState:
90 last_output: Optional[SamplerOutput] = None
91
92
93 -@dataclass
93 +class OutputData(NamedTuple):
94 + outputs: List[SamplerOutput]
95 + seq_group_metadata_list: List[SequenceGroupMetadata]
96 + scheduler_outputs: SchedulerOutputs
97 + is_async: bool
98 + is_last_step: bool
99 + skip: List[int]
100 +
101 +
102 class SchedulerContext:
95 - output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
96 - List[SequenceGroupMetadata], SchedulerOutputs,
97 - bool,
98 - bool]] = field(default_factory=lambda: deque())
99 - request_outputs: List[Union[RequestOutput,
100 - EmbeddingRequestOutput]] = field(
101 - default_factory=lambda: [])
102 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
103 - scheduler_outputs: Optional[SchedulerOutputs] = None
103 +
104 + def __init__(self):
105 + self.output_queue: Deque[OutputData] = deque()
106 + self.request_outputs: List[Union[RequestOutput,
107 + EmbeddingRequestOutput]] = []
108 + self.seq_group_metadata_list: Optional[
109 + List[SequenceGroupMetadata]] = None
110 + self.scheduler_outputs: Optional[SchedulerOutputs] = None
111 +
112 + def append_output(self, outputs: List[SamplerOutput],
113 + seq_group_metadata_list: List[SequenceGroupMetadata],
114 + scheduler_outputs: SchedulerOutputs, is_async: bool,
115 + is_last_step: bool):
116 + self.output_queue.append(
117 + OutputData(outputs=outputs,
118 + seq_group_metadata_list=seq_group_metadata_list,
119 + scheduler_outputs=scheduler_outputs,
120 + is_async=is_async,
121 + is_last_step=is_last_step,
122 + skip=[]))
123
124
125 class LLMEngine:
@@ -1246,23 +1265,15 @@ class LLMEngine:
1265
1266 return
1267
1249 - def _process_model_outputs(self, ctx: SchedulerContext) -> None:
1250 - """Apply the model output to the sequences in the scheduled seq groups.
1268 + def _process_model_outputs(self,
1269 + ctx: SchedulerContext,
1270 + request_id: Optional[str] = None) -> None:
1271 + """Apply the model output to the sequences in the scheduled seq groups
1272 + and return responses.
1273
1252 - virtual_engine: The engine id to operate on
1274 + ctx: The virtual engine context to work on
1275 + request_id: If provided, then only this request is going to be processed
1276
1254 - is_async: Indicates whether this postprocessor runs in
1255 - parallel with the GPU forward pass and is processing
1256 - tokens from the previous step. If this is true, then
1257 - no tokens need to be appended since it is already done
1258 - externally (before the next schedule() call)
1259 -
1260 - sampler_output: Used with multi-step execution to provide
1261 - sampler_output of each step
1262 - is_last_output: Used with multi-step execution to indicate
1263 - the last step (of each multi-step group)
1264 -
1265 - Returns RequestOutputs that can be returned to the client.
1277 """
1278 now = time.time()
1279
@@ -1270,9 +1281,14 @@ class LLMEngine:
1281 return None
1282
1283 # Get pending async postprocessor
1273 - (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1274 - is_last_step) = ctx.output_queue.popleft()
1275 - assert outputs is not None
1284 + if request_id:
1285 + # When we process only one request, no pop is required
1286 + # (since later we will process all of the rest)
1287 + (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1288 + is_last_step, skip) = ctx.output_queue[0]
1289 + else:
1290 + (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1291 + is_last_step, skip) = ctx.output_queue.popleft()
1292
1293 # Sanity check
1294 assert len(seq_group_metadata_list) == len(
@@ -1286,9 +1302,30 @@ class LLMEngine:
1302 else:
1303 outputs_by_sequence_group = outputs
1304
1305 + # Determine the requests we need to operate on
1306 + if request_id:
1307 + indices = []
1308 + for i, seq_group_meta in enumerate(seq_group_metadata_list):
1309 + if seq_group_meta.request_id == request_id:
1310 + assert i not in skip # Cannot be called twice
1311 + indices.append(i)
1312 + break
1313 +
1314 + # If the request_id was not found, then it means that
1315 + # this is a new request that has no pending async
1316 + # postprocessor
1317 + if not indices:
1318 + return
1319 + else:
1320 + indices = range(len(seq_group_metadata_list)) # type: ignore
1321 +
1322 finished_before: List[int] = []
1323 finished_now: List[int] = []
1291 - for i, seq_group_meta in enumerate(seq_group_metadata_list):
1324 + for i in indices:
1325 + if i in skip:
1326 + continue
1327 +
1328 + seq_group_meta = seq_group_metadata_list[i]
1329 scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1330
1331 seq_group = scheduled_seq_group.seq_group
@@ -1343,6 +1380,18 @@ class LLMEngine:
1380 request_output = RequestOutputFactory.create(seq_group)
1381 ctx.request_outputs.append(request_output)
1382
1383 + # When we process a single request, we skip it for the next time,
1384 + # and invoke the request output callback (if there was final output)
1385 + if request_id:
1386 + assert len(indices) == 1
1387 + skip.append(indices[0])
1388 +
1389 + if (finished_now
1390 + and self.process_request_outputs_callback is not None):
1391 + self.process_request_outputs_callback(ctx.request_outputs)
1392 + ctx.request_outputs.clear()
1393 + return
1394 +
1395 # Free currently finished requests
1396 if finished_now:
1397 for scheduler in self.scheduler:
@@ -1354,17 +1403,16 @@ class LLMEngine:
1403 if (finished_now
1404 and self.process_request_outputs_callback is not None):
1405 self.process_request_outputs_callback(ctx.request_outputs)
1406 + ctx.request_outputs.clear()
1407 return
1408
1409 # Create the outputs
1360 - # Note: scheduled_seq_groups and seq_group_metadata_list
1361 - # must match with the indices
1362 - for i, scheduled_seq_group in enumerate(
1363 - scheduler_outputs.scheduled_seq_groups):
1364 -
1365 - if i in finished_before or i in finished_now:
1410 + for i in indices:
1411 + if i in skip or i in finished_before or i in finished_now:
1412 continue # Avoids double processing
1413
1414 + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1415 +
1416 seq_group = scheduled_seq_group.seq_group
1417 seq_group.maybe_set_first_token_time(now)
1418 if (seq_group.is_finished()
@@ -1380,6 +1428,7 @@ class LLMEngine:
1428 if (ctx.request_outputs
1429 and self.process_request_outputs_callback is not None):
1430 self.process_request_outputs_callback(ctx.request_outputs)
1431 + ctx.request_outputs.clear()
1432
1433 # For async case, we need to record the stats here.
1434 # For non-async case, the stats are done in the
@@ -1548,20 +1597,20 @@ class LLMEngine:
1597 execute_model_req.async_callback = self.async_callbacks[
1598 virtual_engine]
1599
1551 - output = self.model_executor.execute_model(
1600 + outputs = self.model_executor.execute_model(
1601 execute_model_req=execute_model_req)
1602
1603 # We need to do this here so that last step's sampled_token_ids can
1604 # be passed to the next iteration for PP.
1605 if self.scheduler_config.is_multi_step:
1557 - self._update_cached_scheduler_output(virtual_engine, output)
1606 + self._update_cached_scheduler_output(virtual_engine, outputs)
1607 else:
1608 # Nothing scheduled => If there is pending async postprocessor,
1609 # then finish it here.
1610 if len(ctx.output_queue) > 0:
1611 self._process_model_outputs(ctx=ctx)
1612 # No outputs in this case
1564 - output = []
1613 + outputs = []
1614
1615 # Finish the current step for all the sequence groups.
1616 if self.scheduler_config.is_multi_step:
@@ -1574,18 +1623,18 @@ class LLMEngine:
1623 self.cached_scheduler_outputs[0] = SchedulerOutputState()
1624
1625 # Add results to the output_queue
1577 - is_async = allow_async_output_proc
1578 - is_last_step = True
1579 - ctx.output_queue.append(
1580 - (output, seq_group_metadata_list, scheduler_outputs, is_async,
1581 - is_last_step))
1582 -
1583 - if output and allow_async_output_proc:
1584 - assert len(output) == 1, (
1626 + ctx.append_output(outputs=outputs,
1627 + seq_group_metadata_list=seq_group_metadata_list,
1628 + scheduler_outputs=scheduler_outputs,
1629 + is_async=allow_async_output_proc,
1630 + is_last_step=True)
1631 +
1632 + if outputs and allow_async_output_proc:
1633 + assert len(outputs) == 1, (
1634 "Async postprocessor expects only a single output set")
1635
1636 self._advance_to_next_step(
1588 - output[0], seq_group_metadata_list,
1637 + outputs[0], seq_group_metadata_list,
1638 scheduler_outputs.scheduled_seq_groups)
1639
1640 # Check if need to run the usual non-async path
@@ -1593,7 +1642,7 @@ class LLMEngine:
1642 self._process_model_outputs(ctx=ctx)
1643
1644 # Log stats.
1596 - self.do_log_stats(scheduler_outputs, output)
1645 + self.do_log_stats(scheduler_outputs, outputs)
1646
1647 # Tracing
1648 self.do_tracing(scheduler_outputs)
vllm/worker/multi_step_model_runner.py
+14 -12
@@ -274,12 +274,13 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
274 self.pinned_sampled_token_ids)
275 if model_output.pythonized:
276 ctx = output_proc_callback.keywords["ctx"]
277 - is_async = False
278 - is_last_step = False
279 - ctx.output_queue.append(
280 - ([model_output.sampler_output
281 - ], ctx.seq_group_metadata_list,
282 - ctx.scheduler_outputs, is_async, is_last_step))
277 + ctx.append_output(
278 + outputs=[model_output.sampler_output],
279 + seq_group_metadata_list=ctx.seq_group_metadata_list,
280 + scheduler_outputs=ctx.scheduler_outputs,
281 + is_async=False,
282 + is_last_step=False)
283 +
284 output_proc_callback()
285 else:
286 cont = False
@@ -319,12 +320,13 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
320 if not is_last_step:
321 ctx = output_proc_callback.keywords[ # type: ignore
322 "ctx"] # type: ignore
322 - is_async = False
323 - is_last_step = False
324 - ctx.output_queue.append(
325 - ([output.sampler_output
326 - ], ctx.seq_group_metadata_list,
327 - ctx.scheduler_outputs, is_async, is_last_step))
323 + ctx.append_output(
324 + outputs=[output.sampler_output],
325 + seq_group_metadata_list=ctx.
326 + seq_group_metadata_list,
327 + scheduler_outputs=ctx.scheduler_outputs,
328 + is_async=False,
329 + is_last_step=False)
330 else:
331 outputs.append(output.sampler_output)
332 else: