A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
[BugFix] Async scheduling: handle model forward errors more cleanly (#31611)
Signed-off-by: njhill <nickhill123@gmail.com>
Nick Hill committed
Jan 4, 2026 at 11:04 UTC
b53b89fdb3f4a857eabee5091187cfa937502711
1 file changed
+9
-16
vllm/v1/engine/core.py
+9
-16
@@ -179,7 +179,7 @@ class EngineCore:
179
# to eliminate pipeline bubbles.
180
self.batch_queue_size = self.model_executor.max_concurrent_batches
181
self.batch_queue: (
182
- deque[tuple[Future[ModelRunnerOutput], SchedulerOutput]] | None
182
+ deque[tuple[Future[ModelRunnerOutput], SchedulerOutput, Future[Any]]] | None
183
) = None
184
if self.batch_queue_size > 1:
185
logger.info("Batch queue is enabled with size %d", self.batch_queue_size)
@@ -337,16 +337,6 @@ class EngineCore:
337
)
338
raise err
339
340
- def _log_err_callback(self, scheduler_output: SchedulerOutput):
341
- """Log error details of a future that's not expected to return a result."""
342
-
343
- def callback(f, sched_output=scheduler_output):
344
- with self.log_error_detail(sched_output):
345
- result = f.result()
346
- assert result is None
347
-
348
- return callback
349
-
340
def step(self) -> tuple[dict[int, EngineCoreOutputs], bool]:
341
"""Schedule, execute, and make output.
342
@@ -423,8 +413,6 @@ class EngineCore:
413
# No sampling required (no requests scheduled).
414
future = cast(Future[ModelRunnerOutput], exec_future)
415
else:
426
- exec_future.add_done_callback(self._log_err_callback(scheduler_output))
427
-
416
if not scheduler_output.pending_structured_output_tokens:
417
# We aren't waiting for any tokens, get any grammar output
418
# and sample immediately.
@@ -441,7 +429,7 @@ class EngineCore:
429
430
if not deferred_scheduler_output:
431
# Add this step's future to the queue.
444
- batch_queue.appendleft((future, scheduler_output))
432
+ batch_queue.appendleft((future, scheduler_output, exec_future))
433
if (
434
model_executed
435
and len(batch_queue) < self.batch_queue_size
@@ -458,9 +446,14 @@ class EngineCore:
446
return None, False
447
448
# Block until the next result is available.
461
- future, scheduler_output = batch_queue.pop()
449
+ future, scheduler_output, exec_model_fut = batch_queue.pop()
450
with self.log_error_detail(scheduler_output):
451
model_output = future.result()
452
+ if model_output is None:
453
+ # None from sample_tokens() implies that the original execute_model()
454
+ # call failed - raise that exception.
455
+ exec_model_fut.result()
456
+ raise RuntimeError("unexpected error")
457
458
# Before processing the model output, process any aborts that happened
459
# during the model execution.
@@ -479,7 +472,7 @@ class EngineCore:
472
deferred_scheduler_output
473
)
474
future = self.model_executor.sample_tokens(grammar_output, non_block=True)
482
- batch_queue.appendleft((future, deferred_scheduler_output))
475
+ batch_queue.appendleft((future, deferred_scheduler_output, exec_future))
476
477
return engine_core_outputs, model_executed
478