A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
Dump input metadata on crash for async scheduling (#21258)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jul 23, 2025 at 21:10 UTC
dc2f159f8ae6d31faab6b769b972d494c9717f39
1 file changed
+14
-4
vllm/v1/engine/core.py
+14
-4
@@ -234,9 +234,14 @@ class EngineCore:
234
self.scheduler.finish_requests(request_ids,
235
RequestStatus.FINISHED_ABORTED)
236
237
- def execute_model(self, scheduler_output: SchedulerOutput):
237
+ def execute_model_with_error_logging(
238
+ self,
239
+ model_fn: Callable[[SchedulerOutput], ModelRunnerOutput],
240
+ scheduler_output: SchedulerOutput,
241
+ ) -> ModelRunnerOutput:
242
+ """Execute the model and log detailed info on failure."""
243
try:
239
- return self.model_executor.execute_model(scheduler_output)
244
+ return model_fn(scheduler_output)
245
except Exception as err:
246
# We do not want to catch BaseException here since we're only
247
# interested in dumping info when the exception is due to an
@@ -259,7 +264,9 @@ class EngineCore:
264
if not self.scheduler.has_requests():
265
return {}, False
266
scheduler_output = self.scheduler.schedule()
262
- model_output = self.execute_model(scheduler_output)
267
+ model_output = self.execute_model_with_error_logging(
268
+ self.model_executor.execute_model, # type: ignore
269
+ scheduler_output)
270
engine_core_outputs = self.scheduler.update_from_output(
271
scheduler_output, model_output) # type: ignore
272
@@ -306,8 +313,11 @@ class EngineCore:
313
# so we need more work.
314
if not scheduled_batch and not self.batch_queue.empty():
315
future, scheduler_output = self.batch_queue.get_nowait()
316
+
317
# Blocking until the first result is available.
310
- model_output = future.result()
318
+ model_output = self.execute_model_with_error_logging(
319
+ lambda _: future.result(), scheduler_output)
320
+
321
self.batch_queue.task_done()
322
engine_core_outputs = (self.scheduler.update_from_output(
323
scheduler_output, model_output))