@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f83ddd4d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix][Frontend/Core] Don't log exception when AsyncLLMEngine gracefully shuts down. (#5290)

Alex Wu committed Jun 5, 2024 at 15:18 UTC 0f83ddd4d71ce1a80cfcaff085b40fef83d1a750
1 file changed +19 -11
vllm/engine/async_llm_engine.py
+19 -11
@@ -29,23 +29,32 @@ class AsyncEngineDeadError(RuntimeError):
29 pass
30
31
32 -def _raise_exception_on_finish(
33 - task: asyncio.Task, error_callback: Callable[[Exception],
34 - None]) -> None:
35 - msg = ("Task finished unexpectedly. This should never happen! "
36 - "Please open an issue on Github.")
32 +def _log_task_completion(task: asyncio.Task,
33 + error_callback: Callable[[Exception], None]) -> None:
34 + """This function is only intended for the `engine.run_engine_loop()` task.
35 +
36 + In particular, that task runs a `while True` loop that can only exit if
37 + there is an exception.
38 + """
39
40 exception = None
41 try:
40 - task.result()
41 - # NOTE: This will be thrown if task exits normally (which it should not)
42 - raise AsyncEngineDeadError(msg)
42 + return_value = task.result()
43 + raise AssertionError(
44 + f"The engine background task should never finish without an "
45 + f"exception. {return_value}")
46 + except asyncio.exceptions.CancelledError:
47 + # We assume that if the task is cancelled, we are gracefully shutting
48 + # down. This should only happen on program exit.
49 + logger.info("Engine is gracefully shutting down.")
50 except Exception as e:
51 exception = e
52 logger.error("Engine background task failed", exc_info=e)
53 error_callback(exception)
54 raise AsyncEngineDeadError(
48 - msg + " See stack trace above for the actual cause.") from e
55 + "Task finished unexpectedly. This should never happen! "
56 + "Please open an issue on Github. See stack trace above for the"
57 + "actual cause.") from e
58
59
60 class AsyncStream:
@@ -438,8 +447,7 @@ class AsyncLLMEngine:
447 self._background_loop_unshielded = asyncio.get_event_loop(
448 ).create_task(self.run_engine_loop())
449 self._background_loop_unshielded.add_done_callback(
441 - partial(_raise_exception_on_finish,
442 - error_callback=self._error_callback))
450 + partial(_log_task_completion, error_callback=self._error_callback))
451 self.background_loop = asyncio.shield(self._background_loop_unshielded)
452
453 def _init_engine(self, *args,