A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix][Frontend/Core] Don't log exception when AsyncLLMEngine gracefully shuts down. (#5290)
Alex Wu committed
Jun 5, 2024 at 15:18 UTC
0f83ddd4d71ce1a80cfcaff085b40fef83d1a750
1 file changed
+19
-11
vllm/engine/async_llm_engine.py
+19
-11
@@ -29,23 +29,32 @@ class AsyncEngineDeadError(RuntimeError):
29
pass
30
31
32
-def _raise_exception_on_finish(
33
- task: asyncio.Task, error_callback: Callable[[Exception],
34
- None]) -> None:
35
- msg = ("Task finished unexpectedly. This should never happen! "
36
- "Please open an issue on Github.")
32
+def _log_task_completion(task: asyncio.Task,
33
+ error_callback: Callable[[Exception], None]) -> None:
34
+ """This function is only intended for the `engine.run_engine_loop()` task.
35
+
36
+ In particular, that task runs a `while True` loop that can only exit if
37
+ there is an exception.
38
+ """
39
40
exception = None
41
try:
40
- task.result()
41
- # NOTE: This will be thrown if task exits normally (which it should not)
42
- raise AsyncEngineDeadError(msg)
42
+ return_value = task.result()
43
+ raise AssertionError(
44
+ f"The engine background task should never finish without an "
45
+ f"exception. {return_value}")
46
+ except asyncio.exceptions.CancelledError:
47
+ # We assume that if the task is cancelled, we are gracefully shutting
48
+ # down. This should only happen on program exit.
49
+ logger.info("Engine is gracefully shutting down.")
50
except Exception as e:
51
exception = e
52
logger.error("Engine background task failed", exc_info=e)
53
error_callback(exception)
54
raise AsyncEngineDeadError(
48
- msg + " See stack trace above for the actual cause.") from e
55
+ "Task finished unexpectedly. This should never happen! "
56
+ "Please open an issue on Github. See stack trace above for the"
57
+ "actual cause.") from e
58
59
60
class AsyncStream:
@@ -438,8 +447,7 @@ class AsyncLLMEngine:
447
self._background_loop_unshielded = asyncio.get_event_loop(
448
).create_task(self.run_engine_loop())
449
self._background_loop_unshielded.add_done_callback(
441
- partial(_raise_exception_on_finish,
442
- error_callback=self._error_callback))
450
+ partial(_log_task_completion, error_callback=self._error_callback))
451
self.background_loop = asyncio.shield(self._background_loop_unshielded)
452
453
def _init_engine(self, *args,