A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] add `dead_error` property to engine client (#8574)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Sep 18, 2024 at 16:10 UTC
0d47bf3bf40edfe9fcfd7e5cd909388497535bc5
1 file changed
+7
vllm/engine/multiprocessing/client.py
+7
@@ -380,6 +380,13 @@ class MQLLMEngineClient:
380
def errored(self) -> bool:
381
return self._errored_with is not None
382
383
+ @property
384
+ def dead_error(self) -> BaseException:
385
+ if self._errored_with is not None:
386
+ return ENGINE_DEAD_ERROR(self._errored_with)
387
+ else:
388
+ return ENGINE_DEAD_ERROR()
389
+
390
async def generate(
391
self,
392
inputs: PromptInputs,