A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix an endless loop issue when engine_step throws a RuntimeError (#339)
coolcloudcol committed
Jul 4, 2023 at 06:22 UTC
7717d0838b2b74313184a462bb6e08711a5950c6
1 file changed
+5
-1
vllm/engine/async_llm_engine.py
+5
-1
@@ -144,7 +144,11 @@ class AsyncLLMEngine:
144
145
# Kick the engine if the engine is not running.
146
if not self.is_engine_running:
147
- await self.engine_step(request_id)
147
+ try:
148
+ await self.engine_step(request_id)
149
+ except RuntimeError as e:
150
+ await self.abort(request_id)
151
+ raise e
152
153
# Wait for new output. The group_event will be set in engine_step
154
# when there is new output available for the sequence group.