A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[BugFix]: fix engine timeout due to request abort (#6255)
Signed-off-by: yatta zhang <ytzhang01@foxmail.com> Signed-off-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com> Co-authored-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com>
pushan committed
Jul 11, 2024 at 21:46 UTC
546b101fa05043feb470513a778c31114ea3aa05
1 file changed
+3
-1
vllm/engine/async_llm_engine.py
+3
-1
@@ -553,11 +553,13 @@ class AsyncLLMEngine:
553
request_outputs = await self.engine.step_async(virtual_engine)
554
555
# Put the outputs into the corresponding streams.
556
+ finished = True
557
for request_output in request_outputs:
558
self._request_tracker.process_request_output(
559
request_output, verbose=self.log_requests)
560
+ finished = finished and request_output.finished
561
560
- return len(request_outputs) > 0
562
+ return not finished
563
564
async def _engine_abort(self, request_ids: Iterable[str]):
565
if self.engine_use_ray: