@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 546b101fa0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[BugFix]: fix engine timeout due to request abort (#6255)

Signed-off-by: yatta zhang <ytzhang01@foxmail.com> Signed-off-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com> Co-authored-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com>

pushan committed Jul 11, 2024 at 21:46 UTC 546b101fa05043feb470513a778c31114ea3aa05
1 file changed +3 -1
vllm/engine/async_llm_engine.py
+3 -1
@@ -553,11 +553,13 @@ class AsyncLLMEngine:
553 request_outputs = await self.engine.step_async(virtual_engine)
554
555 # Put the outputs into the corresponding streams.
556 + finished = True
557 for request_output in request_outputs:
558 self._request_tracker.process_request_output(
559 request_output, verbose=self.log_requests)
560 + finished = finished and request_output.finished
561
560 - return len(request_outputs) > 0
562 + return not finished
563
564 async def _engine_abort(self, request_ids: Iterable[str]):
565 if self.engine_use_ray: