A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[BugFix] Ensure worker model loop is always stopped at the right time (#5987)
Nick Hill committed
Jul 1, 2024 at 16:17 UTC
c87ebc3ef9ae6e8d6babbca782510ff924b3abc7
1 file changed
+1
-1
vllm/engine/llm_engine.py
+1
-1
@@ -838,7 +838,7 @@ class LLMEngine:
838
# Tracing
839
self.do_tracing(scheduler_outputs)
840
841
- if not request_outputs:
841
+ if not self.has_unfinished_requests():
842
# Stop the execute model loop in parallel workers until there are
843
# more requests to process. This avoids waiting indefinitely in
844
# torch.distributed ops which may otherwise timeout, and unblocks