A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[BugFix] Add sleep to fix tight loop and release GIL (#29476)
Signed-off-by: alec-flowers <aflowers@nvidia.com> Signed-off-by: Alec <35311602+alec-flowers@users.noreply.github.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Alec committed
Dec 18, 2025 at 09:52 UTC
62be3670cb97e7196c30be26fe347d12d183429c
1 file changed
+7
vllm/v1/engine/core.py
+7
@@ -923,6 +923,13 @@ class EngineCoreProc(EngineCore):
923
# Post-step hook.
924
self.post_step(model_executed)
925
926
+ # If no model execution happened but there are waiting requests
927
+ # (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow
928
+ # background threads (like NIXL handshake) to make progress.
929
+ # Without this, the tight polling loop can starve background threads.
930
+ if not model_executed and self.scheduler.has_unfinished_requests():
931
+ time.sleep(0.001)
932
+
933
return model_executed
934
935
def _handle_client_request(