A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Bugfix] fix encoder cache leak of waiting requests in scheduler to solve stuck in CPU scheduling (#31857)
Signed-off-by: frelam <frelam112233@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Frelam committed
Jan 10, 2026 at 14:27 UTC
a01a1c0d691f89fa4788102eb0bd9b15607c1cca
1 file changed
+5
vllm/v1/core/sched/scheduler.py
+5
@@ -598,6 +598,11 @@ class Scheduler(SchedulerInterface):
598
599
if new_blocks is None:
600
# The request cannot be scheduled.
601
+
602
+ # NOTE: we need to untouch the request from the encode cache
603
+ # manager
604
+ if request.has_encoder_inputs:
605
+ self.encoder_cache_manager.free(request)
606
break
607
608
# KVTransfer: the connector uses this info to determine