A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[v1][P/D] Fix a edge case in kv cache schedule (#19182)
Co-authored-by: jinghui <jinghui@fb.com>
Jinghui Zhang committed
Jun 5, 2025 at 23:32 UTC
90b78ec5f97384f10cb210368421ae3da17a3823
1 file changed
+2
vllm/v1/core/sched/scheduler.py
+2
@@ -1009,6 +1009,8 @@ class Scheduler(SchedulerInterface):
1009
# Now that the blocks are ready, actually cache them.
1010
block_ids = self.kv_cache_manager.get_block_ids(request.request_id)[0]
1011
num_computed_tokens = len(block_ids) * self.block_size
1012
+ # Handle the case where num request tokens less then one block.
1013
+ num_computed_tokens = min(num_computed_tokens, request.num_tokens)
1014
if num_computed_tokens == request.num_tokens:
1015
num_computed_tokens -= 1
1016
self.kv_cache_manager.cache_blocks(