@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9855aea21b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix][V1] Re-compute an entire block when fully cache hit (#11186)

Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>

Cody Yu committed Dec 13, 2024 at 17:08 UTC 9855aea21b6aec48b12cef3a1614e7796b970a73
1 file changed +7 -3
vllm/v1/core/scheduler.py
+7 -3
@@ -199,9 +199,13 @@ class Scheduler:
199 if num_new_tokens == 0:
200 # The happens when prompt length is divisible by the block
201 # size and all blocks are cached. Now we force to recompute
202 - # the last token.
203 - num_computed_tokens -= 1
204 - num_new_tokens = 1
202 + # the last block. Note that we have to re-compute an entire
203 + # block because allocate_slots() assumes num_computed_tokens
204 + # is always a multiple of the block size. This limitation
205 + # can potentially be removed in the future to slightly
206 + # improve the performance.
207 + num_computed_tokens -= self.block_size
208 + num_new_tokens = self.block_size
209 computed_blocks.pop()
210 num_new_tokens = min(num_new_tokens, token_budget)
211 assert num_new_tokens > 0