A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][V1] Re-compute an entire block when fully cache hit (#11186)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Dec 13, 2024 at 17:08 UTC
9855aea21b6aec48b12cef3a1614e7796b970a73
1 file changed
+7
-3
vllm/v1/core/scheduler.py
+7
-3
@@ -199,9 +199,13 @@ class Scheduler:
199
if num_new_tokens == 0:
200
# The happens when prompt length is divisible by the block
201
# size and all blocks are cached. Now we force to recompute
202
- # the last token.
203
- num_computed_tokens -= 1
204
- num_new_tokens = 1
202
+ # the last block. Note that we have to re-compute an entire
203
+ # block because allocate_slots() assumes num_computed_tokens
204
+ # is always a multiple of the block size. This limitation
205
+ # can potentially be removed in the future to slightly
206
+ # improve the performance.
207
+ num_computed_tokens -= self.block_size
208
+ num_new_tokens = self.block_size
209
computed_blocks.pop()
210
num_new_tokens = min(num_new_tokens, token_budget)
211
assert num_new_tokens > 0