A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Simplify prefix caching logic by removing `num_evictable_computed_blocks` (#11310)
Chen Zhang committed
Dec 18, 2024 at 20:17 UTC
c6b0a7d3ba03ca414be1174e9bd86a97191b7090
1 file changed
+2
-11
vllm/v1/core/kv_cache_manager.py
+2
-11
@@ -201,23 +201,15 @@ class KVCacheManager:
201
f"num_tokens must be greater than 0, got {num_tokens}")
202
203
# Touch the computed blocks to make sure they won't be evicted.
204
- num_evictable_computed_blocks = 0
204
if self.enable_caching:
205
self._touch(computed_blocks)
207
-
208
- # If a computed block of a request is an eviction candidate (in the
209
- # free queue and ref_cnt == 0), it cannot be counted as a free block
210
- # when allocating this request.
211
- num_evictable_computed_blocks = len(
212
- [blk for blk in computed_blocks if blk.ref_cnt == 0])
206
else:
207
assert not computed_blocks, (
208
"Computed blocks should be empty when "
209
"prefix caching is disabled")
210
211
num_required_blocks = cdiv(num_tokens, self.block_size)
219
- if (num_required_blocks > self.free_block_queue.num_free_blocks -
220
- num_evictable_computed_blocks):
212
+ if (num_required_blocks > self.free_block_queue.num_free_blocks):
213
# Cannot allocate new blocks.
214
return None
215
@@ -225,8 +217,7 @@ class KVCacheManager:
217
# preallocated blocks.
218
num_new_blocks = min(
219
num_required_blocks + self.num_preallocate_blocks,
228
- self.free_block_queue.num_free_blocks -
229
- num_evictable_computed_blocks,
220
+ self.free_block_queue.num_free_blocks,
221
# Should not exceed the maximum number of blocks per request.
222
# This is especially because the block table has the shape
223
# [..., max_num_blocks_per_req].