A high-throughput and memory-efficient inference and serving engine for LLMs
Apply LRU policy only to proper cache entries (#42656)
Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
Stan Wozniak committed
Jun 16, 2026 at 23:49 UTC
520828789c38acfeefc3095c810e3148b76bac03
3 files changed
+29
-56
tests/v1/core/test_prefix_caching.py
+12
-10
@@ -291,13 +291,12 @@ def test_prefill(hash_fn):
291
# All blocks should be available.
292
assert free_block_queue.num_free_blocks == 10
293
# The order should be
294
+ # [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
295
# [unallocated (6, 7, 8, 9, 10)]
295
- # [unique_req0 (4)]
296
- # [unique_req1 (5)]
296
# [common (3, 2, 1)]
297
assert [
298
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
300
- ] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
299
+ ] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
300
301
# Cache hit in the common prefix when the original block is already free.
302
# Incomplete 1 block (6 tokens)
@@ -311,7 +310,7 @@ def test_prefill(hash_fn):
310
blocks = manager.allocate_slots(
311
req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks
312
)
314
- assert blocks is not None and blocks.get_block_ids() == ([6],)
313
+ assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5]
314
315
# Although we only have 6 free blocks, we have 8 blocks in
316
# the free block queue due to lazy removal.
@@ -331,7 +330,7 @@ def test_prefill(hash_fn):
330
)
331
# This block ID order also checks the eviction order.
332
assert blocks is not None and blocks.get_block_ids() == (
334
- [7, 8, 9, 10, 4, 5, 6, 3, 2, 1],
333
+ [5, 4, 6, 7, 8, 9, 10, 3, 2, 1],
334
)
335
336
assert free_block_queue.num_free_blocks == 0
@@ -1216,13 +1215,12 @@ def test_prefill_plp():
1215
# All blocks should be available.
1216
assert manager.block_pool.free_block_queue.num_free_blocks == 10
1217
# The order should be
1218
+ # [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
1219
# [unallocated (6, 7, 8, 9, 10)]
1220
- # [unique_req0 (4)]
1221
- # [unique_req1 (5)]
1220
# [common (3, 2, 1)]
1221
assert [
1222
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
1225
- ] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
1223
+ ] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
1224
1225
# Request #2 is a prompt-logprobs request:
1226
# NO cache hit in the common prefix; duplicates request #0 cached blocks
@@ -1351,11 +1349,15 @@ def test_evict():
1349
assert manager.block_pool.free_block_queue.num_free_blocks == 1
1350
1351
manager.free(req0)
1352
+ # partial blocks (without hash) at head, other at tail (LRU policy):
1353
+ assert [
1354
+ b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
1355
+ ] == [6, 10, 5, 4, 3, 2, 1]
1356
manager.free(req1)
1357
assert manager.block_pool.free_block_queue.num_free_blocks == 10
1358
assert [
1359
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
1358
- ] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7]
1360
+ ] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7]
1361
1362
# Touch the first 2 blocks.
1363
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
@@ -1365,7 +1367,7 @@ def test_evict():
1367
blocks = manager.allocate_slots(
1368
req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks
1369
)
1368
- assert blocks is not None and blocks.get_block_ids() == ([10],)
1370
+ assert blocks is not None and blocks.get_block_ids() == ([6],)
1371
assert manager.block_pool.free_block_queue.num_free_blocks == 7
1372
1373
vllm/v1/core/block_pool.py
+14
-15
@@ -416,29 +416,28 @@ class BlockPool:
416
if self.metrics_collector:
417
self.metrics_collector.on_block_accessed(block)
418
419
- def free_blocks(
420
- self, ordered_blocks: Iterable[KVCacheBlock], prepend: bool = False
421
- ) -> None:
419
+ def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
420
"""Free a list of blocks. The blocks should be ordered by their
421
eviction priority, where the first block will be evicted first.
422
423
Args:
424
ordered_blocks: A list of blocks to free ordered by their eviction
425
priority.
428
- prepend: Whether to put newly-free blocks at the front of the free
429
- queue to be prioritized for reuse.
426
"""
431
- # Materialize the iterable to allow multiple passes.
432
- blocks_list = list(ordered_blocks)
433
- for block in blocks_list:
427
+ # Identify blocks with hash (LRU cache) and without it (will never match in APC)
428
+ blocks_with_hash = []
429
+ blocks_without_hash = []
430
+ for block in ordered_blocks:
431
block.ref_cnt -= 1
435
- freed_blocks = [
436
- block for block in blocks_list if block.ref_cnt == 0 and not block.is_null
437
- ]
438
- if prepend:
439
- self.free_block_queue.prepend_n(freed_blocks)
440
- else:
441
- self.free_block_queue.append_n(freed_blocks)
432
+ if block.ref_cnt == 0 and not block.is_null:
433
+ if block.block_hash is None:
434
+ blocks_without_hash.append(block)
435
+ else:
436
+ blocks_with_hash.append(block)
437
+
438
+ # Blocks without hash always get evicted first - prepend them last to the tail
439
+ self.free_block_queue.prepend_n(blocks_without_hash)
440
+ self.free_block_queue.append_n(blocks_with_hash)
441
442
def evict_blocks(self, block_ids: set[int]) -> None:
443
"""evict blocks from the prefix cache by their block IDs.
vllm/v1/core/single_type_kv_cache_manager.py
+3
-31
@@ -504,12 +504,7 @@ class SingleTypeKVCacheManager(ABC):
504
# range), so we must cap to the number of blocks that currently exist for
505
# this request.
506
num_skipped_blocks = min(num_skipped_blocks, len(blocks))
507
-
508
- # Reuse skipped local blocks in order:
509
- # scratch blocks: no prefix-cache value, reuse first.
510
- # cached blocks: reusable prefix-cache value, reuse last.
511
- removed_cached_blocks: list[KVCacheBlock] = []
512
- removed_uncached_blocks: list[KVCacheBlock] = []
507
+ removed_blocks: list[KVCacheBlock] = []
508
# Because the block starts from index 0, the num_skipped_block-th block
509
# corresponds to index num_skipped_blocks - 1.
510
for i in range(num_skipped_blocks - 1, -1, -1):
@@ -518,16 +513,9 @@ class SingleTypeKVCacheManager(ABC):
513
# should also have been set to null blocks by the previous calls
514
# to this function.
515
break
521
- if blocks[i].block_hash is None:
522
- removed_uncached_blocks.append(blocks[i])
523
- else:
524
- removed_cached_blocks.append(blocks[i])
516
+ removed_blocks.append(blocks[i])
517
blocks[i] = self._null_block
526
- # `prepend=True` makes uncached scratch blocks the next allocation
527
- # candidates, while cached blocks stay behind them as best-effort
528
- # prefix-cache entries.
529
- self.block_pool.free_blocks(removed_cached_blocks)
530
- self.block_pool.free_blocks(removed_uncached_blocks, prepend=True)
518
+ self.block_pool.free_blocks(removed_blocks)
519
520
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
521
"""
@@ -777,22 +765,6 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
765
766
return mask
767
780
- def free(self, request_id: str) -> None:
781
- # similar to remove_skipped_blocks(), prepend the uncached blocks
782
- # and append the cached blocks to the free queue
783
- req_blocks = self.req_to_blocks.pop(request_id, [])
784
- if req_blocks:
785
- cached_blocks: list[KVCacheBlock] = []
786
- uncached_blocks: list[KVCacheBlock] = []
787
- for block in reversed(req_blocks):
788
- if block.block_hash is None:
789
- uncached_blocks.append(block)
790
- else:
791
- cached_blocks.append(block)
792
- self.block_pool.free_blocks(cached_blocks)
793
- self.block_pool.free_blocks(uncached_blocks, prepend=True)
794
- self.num_cached_block.pop(request_id, None)
795
-
768
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
769
"""
770
Get the number of tokens that will be skipped for attention computation.