@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5095e96606
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1] Revert `uncache_blocks` and support recaching full blocks (#12415)

Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>

Cody Yu committed Feb 3, 2025 at 15:04 UTC 5095e966069b9e65b7c4c63427e06cebacaad0a0
2 files changed +16 -55
tests/v1/core/test_prefix_caching.py
-30
@@ -629,33 +629,3 @@ def test_reset_prefix_cache():
629 assert manager.reset_prefix_cache()
630 assert not manager.cached_block_hash_to_block
631 assert all([blk.block_hash is None for blk in manager.block_pool])
632 -
633 -
634 -def test_uncache_blocks():
635 - manager = KVCacheManager(
636 - block_size=16,
637 - num_gpu_blocks=10,
638 - max_model_len=8192,
639 - sliding_window=None,
640 - enable_caching=True,
641 - num_preallocate_tokens=0,
642 - )
643 -
644 - req0 = make_request("0", list(range(30)))
645 - blocks = manager.allocate_slots(req0, 30)
646 - assert [b.block_id for b in blocks] == [0, 1]
647 - assert len(manager.cached_block_hash_to_block) == 1
648 -
649 - req0.num_computed_tokens = 30
650 -
651 - # Simulate speculative tokens.
652 - for _ in range(5):
653 - req0.append_output_token_ids(8)
654 - manager.allocate_slots(req0, 5)
655 - assert len(manager.cached_block_hash_to_block) == 2
656 -
657 - # After sampling, assuming only 1 token is accepted.
658 - req0.num_computed_tokens = 31
659 - num_uncached_blocks = manager.uncache_blocks(req0)
660 - assert num_uncached_blocks == 1
661 - assert len(manager.cached_block_hash_to_block) == 1
vllm/v1/core/kv_cache_manager.py
+16 -25
@@ -252,29 +252,6 @@ class KVCacheManager:
252 if block.ref_cnt == 0:
253 self.free_block_queue.append(block)
254
255 - def uncache_blocks(self, request: Request) -> int:
256 - """Uncache the blocks that are no longer full based on the
257 - num_computed_tokens in the given request. This happens when
258 - the blocks were full and cached due to speculative tokens, but the
259 - speculative tokens are not accepted.
260 -
261 - Args:
262 - request: The request.
263 -
264 - Returns:
265 - The number of uncached blocks.
266 - """
267 - blocks = self.req_to_blocks[request.request_id]
268 - num_computed_tokens = request.num_computed_tokens
269 - num_full_blocks = num_computed_tokens // self.block_size
270 - num_uncached_blocks = 0
271 - for block in blocks[num_full_blocks:]:
272 - # If the block is not cached, the following blocks are not cached.
273 - if not self._maybe_evict_cached_block(block):
274 - break
275 - num_uncached_blocks += 1
276 - return num_uncached_blocks
277 -
255 def reset_prefix_cache(self) -> bool:
256 """Reset prefix cache. This function may be used in RLHF
257 flows to invalid prefix caching after the weights are updated,
@@ -470,8 +447,22 @@ class KVCacheManager:
447 assert prev_block.block_hash is not None
448 prev_block_hash_value = prev_block.block_hash.hash_value
449
473 - for i, blk in enumerate(full_blocks):
474 - blk_idx = blk_start_idx + i
450 + # Find the first uncached block. This case should only happen when
451 + # speculative decoding is used.
452 + offset = 0
453 + for blk in full_blocks:
454 + if blk.block_hash is None:
455 + break
456 + else:
457 + prev_block_hash_value = blk.block_hash.hash_value
458 + offset += 1
459 + else:
460 + # All blocks are cached.
461 + return
462 +
463 + for i, blk in enumerate(full_blocks[offset:]):
464 + blk_idx = blk_start_idx + offset + i
465 + assert blk.block_hash is None
466
467 if blk_idx < num_cached_block_hashes:
468 # The block hash may already be computed in