@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0ef37233e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Add `uncache_blocks` (#12333)

Cody Yu committed Jan 22, 2025 at 20:19 UTC f0ef37233ea0ba5251edaea7362984110411e7eb
2 files changed +61 -2
tests/v1/core/test_prefix_caching.py
+30
@@ -626,3 +626,33 @@ def test_reset_prefix_cache():
626 assert manager.reset_prefix_cache()
627 assert not manager.cached_block_hash_to_block
628 assert all([blk.block_hash is None for blk in manager.block_pool])
629 +
630 +
631 +def test_uncache_blocks():
632 + manager = KVCacheManager(
633 + block_size=16,
634 + num_gpu_blocks=10,
635 + max_model_len=8192,
636 + sliding_window=None,
637 + enable_caching=True,
638 + num_preallocate_tokens=0,
639 + )
640 +
641 + req0 = make_request("0", list(range(30)))
642 + blocks = manager.allocate_slots(req0, 30, [])
643 + assert [b.block_id for b in blocks] == [0, 1]
644 + assert len(manager.cached_block_hash_to_block) == 1
645 +
646 + req0.num_computed_tokens = 30
647 +
648 + # Simulate speculative tokens.
649 + for _ in range(5):
650 + req0.append_output_token_ids(8)
651 + manager.append_slots(req0, 5)
652 + assert len(manager.cached_block_hash_to_block) == 2
653 +
654 + # After sampling, assuming only 1 token is accepted.
655 + req0.num_computed_tokens = 31
656 + num_uncached_blocks = manager.uncache_blocks(req0)
657 + assert num_uncached_blocks == 1
658 + assert len(manager.cached_block_hash_to_block) == 1
vllm/v1/core/kv_cache_manager.py
+31 -2
@@ -285,6 +285,29 @@ class KVCacheManager:
285 if block.ref_cnt == 0:
286 self.free_block_queue.append(block)
287
288 + def uncache_blocks(self, request: Request) -> int:
289 + """Uncache the blocks that are no longer full based on the
290 + num_computed_tokens in the given request. This happens when
291 + the blocks were full and cached due to speculative tokens, but the
292 + speculative tokens are not accepted.
293 +
294 + Args:
295 + request: The request.
296 +
297 + Returns:
298 + The number of uncached blocks.
299 + """
300 + blocks = self.req_to_blocks[request.request_id]
301 + num_computed_tokens = request.num_computed_tokens
302 + num_full_blocks = num_computed_tokens // self.block_size
303 + num_uncached_blocks = 0
304 + for block in blocks[num_full_blocks:]:
305 + # If the block is not cached, the following blocks are not cached.
306 + if not self._maybe_evict_cached_block(block):
307 + break
308 + num_uncached_blocks += 1
309 + return num_uncached_blocks
310 +
311 def reset_prefix_cache(self) -> bool:
312 """Reset prefix cache. This function may be used in RLHF
313 flows to invalid prefix caching after the weights are updated,
@@ -386,7 +409,7 @@ class KVCacheManager:
409
410 # If the block is cached, evict it.
411 if self.enable_caching:
389 - self._evict_cached_block(curr_block)
412 + self._maybe_evict_cached_block(curr_block)
413
414 curr_block.incr_ref()
415 ret.append(curr_block)
@@ -394,13 +417,16 @@ class KVCacheManager:
417
418 return ret
419
397 - def _evict_cached_block(self, block: KVCacheBlock) -> None:
420 + def _maybe_evict_cached_block(self, block: KVCacheBlock) -> bool:
421 """
422 If a block is cached in `cached_block_hash_to_block`, we reset its hash
423 metadata and evict it from the cache.
424
425 Args:
426 block: The block to evict.
427 +
428 + Returns:
429 + True if the block is evicted, False otherwise.
430 """
431 block_hash = block.block_hash
432 if block_hash and block_hash in self.cached_block_hash_to_block:
@@ -410,6 +436,9 @@ class KVCacheManager:
436 if len(self.cached_block_hash_to_block[block_hash]) == 0:
437 del self.cached_block_hash_to_block[block_hash]
438
439 + return True
440 + return False
441 +
442 def _get_cached_block(self,
443 block_hash: BlockHashType) -> Optional[KVCacheBlock]:
444 """Get a cached block by the block hash, or None if cache miss.