@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6b0a7d3ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Simplify prefix caching logic by removing `num_evictable_computed_blocks` (#11310)

Chen Zhang committed Dec 18, 2024 at 20:17 UTC c6b0a7d3ba03ca414be1174e9bd86a97191b7090
1 file changed +2 -11
vllm/v1/core/kv_cache_manager.py
+2 -11
@@ -201,23 +201,15 @@ class KVCacheManager:
201 f"num_tokens must be greater than 0, got {num_tokens}")
202
203 # Touch the computed blocks to make sure they won't be evicted.
204 - num_evictable_computed_blocks = 0
204 if self.enable_caching:
205 self._touch(computed_blocks)
207 -
208 - # If a computed block of a request is an eviction candidate (in the
209 - # free queue and ref_cnt == 0), it cannot be counted as a free block
210 - # when allocating this request.
211 - num_evictable_computed_blocks = len(
212 - [blk for blk in computed_blocks if blk.ref_cnt == 0])
206 else:
207 assert not computed_blocks, (
208 "Computed blocks should be empty when "
209 "prefix caching is disabled")
210
211 num_required_blocks = cdiv(num_tokens, self.block_size)
219 - if (num_required_blocks > self.free_block_queue.num_free_blocks -
220 - num_evictable_computed_blocks):
212 + if (num_required_blocks > self.free_block_queue.num_free_blocks):
213 # Cannot allocate new blocks.
214 return None
215
@@ -225,8 +217,7 @@ class KVCacheManager:
217 # preallocated blocks.
218 num_new_blocks = min(
219 num_required_blocks + self.num_preallocate_blocks,
228 - self.free_block_queue.num_free_blocks -
229 - num_evictable_computed_blocks,
220 + self.free_block_queue.num_free_blocks,
221 # Should not exceed the maximum number of blocks per request.
222 # This is especially because the block table has the shape
223 # [..., max_num_blocks_per_req].