@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78ed8f57d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc][V1] Fix type in v1 prefix caching (#11151)

Cody Yu committed Dec 12, 2024 at 16:57 UTC 78ed8f57d8815cdd5567533f7d3e25b959d861ab
3 files changed +27 -15
tests/v1/core/test_prefix_caching.py
+8 -4
@@ -49,7 +49,7 @@ def test_prefill():
49 block_hash = hash_block_tokens(parent_block_hash, block_tokens)
50 assert manager.block_pool[block_id].block_hash == block_hash
51 assert manager.block_pool[block_id].ref_cnt == 1
52 - parent_block_hash = block_hash
52 + parent_block_hash = block_hash.hash_value
53
54 # Check partial/preallocated block metadata
55 for block_id in (3, 4):
@@ -360,11 +360,15 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
360 assert not computed_blocks
361 # Just ask for 1 block.
362 blocks = manager.allocate_slots(req, block_size, computed_blocks)
363 + req.num_computed_tokens = block_size
364 assert len(blocks) == 1 + num_preallocated_blocks
365
365 - # Append slots to the block.
366 - req.num_computed_tokens = block_size * len(blocks) # Assume all used.
367 - blocks = manager.append_slots(req, block_size) # Append 1 block.
366 + # Assume all computed.
367 + manager.append_slots(req, block_size * (len(blocks) - 1))
368 + req.num_computed_tokens = block_size * len(blocks)
369 +
370 + # Append 1 block.
371 + blocks = manager.append_slots(req, block_size)
372 assert len(blocks) == 1 + num_preallocated_blocks
373
374
vllm/v1/core/kv_cache_manager.py
+4 -4
@@ -375,8 +375,8 @@ class KVCacheManager:
375 prev_block: The previous block in the chain.
376 """
377 # Update the new blocks with the block hashes through the chain.
378 - prev_block_hash = (prev_block.block_hash
379 - if prev_block is not None else None)
378 + prev_block_hash_value = (prev_block.block_hash.hash_value
379 + if prev_block is not None else None)
380 for i, blk in enumerate(full_blocks):
381 blk_idx = blk_start_idx + i
382
@@ -390,10 +390,10 @@ class KVCacheManager:
390 f"{request.request_id}({request})")
391
392 # Compute the hash of the current block.
393 - block_hash = hash_block_tokens(prev_block_hash,
393 + block_hash = hash_block_tokens(prev_block_hash_value,
394 tuple(block_tokens))
395
396 # Update and added the full block to the cache.
397 blk.block_hash = block_hash
398 self.cached_block_hash_to_block[block_hash][blk.block_id] = blk
399 - prev_block_hash = block_hash
399 + prev_block_hash_value = block_hash.hash_value
vllm/v1/core/kv_cache_utils.py
+15 -7
@@ -1,12 +1,19 @@
1 """KV-Cache Utilities."""
2 from dataclasses import dataclass
3 -from typing import List, Optional, Tuple
3 +from typing import List, NamedTuple, Optional, Tuple
4
5 from vllm.logger import init_logger
6
7 logger = init_logger(__name__)
8
9 -BlockHashType = Tuple[int, Tuple[int]]
9 +
10 +class BlockHashType(NamedTuple):
11 + """Hash value of a block and the token IDs in the block.
12 + The reason we keep a tuple of token IDs is to make sure no hash
13 + collision happens when the hash value is the same.
14 + """
15 + hash_value: int
16 + token_ids: Tuple[int]
17
18
19 @dataclass
@@ -171,8 +178,8 @@ def hash_block_tokens(parent_block_hash: Optional[int],
178 The hash value of the block and the token ids in the block.
179 The entire tuple is used as the hash key of the block.
180 """
174 - return (hash(
175 - (parent_block_hash, *curr_block_token_ids)), curr_block_token_ids)
181 + return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
182 + curr_block_token_ids)
183
184
185 def hash_request_tokens(block_size: int,
@@ -188,14 +195,15 @@ def hash_request_tokens(block_size: int,
195 The list of computed hash values.
196 """
197 ret = []
191 - parent_block_hash = None
198 + parent_block_hash_value = None
199 for start in range(0, len(token_ids), block_size):
200 end = start + block_size
201 block_token_ids = tuple(token_ids[start:end])
202 # Do not hash the block if it is not full.
203 if len(block_token_ids) < block_size:
204 break
198 - block_hash = hash_block_tokens(parent_block_hash, block_token_ids)
205 + block_hash = hash_block_tokens(parent_block_hash_value,
206 + block_token_ids)
207 ret.append(block_hash)
200 - parent_block_hash = block_hash
208 + parent_block_hash_value = block_hash.hash_value
209 return ret