A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc][V1] Fix type in v1 prefix caching (#11151)
Cody Yu committed
Dec 12, 2024 at 16:57 UTC
78ed8f57d8815cdd5567533f7d3e25b959d861ab
3 files changed
+27
-15
tests/v1/core/test_prefix_caching.py
+8
-4
@@ -49,7 +49,7 @@ def test_prefill():
49
block_hash = hash_block_tokens(parent_block_hash, block_tokens)
50
assert manager.block_pool[block_id].block_hash == block_hash
51
assert manager.block_pool[block_id].ref_cnt == 1
52
- parent_block_hash = block_hash
52
+ parent_block_hash = block_hash.hash_value
53
54
# Check partial/preallocated block metadata
55
for block_id in (3, 4):
@@ -360,11 +360,15 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
360
assert not computed_blocks
361
# Just ask for 1 block.
362
blocks = manager.allocate_slots(req, block_size, computed_blocks)
363
+ req.num_computed_tokens = block_size
364
assert len(blocks) == 1 + num_preallocated_blocks
365
365
- # Append slots to the block.
366
- req.num_computed_tokens = block_size * len(blocks) # Assume all used.
367
- blocks = manager.append_slots(req, block_size) # Append 1 block.
366
+ # Assume all computed.
367
+ manager.append_slots(req, block_size * (len(blocks) - 1))
368
+ req.num_computed_tokens = block_size * len(blocks)
369
+
370
+ # Append 1 block.
371
+ blocks = manager.append_slots(req, block_size)
372
assert len(blocks) == 1 + num_preallocated_blocks
373
374
vllm/v1/core/kv_cache_manager.py
+4
-4
@@ -375,8 +375,8 @@ class KVCacheManager:
375
prev_block: The previous block in the chain.
376
"""
377
# Update the new blocks with the block hashes through the chain.
378
- prev_block_hash = (prev_block.block_hash
379
- if prev_block is not None else None)
378
+ prev_block_hash_value = (prev_block.block_hash.hash_value
379
+ if prev_block is not None else None)
380
for i, blk in enumerate(full_blocks):
381
blk_idx = blk_start_idx + i
382
@@ -390,10 +390,10 @@ class KVCacheManager:
390
f"{request.request_id}({request})")
391
392
# Compute the hash of the current block.
393
- block_hash = hash_block_tokens(prev_block_hash,
393
+ block_hash = hash_block_tokens(prev_block_hash_value,
394
tuple(block_tokens))
395
396
# Update and added the full block to the cache.
397
blk.block_hash = block_hash
398
self.cached_block_hash_to_block[block_hash][blk.block_id] = blk
399
- prev_block_hash = block_hash
399
+ prev_block_hash_value = block_hash.hash_value
vllm/v1/core/kv_cache_utils.py
+15
-7
@@ -1,12 +1,19 @@
1
"""KV-Cache Utilities."""
2
from dataclasses import dataclass
3
-from typing import List, Optional, Tuple
3
+from typing import List, NamedTuple, Optional, Tuple
4
5
from vllm.logger import init_logger
6
7
logger = init_logger(__name__)
8
9
-BlockHashType = Tuple[int, Tuple[int]]
9
+
10
+class BlockHashType(NamedTuple):
11
+ """Hash value of a block and the token IDs in the block.
12
+ The reason we keep a tuple of token IDs is to make sure no hash
13
+ collision happens when the hash value is the same.
14
+ """
15
+ hash_value: int
16
+ token_ids: Tuple[int]
17
18
19
@dataclass
@@ -171,8 +178,8 @@ def hash_block_tokens(parent_block_hash: Optional[int],
178
The hash value of the block and the token ids in the block.
179
The entire tuple is used as the hash key of the block.
180
"""
174
- return (hash(
175
- (parent_block_hash, *curr_block_token_ids)), curr_block_token_ids)
181
+ return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
182
+ curr_block_token_ids)
183
184
185
def hash_request_tokens(block_size: int,
@@ -188,14 +195,15 @@ def hash_request_tokens(block_size: int,
195
The list of computed hash values.
196
"""
197
ret = []
191
- parent_block_hash = None
198
+ parent_block_hash_value = None
199
for start in range(0, len(token_ids), block_size):
200
end = start + block_size
201
block_token_ids = tuple(token_ids[start:end])
202
# Do not hash the block if it is not full.
203
if len(block_token_ids) < block_size:
204
break
198
- block_hash = hash_block_tokens(parent_block_hash, block_token_ids)
205
+ block_hash = hash_block_tokens(parent_block_hash_value,
206
+ block_token_ids)
207
ret.append(block_hash)
200
- parent_block_hash = block_hash
208
+ parent_block_hash_value = block_hash.hash_value
209
return ret