A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Core] Improve hash collision avoidance in prefix caching (#12621)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Feb 3, 2025 at 19:28 UTC
73b35cca7f3745d07d439c197768b25d88b6ab7f
3 files changed
+45
-10
tests/core/block/test_prefix_caching_block.py
+2
-2
@@ -65,8 +65,8 @@ class TestPrefixCachingBlock:
65
66
previous_block = MagicMock(spec=PrefixCachingBlock)
67
prev_block_hash = random.randint(0, 1000)
68
- previous_block.content_hash = (prev_block_hash
69
- if prev_block_has_hash else None)
68
+ previous_block.content_hash = (prev_block_hash if prev_block_has_hash
69
+ else hash('None'))
70
71
num_to_fill = block_size if is_curr_block_full else random.randint(
72
0, block_size - 1)
vllm/core/block/prefix_caching_block.py
+34
-8
@@ -65,6 +65,15 @@ class PrefixCachingBlockAllocator(BlockAllocator):
65
from 0 to num_blocks - 1.
66
"""
67
68
+ # Note that we use 'None' as a string here instead of None because
69
+ # as of Python 3.12, hash(None) returns a constant predictable value.
70
+ # This could possibly make it easier to find and exploit hash
71
+ # collisions. 'None' as a string will be hashed differently per process,
72
+ # but consistently within the same process. This is the same as the
73
+ # behavior of None prior to Python 3.12.
74
+ _none_hash: int = hash('None')
75
+
76
+ # Implements Block.Factory.
77
def __init__(
78
self,
79
num_blocks: int,
@@ -122,7 +131,6 @@ class PrefixCachingBlockAllocator(BlockAllocator):
131
132
self.metric_data = CacheMetricData()
133
125
- # Implements Block.Factory.
134
def _create_block(
135
self,
136
prev_block: Optional[Block],
@@ -737,6 +745,14 @@ class PrefixCachingBlock(Block):
745
such as adapters that influence the block, apart from the token_ids.
746
"""
747
748
+ # Note that we use 'None' as a string here instead of None because
749
+ # as of Python 3.12, hash(None) returns a constant predictable value.
750
+ # This could possibly make it easier to find and exploit hash
751
+ # collisions. 'None' as a string will be hashed differently per process,
752
+ # but consistently within the same process. This is the same as the
753
+ # behavior of None prior to Python 3.12.
754
+ _none_hash: int = hash('None')
755
+
756
def __init__(
757
self,
758
prev_block: Optional[Block],
@@ -891,13 +907,13 @@ class PrefixCachingBlock(Block):
907
908
is_first_block = self._prev_block is None
909
prev_block_hash = (
894
- None if is_first_block else
910
+ self._none_hash if is_first_block else
911
self._prev_block.content_hash # type: ignore
912
)
913
914
# Previous block exists but does not yet have a hash.
915
# Return no hash in this case.
900
- if prev_block_hash is None and not is_first_block:
916
+ if prev_block_hash == self._none_hash and not is_first_block:
917
return None
918
919
self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
@@ -907,8 +923,9 @@ class PrefixCachingBlock(Block):
923
extra_hash=self._extra_hash)
924
return self._cached_content_hash
925
910
- @staticmethod
911
- def hash_block_tokens(is_first_block: bool,
926
+ @classmethod
927
+ def hash_block_tokens(cls,
928
+ is_first_block: bool,
929
prev_block_hash: Optional[int],
930
cur_block_token_ids: List[int],
931
extra_hash: Optional[int] = None) -> int:
@@ -929,7 +946,8 @@ class PrefixCachingBlock(Block):
946
Returns:
947
- int: The computed hash value for the block.
948
"""
932
- assert (prev_block_hash is None) == is_first_block
949
+ if is_first_block and prev_block_hash is None:
950
+ prev_block_hash = cls._none_hash
951
return hash((is_first_block, prev_block_hash, *cur_block_token_ids,
952
extra_hash))
953
@@ -949,6 +967,14 @@ class ComputedBlocksTracker:
967
cached block hashes in the allocator.
968
"""
969
970
+ # Note that we use 'None' as a string here instead of None because
971
+ # as of Python 3.12, hash(None) returns a constant predictable value.
972
+ # This could possibly make it easier to find and exploit hash
973
+ # collisions. 'None' as a string will be hashed differently per process,
974
+ # but consistently within the same process. This is the same as the
975
+ # behavior of None prior to Python 3.12.
976
+ _none_hash: int = hash('None')
977
+
978
def __init__(
979
self,
980
allocator: DeviceAwareBlockAllocator,
@@ -994,7 +1020,7 @@ class ComputedBlocksTracker:
1020
# We need to know the hash of the previous block to compute the hash of
1021
# the current block so that blocks could be uniquely identified across
1022
# sequences of prefixes.
997
- prev_block_hash = (None if cur_num_blocks_recorded == 0 else
1023
+ prev_block_hash = (self._none_hash if cur_num_blocks_recorded == 0 else
1024
block_hashes_recorded[-1])
1025
# Only update the computed block hashes for the new blocks
1026
for i in range(cur_num_blocks_recorded, num_computed_blocks):
@@ -1009,7 +1035,7 @@ class ComputedBlocksTracker:
1035
# This has to be kept in sync with the allocator's hash
1036
# calculation.
1037
block_hash = PrefixCachingBlock.hash_block_tokens(
1012
- is_first_block=prev_block_hash is None,
1038
+ is_first_block=prev_block_hash == self._none_hash,
1039
prev_block_hash=prev_block_hash,
1040
cur_block_token_ids=block_token_ids,
1041
extra_hash=extra_hash,
vllm/v1/core/kv_cache_utils.py
+9
@@ -263,6 +263,15 @@ def hash_block_tokens(
263
The hash value of the block and the token ids in the block.
264
The entire tuple is used as the hash key of the block.
265
"""
266
+ if not parent_block_hash:
267
+ # Note that we use 'None' as a string here instead of None because
268
+ # as of Python 3.12, hash(None) returns a constant predictable value.
269
+ # This could possibly make it easier to find and exploit hash
270
+ # collisions. 'None' as a string will be hashed differently per process,
271
+ # but consistently within the same process. This is the same as the
272
+ # behavior of None prior to Python 3.12.
273
+ parent_block_hash = hash('None')
274
+
275
curr_block_token_ids_tuple = tuple(curr_block_token_ids)
276
return BlockHashType(
277
hash((parent_block_hash, curr_block_token_ids_tuple, extra_keys)),