@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73b35cca7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Core] Improve hash collision avoidance in prefix caching (#12621)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Feb 3, 2025 at 19:28 UTC 73b35cca7f3745d07d439c197768b25d88b6ab7f
3 files changed +45 -10
tests/core/block/test_prefix_caching_block.py
+2 -2
@@ -65,8 +65,8 @@ class TestPrefixCachingBlock:
65
66 previous_block = MagicMock(spec=PrefixCachingBlock)
67 prev_block_hash = random.randint(0, 1000)
68 - previous_block.content_hash = (prev_block_hash
69 - if prev_block_has_hash else None)
68 + previous_block.content_hash = (prev_block_hash if prev_block_has_hash
69 + else hash('None'))
70
71 num_to_fill = block_size if is_curr_block_full else random.randint(
72 0, block_size - 1)
vllm/core/block/prefix_caching_block.py
+34 -8
@@ -65,6 +65,15 @@ class PrefixCachingBlockAllocator(BlockAllocator):
65 from 0 to num_blocks - 1.
66 """
67
68 + # Note that we use 'None' as a string here instead of None because
69 + # as of Python 3.12, hash(None) returns a constant predictable value.
70 + # This could possibly make it easier to find and exploit hash
71 + # collisions. 'None' as a string will be hashed differently per process,
72 + # but consistently within the same process. This is the same as the
73 + # behavior of None prior to Python 3.12.
74 + _none_hash: int = hash('None')
75 +
76 + # Implements Block.Factory.
77 def __init__(
78 self,
79 num_blocks: int,
@@ -122,7 +131,6 @@ class PrefixCachingBlockAllocator(BlockAllocator):
131
132 self.metric_data = CacheMetricData()
133
125 - # Implements Block.Factory.
134 def _create_block(
135 self,
136 prev_block: Optional[Block],
@@ -737,6 +745,14 @@ class PrefixCachingBlock(Block):
745 such as adapters that influence the block, apart from the token_ids.
746 """
747
748 + # Note that we use 'None' as a string here instead of None because
749 + # as of Python 3.12, hash(None) returns a constant predictable value.
750 + # This could possibly make it easier to find and exploit hash
751 + # collisions. 'None' as a string will be hashed differently per process,
752 + # but consistently within the same process. This is the same as the
753 + # behavior of None prior to Python 3.12.
754 + _none_hash: int = hash('None')
755 +
756 def __init__(
757 self,
758 prev_block: Optional[Block],
@@ -891,13 +907,13 @@ class PrefixCachingBlock(Block):
907
908 is_first_block = self._prev_block is None
909 prev_block_hash = (
894 - None if is_first_block else
910 + self._none_hash if is_first_block else
911 self._prev_block.content_hash # type: ignore
912 )
913
914 # Previous block exists but does not yet have a hash.
915 # Return no hash in this case.
900 - if prev_block_hash is None and not is_first_block:
916 + if prev_block_hash == self._none_hash and not is_first_block:
917 return None
918
919 self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
@@ -907,8 +923,9 @@ class PrefixCachingBlock(Block):
923 extra_hash=self._extra_hash)
924 return self._cached_content_hash
925
910 - @staticmethod
911 - def hash_block_tokens(is_first_block: bool,
926 + @classmethod
927 + def hash_block_tokens(cls,
928 + is_first_block: bool,
929 prev_block_hash: Optional[int],
930 cur_block_token_ids: List[int],
931 extra_hash: Optional[int] = None) -> int:
@@ -929,7 +946,8 @@ class PrefixCachingBlock(Block):
946 Returns:
947 - int: The computed hash value for the block.
948 """
932 - assert (prev_block_hash is None) == is_first_block
949 + if is_first_block and prev_block_hash is None:
950 + prev_block_hash = cls._none_hash
951 return hash((is_first_block, prev_block_hash, *cur_block_token_ids,
952 extra_hash))
953
@@ -949,6 +967,14 @@ class ComputedBlocksTracker:
967 cached block hashes in the allocator.
968 """
969
970 + # Note that we use 'None' as a string here instead of None because
971 + # as of Python 3.12, hash(None) returns a constant predictable value.
972 + # This could possibly make it easier to find and exploit hash
973 + # collisions. 'None' as a string will be hashed differently per process,
974 + # but consistently within the same process. This is the same as the
975 + # behavior of None prior to Python 3.12.
976 + _none_hash: int = hash('None')
977 +
978 def __init__(
979 self,
980 allocator: DeviceAwareBlockAllocator,
@@ -994,7 +1020,7 @@ class ComputedBlocksTracker:
1020 # We need to know the hash of the previous block to compute the hash of
1021 # the current block so that blocks could be uniquely identified across
1022 # sequences of prefixes.
997 - prev_block_hash = (None if cur_num_blocks_recorded == 0 else
1023 + prev_block_hash = (self._none_hash if cur_num_blocks_recorded == 0 else
1024 block_hashes_recorded[-1])
1025 # Only update the computed block hashes for the new blocks
1026 for i in range(cur_num_blocks_recorded, num_computed_blocks):
@@ -1009,7 +1035,7 @@ class ComputedBlocksTracker:
1035 # This has to be kept in sync with the allocator's hash
1036 # calculation.
1037 block_hash = PrefixCachingBlock.hash_block_tokens(
1012 - is_first_block=prev_block_hash is None,
1038 + is_first_block=prev_block_hash == self._none_hash,
1039 prev_block_hash=prev_block_hash,
1040 cur_block_token_ids=block_token_ids,
1041 extra_hash=extra_hash,
vllm/v1/core/kv_cache_utils.py
+9
@@ -263,6 +263,15 @@ def hash_block_tokens(
263 The hash value of the block and the token ids in the block.
264 The entire tuple is used as the hash key of the block.
265 """
266 + if not parent_block_hash:
267 + # Note that we use 'None' as a string here instead of None because
268 + # as of Python 3.12, hash(None) returns a constant predictable value.
269 + # This could possibly make it easier to find and exploit hash
270 + # collisions. 'None' as a string will be hashed differently per process,
271 + # but consistently within the same process. This is the same as the
272 + # behavior of None prior to Python 3.12.
273 + parent_block_hash = hash('None')
274 +
275 curr_block_token_ids_tuple = tuple(curr_block_token_ids)
276 return BlockHashType(
277 hash((parent_block_hash, curr_block_token_ids_tuple, extra_keys)),