@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89003c4082
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[v1][Bugfix] Add extra_keys to block_hash for prefix caching (#12603)

This pr adds extra key to block hash, to generate different hash value for two blocks with the same token string but different extra_keys in their parent blocks. For example, it can generate different hash value for the second block of the following two requests: ```python request1 = make_request( request_id=0, prompt_token_ids=[_ for _ in range(6)], mm_positions=[{ "offset": 0, "length": 3 }, { "offset": 3, "length": 3 }], mm_hashes=["hash1", "hash2"], ) request2 = make_request( request_id=1, prompt_token_ids=[_ for _ in range(6)], mm_positions=[{ "offset": 0, "length": 3 }, { "offset": 3, "length": 3 }], mm_hashes=["hash3", "hash2"], ) ``` --------- Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Feb 1, 2025 at 05:13 UTC 89003c4082db880e103e84f5015424e79f9aa762
2 files changed +37 -3
tests/v1/core/test_kv_cache_utils.py
+33 -1
@@ -192,7 +192,7 @@ def test_hash_block_tokens():
192 extra_keys)
193 assert isinstance(block_hash, BlockHashType)
194 assert block_hash.hash_value == hash(
195 - (parent_block_hash, *curr_block_token_ids))
195 + (parent_block_hash, curr_block_token_ids, extra_keys))
196 assert block_hash.token_ids == curr_block_token_ids
197 assert block_hash.extra_keys == extra_keys
198
@@ -227,6 +227,38 @@ def test_hash_request_tokens():
227 assert block_hashes[1].extra_keys == ("hash2", )
228
229
230 +def test_hash_tokens_different_mm_input():
231 + request1 = make_request(
232 + request_id=0,
233 + prompt_token_ids=[_ for _ in range(6)],
234 + mm_positions=[{
235 + "offset": 0,
236 + "length": 3
237 + }, {
238 + "offset": 3,
239 + "length": 3
240 + }],
241 + mm_hashes=["hash1", "hash2"],
242 + )
243 + request2 = make_request(
244 + request_id=1,
245 + prompt_token_ids=[_ for _ in range(6)],
246 + mm_positions=[{
247 + "offset": 0,
248 + "length": 3
249 + }, {
250 + "offset": 3,
251 + "length": 3
252 + }],
253 + mm_hashes=["hash3", "hash2"],
254 + )
255 + block_size = 3
256 + block_hashes1 = hash_request_tokens(block_size, request1)
257 + block_hashes2 = hash_request_tokens(block_size, request2)
258 + assert block_hashes1[0] != block_hashes2[0]
259 + assert block_hashes1[1] != block_hashes2[1]
260 +
261 +
262 def test_hash_request_tokens_no_mm_inputs():
263 request = make_request(
264 request_id=0,
vllm/v1/core/kv_cache_utils.py
+4 -2
@@ -262,8 +262,10 @@ def hash_block_tokens(
262 The hash value of the block and the token ids in the block.
263 The entire tuple is used as the hash key of the block.
264 """
265 - return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
266 - tuple(curr_block_token_ids), extra_keys)
265 + curr_block_token_ids_tuple = tuple(curr_block_token_ids)
266 + return BlockHashType(
267 + hash((parent_block_hash, curr_block_token_ids_tuple, extra_keys)),
268 + curr_block_token_ids_tuple, extra_keys)
269
270
271 def hash_request_tokens(block_size: int,