@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c3230d8c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Simpify vision block hash for prefix caching by removing offset from hash (#11646)

Chen Zhang committed Dec 31, 2024 at 16:56 UTC 8c3230d8c1cf114618c2316c54bf06b7d0c198b6
2 files changed +6 -6
tests/v1/core/test_prefix_caching.py
+4 -4
@@ -469,9 +469,9 @@ def test_mm_prefix_caching():
469 # Completed block should have hashes with extra keys.
470 assert not computed_blocks
471 assert len(req0.kv_block_hashes) == 3
472 - assert req0.kv_block_hashes[0].extra_keys == (("aaa", 0), )
473 - assert req0.kv_block_hashes[1].extra_keys == (("aaa", 5), ("bbb", 0))
474 - assert req0.kv_block_hashes[2].extra_keys == (("bbb", 2), )
472 + assert req0.kv_block_hashes[0].extra_keys == ("aaa", )
473 + assert req0.kv_block_hashes[1].extra_keys == ("aaa", "bbb")
474 + assert req0.kv_block_hashes[2].extra_keys == ("bbb", )
475
476 blocks = manager.allocate_slots(req0, 59, computed_blocks)
477 assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
@@ -485,7 +485,7 @@ def test_mm_prefix_caching():
485
486 # The just completed block should have hashes with extra keys.
487 assert len(req0.kv_block_hashes) == 4
488 - assert req0.kv_block_hashes[3].extra_keys == (("ccc", 0), )
488 + assert req0.kv_block_hashes[3].extra_keys == ("ccc", )
489
490 # Cache hit.
491 unique_token_ids = [-1] * 7 + [200] * 5
vllm/v1/core/kv_cache_utils.py
+2 -2
@@ -218,8 +218,8 @@ def generate_block_hash_extra_keys(
218 continue
219
220 # The block contains the current mm input.
221 - mm_start = max(0, start_token_idx - offset)
222 - extra_keys.append((mm_hashes[curr_mm_idx], mm_start))
221 + extra_keys.append(mm_hashes[curr_mm_idx])
222 +
223 if end_token_idx >= offset + length:
224 # If this block contains the end of the current mm input,
225 # move to the next mm input as this block may also contain