@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 994fc655b7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][Prefix Cache] Move the logic of num_computed_tokens into KVCacheManager (#12003)

Chen Zhang committed Jan 15, 2025 at 15:55 UTC 994fc655b71f59f61b82cc44e868091dae493a84
3 files changed +61 -35
tests/v1/core/test_prefix_caching.py
+47 -24
@@ -49,9 +49,10 @@ def test_prefill():
49 unique_token_ids = [3] * 7
50 all_token_ids = common_token_ids + unique_token_ids
51 req0 = make_request("0", all_token_ids)
52 - computed_blocks = manager.get_computed_blocks(req0)
52 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
53 assert len(req0.kv_block_hashes) == 3
54 assert not computed_blocks
55 + assert num_computed_tokens == 0
56 blocks = manager.allocate_slots(req0, 55, computed_blocks)
57 assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
58
@@ -73,9 +74,10 @@ def test_prefill():
74 # Incomplete 1 block (5 tokens)
75 unique_token_ids = [3] * 5
76 req1 = make_request("1", common_token_ids + unique_token_ids)
76 - computed_blocks = manager.get_computed_blocks(req1)
77 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
78 assert len(req1.kv_block_hashes) == 3
79 assert [b.block_id for b in computed_blocks] == [0, 1, 2]
80 + assert num_computed_tokens == 3 * 16
81 num_new_tokens = 53 - 3 * 16
82 blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
83 assert [b.block_id for b in blocks] == [5, 6]
@@ -91,7 +93,7 @@ def test_prefill():
93 # All blocks should be available.
94 assert manager.free_block_queue.num_free_blocks == 10
95 # The order should be
94 - # [unallocated (7, 8)]
96 + # [unallocated (7, 8, 9)]
97 # [unique_req0 (4, 3)]
98 # [unique_req1 (6, 5)]
99 # [common (2, 1, 0)]
@@ -103,9 +105,10 @@ def test_prefill():
105 # Incomplete 1 block (6 tokens)
106 unique_token_ids = [3] * 6
107 req2 = make_request("2", common_token_ids + unique_token_ids)
106 - computed_blocks = manager.get_computed_blocks(req2)
108 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
109 assert len(req2.kv_block_hashes) == 3
110 assert [b.block_id for b in computed_blocks] == [0, 1, 2]
111 + assert num_computed_tokens == 3 * 16
112 num_new_tokens = 53 - 3 * 16
113 blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
114 assert [b.block_id for b in blocks] == [7, 8]
@@ -123,8 +126,9 @@ def test_prefill():
126
127 # Cache miss and eviction.
128 req3 = make_request("3", [99] * (16 * 9))
126 - computed_blocks = manager.get_computed_blocks(req3)
129 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
130 assert not computed_blocks
131 + assert num_computed_tokens == 0
132 blocks = manager.allocate_slots(req3, 16 * 9, computed_blocks)
133 # This block ID order also checks the eviction order.
134 assert [b.block_id for b in blocks] == [9, 4, 3, 6, 5, 8, 7, 2, 1, 0]
@@ -150,8 +154,9 @@ def test_decode():
154 # Incomplete 1 block (7 tokens)
155 unique_token_ids = [3] * 7
156 req0 = make_request("0", common_token_ids + unique_token_ids)
153 - computed_blocks = manager.get_computed_blocks(req0)
157 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
158 assert not computed_blocks
159 + assert num_computed_tokens == 0
160 blocks = manager.allocate_slots(req0, 55, computed_blocks)
161 assert [b.block_id for b in blocks] == [0, 1, 2, 3, 4]
162
@@ -197,16 +202,18 @@ def test_evict():
202
203 last_token_id = 5 * 16 + 7
204 req0 = make_request("0", list(range(last_token_id)))
200 - computed_blocks = manager.get_computed_blocks(req0)
205 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
206 assert not computed_blocks
207 + assert num_computed_tokens == 0
208 blocks = manager.allocate_slots(req0, 5 * 16 + 7, computed_blocks)
209 assert len(blocks) == 7 # 5 full + 1 partial + 1 preallocated
210
211 # 3 blocks.
212 req1 = make_request("1", list(range(last_token_id,
213 last_token_id + 3 * 16)))
208 - computed_blocks = manager.get_computed_blocks(req1)
214 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
215 assert not computed_blocks
216 + assert num_computed_tokens == 0
217 blocks = manager.allocate_slots(req1, 3 * 16, computed_blocks)
218 assert len(blocks) == 3 # 3 full blocks
219 last_token_id += 3 * 16
@@ -222,8 +229,9 @@ def test_evict():
229
230 # Touch the first 2 blocks.
231 req2 = make_request("2", list(range(2 * 16 + 3)))
225 - computed_blocks = manager.get_computed_blocks(req2)
232 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
233 assert [b.block_id for b in computed_blocks] == [0, 1]
234 + assert num_computed_tokens == 2 * 16
235 blocks = manager.allocate_slots(req2, 3, computed_blocks)
236 assert [b.block_id for b in blocks] == [6, 5]
237 assert manager.free_block_queue.num_free_blocks == 6
@@ -247,8 +255,9 @@ def test_hash_block_correct_reuse():
255 # Allocate 1 block and cache it.
256 num_tokens = block_size * 1
257 req = make_request("0", list(range(num_tokens)))
250 - computed_blocks = manager.get_computed_blocks(req)
258 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
259 assert not computed_blocks
260 + assert num_computed_tokens == 0
261 blocks = manager.allocate_slots(req, num_tokens, computed_blocks)
262 assert len(blocks) == 1
263
@@ -258,8 +267,9 @@ def test_hash_block_correct_reuse():
267 # Allocate a new block that's not full, make sure hash info on the
268 # block is cleared.
269 req = make_request("1", list(range(num_tokens - 1)))
261 - computed_blocks = manager.get_computed_blocks(req)
270 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
271 assert not computed_blocks
272 + assert num_computed_tokens == 0
273 blocks = manager.allocate_slots(req, num_tokens - 1, computed_blocks)
274 assert len(blocks) == 1
275
@@ -284,16 +294,18 @@ def test_computed_blocks_not_evicted():
294 # Allocate a block and cache it.
295 num_tokens = block_size * 1
296 req0 = make_request("0", list(range(num_tokens)))
287 - computed_blocks = manager.get_computed_blocks(req0)
297 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
298 assert not computed_blocks
299 + assert num_computed_tokens == 0
300 blocks = manager.allocate_slots(req0, num_tokens, computed_blocks)
301 assert len(blocks) == 1
302 assert blocks[0].block_id == 0
303
304 # Allocate another block.
305 req1 = make_request("1", list(range(num_tokens, num_tokens * 2)))
295 - computed_blocks = manager.get_computed_blocks(req1)
306 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
307 assert not computed_blocks
308 + assert num_computed_tokens == 0
309 blocks = manager.allocate_slots(req1, num_tokens, computed_blocks)
310 assert len(blocks) == 1
311 assert blocks[0].block_id == 1
@@ -305,9 +317,10 @@ def test_computed_blocks_not_evicted():
317 # Now if we have a cache hit on the first block, we should evict the second
318 # cached block rather than the first one.
319 req2 = make_request("2", list(range(num_tokens * 2)))
308 - computed_blocks = manager.get_computed_blocks(req2)
320 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
321 assert len(computed_blocks) == 1
322 assert computed_blocks[0].block_id == 0
323 + assert num_computed_tokens == block_size
324
325 blocks = manager.allocate_slots(req2, num_tokens * 2 - num_tokens,
326 computed_blocks)
@@ -331,8 +344,9 @@ def test_basic_prefix_caching_disabled():
344
345 req1 = make_request("1", list(range(10))) # 2 blocks and some more
346
334 - computed_blocks = manager.get_computed_blocks(req1)
347 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
348 assert not computed_blocks
349 + assert num_computed_tokens == 0
350 blocks = manager.allocate_slots(req1, 10, computed_blocks)
351 assert len(blocks) == 3
352
@@ -341,15 +355,17 @@ def test_basic_prefix_caching_disabled():
355
356 # No caching.
357 req2 = make_request("2", list(range(16))) # shared prefix
344 - computed_blocks = manager.get_computed_blocks(req2)
358 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
359 assert not computed_blocks
360 + assert num_computed_tokens == 0
361 blocks = manager.allocate_slots(req2, 16, computed_blocks)
362 assert len(blocks) == 4
363
364 # New requests should not have any blocks.
365 req3 = make_request("3", list(range(4)))
351 - computed_blocks = manager.get_computed_blocks(req3)
366 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
367 assert not computed_blocks
368 + assert num_computed_tokens == 0
369 blocks = manager.allocate_slots(req3, 4, computed_blocks)
370 assert not blocks
371
@@ -371,8 +387,9 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
387 num_preallocated_blocks = cdiv(num_preallocate_tokens, block_size)
388
389 req = make_request("0", list(range(block_size * 30)))
374 - computed_blocks = manager.get_computed_blocks(req)
390 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
391 assert not computed_blocks
392 + assert num_computed_tokens == 0
393 # Just ask for 1 block.
394 blocks = manager.allocate_slots(req, block_size, computed_blocks)
395 req.num_computed_tokens = block_size
@@ -469,10 +486,11 @@ def test_mm_prefix_caching():
486 all_token_ids,
487 mm_positions=mm_positions,
488 mm_hashes=mm_hashes)
472 - computed_blocks = manager.get_computed_blocks(req0)
489 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
490
491 # Completed block should have hashes with extra keys.
492 assert not computed_blocks
493 + assert num_computed_tokens == 0
494 assert len(req0.kv_block_hashes) == 3
495 assert req0.kv_block_hashes[0].extra_keys == ("aaa", )
496 assert req0.kv_block_hashes[1].extra_keys == ("aaa", "bbb")
@@ -503,8 +521,9 @@ def test_mm_prefix_caching():
521 all_token_ids,
522 mm_positions=mm_positions,
523 mm_hashes=mm_hashes)
506 - computed_blocks = manager.get_computed_blocks(req1)
524 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
525 assert len(computed_blocks) == 3
526 + assert num_computed_tokens == 3 * 16
527
528
529 def test_prefill_not_enough_free_blocks_with_computed_blocks():
@@ -527,15 +546,17 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
546 # | Common-0 | Common-1 | Common-2 | ... |
547 common_token_ids = [i for i in range(3) for _ in range(16)]
548 req0 = make_request("0", common_token_ids)
530 - computed_blocks = manager.get_computed_blocks(req0)
549 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
550 assert not computed_blocks
551 + assert num_computed_tokens == 0
552 manager.allocate_slots(req0, 48, computed_blocks)
553 block_part0 = manager.req_to_blocks[req0.request_id]
554
555 # | Common-0 | Common-1 | Common-2 | Req1-3 | Req1-4 | Req1-5 | ... |
556 req1 = make_request("1", common_token_ids * 2)
537 - computed_blocks = manager.get_computed_blocks(req1)
557 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
558 assert computed_blocks == block_part0
559 + assert num_computed_tokens == 3 * 16
560 manager.allocate_slots(req1, 48, computed_blocks)
561 block_part1 = manager.req_to_blocks[req1.request_id]
562 # | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) |
@@ -547,8 +568,9 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
568 # | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) |
569 # | Req1-5(F)| Req2-0 | Req2-1 | ... |
570 req2 = make_request("2", [7] * block_size * 2)
550 - computed_blocks = manager.get_computed_blocks(req2)
571 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
572 assert not computed_blocks
573 + assert num_computed_tokens == 0
574 manager.allocate_slots(req2, block_size * 2, computed_blocks)
575
576 # Req3 is Req2 + 3 new blocks, so the first 6 blocks are computed,
@@ -556,8 +578,9 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
578 # In this case, the ref_cnt of the computed blocks should not be changed.
579 assert manager.free_block_queue.num_free_blocks == 5
580 req3 = make_request("3", common_token_ids * 3)
559 - computed_blocks = manager.get_computed_blocks(req3)
581 + computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
582 assert computed_blocks == block_part1
583 + assert num_computed_tokens == 6 * 16
584 # Req3 cannot be allocated.
585 assert manager.allocate_slots(req3, 48, computed_blocks) is None
586 # Block 0-2 are used by Req 1.
vllm/v1/core/kv_cache_manager.py
+12 -5
@@ -1,5 +1,5 @@
1 from collections import defaultdict
2 -from typing import Dict, Iterable, List, Optional
2 +from typing import Dict, Iterable, List, Optional, Tuple
3
4 from vllm.logger import init_logger
5 from vllm.utils import cdiv
@@ -69,7 +69,8 @@ class KVCacheManager:
69 # is finished.
70 self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
71
72 - def get_computed_blocks(self, request: Request) -> List[KVCacheBlock]:
72 + def get_computed_blocks(
73 + self, request: Request) -> Tuple[List[KVCacheBlock], int]:
74 """Get the computed (cached) blocks for the request.
75 Note that the computed blocks must be full.
76
@@ -77,11 +78,13 @@ class KVCacheManager:
78 request: The request to get the computed blocks.
79
80 Returns:
80 - A list of blocks that are computed for the request.
81 + A tuple containing:
82 + - A list of blocks that are computed for the request.
83 + - The number of computed tokens.
84 """
85 if not self.enable_caching:
86 # Prefix caching is disabled.
84 - return []
87 + return [], 0
88
89 computed_blocks = []
90
@@ -101,7 +104,11 @@ class KVCacheManager:
104 else:
105 break
106
104 - return computed_blocks
107 + # NOTE(woosuk): Since incomplete blocks are not eligible for
108 + # sharing, `num_computed_tokens` is always a multiple of
109 + # `block_size`.
110 + num_computed_tokens = len(computed_blocks) * self.block_size
111 + return computed_blocks, num_computed_tokens
112
113 def append_slots(
114 self,
vllm/v1/core/scheduler.py
+2 -6
@@ -184,12 +184,8 @@ class Scheduler:
184
185 request = self.waiting[0]
186 # Get already-cached tokens.
187 - computed_blocks = self.kv_cache_manager.get_computed_blocks(
188 - request)
189 - # NOTE(woosuk): Since incomplete blocks are not eligible for
190 - # sharing, `num_computed_tokens` is always a multiple of
191 - # `block_size`.
192 - num_computed_tokens = len(computed_blocks) * self.block_size
187 + computed_blocks, num_computed_tokens = \
188 + self.kv_cache_manager.get_computed_blocks(request)
189 # Number of tokens to be scheduled.
190 # We use `request.num_tokens` instead of
191 # `request.num_prompt_tokens` to consider the resumed requests,