A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Core][v1] Unify allocating slots in prefill and decode in KV cache manager (#12608)
As mentioned in RFC https://github.com/vllm-project/vllm/issues/12254, this PR achieves the task: combine allocate_slots and append_slots. There should be no functionality change, except that in decode, also raise exception when num_tokens is zero (like prefill), and change the unit test case accordingly. @comaniac @rickyyx @WoosukKwon @youkaichao @heheda12345 @simon-mo --------- Signed-off-by: Shawn Du <shawnd200@outlook.com>
Shawn Du committed
Feb 2, 2025 at 16:40 UTC
f8ece6e17fbf4ff3a98d6d53cb3a03c50c02828c
3 files changed
+78
-116
tests/v1/core/test_prefix_caching.py
+13
-11
@@ -164,7 +164,7 @@ def test_decode():
164
req0.num_computed_tokens = 55
165
for _ in range(4):
166
req0.append_output_token_ids(8)
167
- new_blocks = manager.append_slots(req0, 4)
167
+ new_blocks = manager.allocate_slots(req0, 4)
168
assert new_blocks is not None and len(new_blocks) == 0
169
assert manager.req_to_blocks[req0.request_id][-2].block_hash is None
170
@@ -175,7 +175,7 @@ def test_decode():
175
# the preallocated block.
176
for _ in range(5 + 10):
177
req0.append_output_token_ids(7)
178
- new_blocks = manager.append_slots(req0, 15)
178
+ new_blocks = manager.allocate_slots(req0, 15)
179
assert new_blocks is not None and len(new_blocks) == 0
180
assert manager.req_to_blocks[req0.request_id][-2].block_hash is not None
181
@@ -185,7 +185,7 @@ def test_decode():
185
# the preallocated block.
186
for _ in range(6 + 11):
187
req0.append_output_token_ids(12)
188
- new_blocks = manager.append_slots(req0, 17)
188
+ new_blocks = manager.allocate_slots(req0, 17)
189
# Plus one preallocated block.
190
assert new_blocks is not None and len(new_blocks) == 2
191
@@ -395,12 +395,14 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
395
req.num_computed_tokens = block_size
396
assert len(blocks) == 1 + num_preallocated_blocks
397
398
- # Assume all computed.
399
- manager.append_slots(req, block_size * (len(blocks) - 1))
400
- req.num_computed_tokens = block_size * len(blocks)
398
+ # Assume all computed, only when num_preallocate_tokens > 0, we need to
399
+ # consume the previously preallocated blocks.
400
+ if num_preallocated_blocks > 0:
401
+ manager.allocate_slots(req, block_size * (len(blocks) - 1))
402
+ req.num_computed_tokens = block_size * len(blocks)
403
404
# Append 1 block.
403
- blocks = manager.append_slots(req, block_size)
405
+ blocks = manager.allocate_slots(req, block_size)
406
assert len(blocks) == 1 + num_preallocated_blocks
407
408
@@ -503,7 +505,7 @@ def test_mm_prefix_caching():
505
# Append slots without allocating a new block.
506
for _ in range(5):
507
req0.append_output_token_ids(8)
506
- new_blocks = manager.append_slots(req0, 5)
508
+ new_blocks = manager.allocate_slots(req0, 5)
509
assert new_blocks is not None and len(new_blocks) == 0
510
511
# The just completed block should have hashes with extra keys.
@@ -603,7 +605,7 @@ def test_reset_prefix_cache():
605
unique_token_ids = [3] * 7
606
all_token_ids = full_block_token_ids + unique_token_ids
607
req0 = make_request("0", all_token_ids)
606
- blocks = manager.allocate_slots(req0, 55, [])
608
+ blocks = manager.allocate_slots(req0, 55)
609
assert [b.block_id for b in blocks] == [0, 1, 2, 3]
610
611
unique_token_ids = [4] * 7
@@ -639,7 +641,7 @@ def test_uncache_blocks():
641
)
642
643
req0 = make_request("0", list(range(30)))
642
- blocks = manager.allocate_slots(req0, 30, [])
644
+ blocks = manager.allocate_slots(req0, 30)
645
assert [b.block_id for b in blocks] == [0, 1]
646
assert len(manager.cached_block_hash_to_block) == 1
647
@@ -648,7 +650,7 @@ def test_uncache_blocks():
650
# Simulate speculative tokens.
651
for _ in range(5):
652
req0.append_output_token_ids(8)
651
- manager.append_slots(req0, 5)
653
+ manager.allocate_slots(req0, 5)
654
assert len(manager.cached_block_hash_to_block) == 2
655
656
# After sampling, assuming only 1 token is accepted.
vllm/v1/core/kv_cache_manager.py
+64
-104
@@ -1,5 +1,5 @@
1
from collections import defaultdict
2
-from typing import Dict, Iterable, List, Optional, Tuple
2
+from typing import DefaultDict, Dict, Iterable, List, Optional, Tuple
3
4
from vllm.logger import init_logger
5
from vllm.utils import cdiv
@@ -67,7 +67,8 @@ class KVCacheManager:
67
# Mapping from request ID to blocks to track the blocks allocated
68
# for each request, so that we can free the blocks when the request
69
# is finished.
70
- self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
70
+ self.req_to_blocks: DefaultDict[str,
71
+ List[KVCacheBlock]] = defaultdict(list)
72
73
@property
74
def usage(self) -> float:
@@ -115,33 +116,75 @@ class KVCacheManager:
116
num_computed_tokens = len(computed_blocks) * self.block_size
117
return computed_blocks, num_computed_tokens
118
118
- def append_slots(
119
+ def allocate_slots(
120
self,
121
request: Request,
122
num_tokens: int,
123
+ new_computed_blocks: Optional[List[KVCacheBlock]] = None
124
) -> Optional[List[KVCacheBlock]]:
123
- """Append slots to the block table of the request.
124
- We first append slots to already allocated blocks. If the allocated
125
- blocks are not enough, we allocate new blocks.
125
+ """Add slots for a request with new tokens to append.
126
127
Args:
128
- request: The request to append slots.
129
- num_tokens: The number of tokens to append.
128
+ request: The request to allocate slots.
129
+ num_tokens: The number of tokens to allocate. Note that this does
130
+ not include the tokens that have already been computed.
131
+ new_computed_blocks: A list of new computed blocks just hitting the
132
+ prefix caching.
133
+
134
+ Blocks layout:
135
+ -----------------------------------------------------------------------
136
+ | < computed > | < new computed > | < new > | < pre-allocated > |
137
+ -----------------------------------------------------------------------
138
+ | < required > |
139
+ --------------------------------------------------
140
+ | < full > |
141
+ ------------------------------------------------
142
+ | <new full> |
143
+ --------------
144
+ The following *_blocks are illustrated in this layout.
145
146
Returns:
132
- A list of new blocks if new blocks are allocated, or None
133
- if new blocks are required but cannot be allocated.
147
+ A list of new allocated blocks.
148
"""
135
- num_required_blocks = cdiv(request.num_computed_tokens + num_tokens,
149
+ if num_tokens == 0:
150
+ raise ValueError("num_tokens must be greater than 0")
151
+
152
+ new_computed_blocks = new_computed_blocks or []
153
+
154
+ # The number of computed tokens is the number of computed tokens plus
155
+ # the new prefix caching hits
156
+ num_computed_tokens = (request.num_computed_tokens +
157
+ len(new_computed_blocks) * self.block_size)
158
+ num_required_blocks = cdiv(num_computed_tokens + num_tokens,
159
self.block_size)
160
req_blocks = self.req_to_blocks[request.request_id]
161
+ num_new_blocks = (num_required_blocks - len(req_blocks) -
162
+ len(new_computed_blocks))
163
139
- num_new_blocks = num_required_blocks - len(req_blocks)
140
- if num_new_blocks > self.free_block_queue.num_free_blocks:
141
- # Need to allocate new blocks due to insufficient pre-allocated
142
- # slots, but we cannot allocate new blocks due to the limit.
164
+ # If a computed block of a request is an eviction candidate (in the
165
+ # free queue and ref_cnt == 0), it cannot be counted as a free block
166
+ # when allocating this request.
167
+ num_evictable_computed_blocks = sum(1 for blk in new_computed_blocks
168
+ if blk.ref_cnt == 0)
169
+ if (num_new_blocks > self.free_block_queue.num_free_blocks -
170
+ num_evictable_computed_blocks):
171
+ # Cannot allocate new blocks
172
return None
173
174
+ # Touch the computed blocks to make sure they won't be evicted.
175
+ if self.enable_caching:
176
+ self._touch(new_computed_blocks)
177
+ else:
178
+ assert not new_computed_blocks, (
179
+ "Computed blocks should be empty when "
180
+ "prefix caching is disabled")
181
+
182
+ # Append the new computed blocks to the request blocks until now to
183
+ # avoid the case where the new blocks cannot be allocated.
184
+ req_blocks.extend(new_computed_blocks)
185
+
186
+ # Start to handle new blocks
187
+
188
if num_new_blocks <= 0:
189
# No new block is needed.
190
new_blocks = []
@@ -160,112 +203,29 @@ class KVCacheManager:
203
)
204
assert num_new_blocks > 0
205
206
+ # Concatenate the computed block IDs and the new block IDs.
207
new_blocks = self._get_new_blocks(num_new_blocks)
208
req_blocks.extend(new_blocks)
209
210
if not self.enable_caching:
211
return new_blocks
212
169
- num_computed_full_blocks = (request.num_computed_tokens //
170
- self.block_size)
171
-
213
# NOTE(rickyx): We are assuming the `num_tokens` are actual
214
# tokens rather than lookahead slots (e.g. for speculative decoding).
215
# TODO(rickyx): When supporting speculative decoding, we will need to
216
# differentiate between them so that we can know how many blocks are
217
# full after appending the actual tokens.
177
- num_full_blocks_after_append = (request.num_computed_tokens +
178
- num_tokens) // self.block_size
179
- assert num_full_blocks_after_append <= len(req_blocks)
180
-
181
- new_full_blocks = req_blocks[
182
- num_computed_full_blocks:num_full_blocks_after_append]
183
- if new_full_blocks:
184
- self._cache_full_blocks(
185
- request=request,
186
- blk_start_idx=num_computed_full_blocks,
187
- full_blocks=new_full_blocks,
188
- prev_block=req_blocks[num_computed_full_blocks - 1]
189
- if num_computed_full_blocks >= 1 else None,
190
- )
191
-
192
- return new_blocks
193
-
194
- def allocate_slots(
195
- self,
196
- request: Request,
197
- num_tokens: int,
198
- computed_blocks: List[KVCacheBlock],
199
- ) -> Optional[List[KVCacheBlock]]:
200
- """Allocate slots for a new request.
201
-
202
- Args:
203
- request: The request to allocate slots.
204
- num_tokens: The number of tokens to allocate. Note that this does
205
- not include the tokens that have already been computed.
206
- computed_blocks: A list of computed blocks.
207
-
208
- Returns:
209
- A list of new allocated blocks.
210
- """
211
- if num_tokens == 0:
212
- raise ValueError(
213
- f"num_tokens must be greater than 0, got {num_tokens}")
214
-
215
- # If a computed block of a request is an eviction candidate (in the
216
- # free queue and ref_cnt == 0), it cannot be counted as a free block
217
- # when allocating this request.
218
- num_evictable_computed_blocks = sum(1 for blk in computed_blocks
219
- if blk.ref_cnt == 0)
220
-
221
- num_required_blocks = cdiv(num_tokens, self.block_size)
222
- if (num_required_blocks > self.free_block_queue.num_free_blocks -
223
- num_evictable_computed_blocks):
224
- # Cannot allocate new blocks.
225
- return None
226
-
227
- # Touch the computed blocks to make sure they won't be evicted.
228
- if self.enable_caching:
229
- self._touch(computed_blocks)
230
- else:
231
- assert not computed_blocks, (
232
- "Computed blocks should be empty when "
233
- "prefix caching is disabled")
234
-
235
- # Determine the number of new blocks to allocate considering
236
- # preallocated blocks.
237
- num_new_blocks = min(
238
- num_required_blocks + self.num_preallocate_blocks,
239
- self.free_block_queue.num_free_blocks,
240
- # Should not exceed the maximum number of blocks per request.
241
- # This is especially because the block table has the shape
242
- # [..., max_num_blocks_per_req].
243
- # TODO(woosuk): Check and reject requests if
244
- # num_prompt_tokens + max_tokens > max_model_len.
245
- self.max_num_blocks_per_req - len(computed_blocks),
246
- )
247
- assert num_new_blocks > 0
248
-
249
- # Concatenate the computed block IDs and the new block IDs.
250
- new_blocks = self._get_new_blocks(num_new_blocks)
251
- self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
252
-
253
- if not self.enable_caching:
254
- return new_blocks
255
-
256
- num_computed_tokens = len(computed_blocks) * self.block_size
218
num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
258
-
259
- new_full_blocks = self.req_to_blocks[
260
- request.request_id][len(computed_blocks):num_full_blocks]
219
+ num_computed_full_blocks = num_computed_tokens // self.block_size
220
+ new_full_blocks = req_blocks[num_computed_full_blocks:num_full_blocks]
221
if new_full_blocks:
222
self._cache_full_blocks(
223
request=request,
264
- blk_start_idx=len(computed_blocks),
224
+ blk_start_idx=num_computed_full_blocks,
225
# The new full blocks are the full blocks that are not computed.
226
full_blocks=new_full_blocks,
267
- prev_block=computed_blocks[-1] if computed_blocks else None,
268
- )
227
+ prev_block=(req_blocks[num_computed_full_blocks - 1]
228
+ if num_computed_full_blocks > 0 else None))
229
230
return new_blocks
231
vllm/v1/core/scheduler.py
+1
-1
@@ -138,7 +138,7 @@ class Scheduler:
138
assert num_new_tokens > 0
139
140
while True:
141
- new_blocks = self.kv_cache_manager.append_slots(
141
+ new_blocks = self.kv_cache_manager.allocate_slots(
142
request, num_new_tokens)
143
if new_blocks is None:
144
# The request cannot be scheduled.