@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f8ece6e17f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Core][v1] Unify allocating slots in prefill and decode in KV cache manager (#12608)

As mentioned in RFC https://github.com/vllm-project/vllm/issues/12254, this PR achieves the task: combine allocate_slots and append_slots. There should be no functionality change, except that in decode, also raise exception when num_tokens is zero (like prefill), and change the unit test case accordingly. @comaniac @rickyyx @WoosukKwon @youkaichao @heheda12345 @simon-mo --------- Signed-off-by: Shawn Du <shawnd200@outlook.com>

Shawn Du committed Feb 2, 2025 at 16:40 UTC f8ece6e17fbf4ff3a98d6d53cb3a03c50c02828c
3 files changed +78 -116
tests/v1/core/test_prefix_caching.py
+13 -11
@@ -164,7 +164,7 @@ def test_decode():
164 req0.num_computed_tokens = 55
165 for _ in range(4):
166 req0.append_output_token_ids(8)
167 - new_blocks = manager.append_slots(req0, 4)
167 + new_blocks = manager.allocate_slots(req0, 4)
168 assert new_blocks is not None and len(new_blocks) == 0
169 assert manager.req_to_blocks[req0.request_id][-2].block_hash is None
170
@@ -175,7 +175,7 @@ def test_decode():
175 # the preallocated block.
176 for _ in range(5 + 10):
177 req0.append_output_token_ids(7)
178 - new_blocks = manager.append_slots(req0, 15)
178 + new_blocks = manager.allocate_slots(req0, 15)
179 assert new_blocks is not None and len(new_blocks) == 0
180 assert manager.req_to_blocks[req0.request_id][-2].block_hash is not None
181
@@ -185,7 +185,7 @@ def test_decode():
185 # the preallocated block.
186 for _ in range(6 + 11):
187 req0.append_output_token_ids(12)
188 - new_blocks = manager.append_slots(req0, 17)
188 + new_blocks = manager.allocate_slots(req0, 17)
189 # Plus one preallocated block.
190 assert new_blocks is not None and len(new_blocks) == 2
191
@@ -395,12 +395,14 @@ def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
395 req.num_computed_tokens = block_size
396 assert len(blocks) == 1 + num_preallocated_blocks
397
398 - # Assume all computed.
399 - manager.append_slots(req, block_size * (len(blocks) - 1))
400 - req.num_computed_tokens = block_size * len(blocks)
398 + # Assume all computed, only when num_preallocate_tokens > 0, we need to
399 + # consume the previously preallocated blocks.
400 + if num_preallocated_blocks > 0:
401 + manager.allocate_slots(req, block_size * (len(blocks) - 1))
402 + req.num_computed_tokens = block_size * len(blocks)
403
404 # Append 1 block.
403 - blocks = manager.append_slots(req, block_size)
405 + blocks = manager.allocate_slots(req, block_size)
406 assert len(blocks) == 1 + num_preallocated_blocks
407
408
@@ -503,7 +505,7 @@ def test_mm_prefix_caching():
505 # Append slots without allocating a new block.
506 for _ in range(5):
507 req0.append_output_token_ids(8)
506 - new_blocks = manager.append_slots(req0, 5)
508 + new_blocks = manager.allocate_slots(req0, 5)
509 assert new_blocks is not None and len(new_blocks) == 0
510
511 # The just completed block should have hashes with extra keys.
@@ -603,7 +605,7 @@ def test_reset_prefix_cache():
605 unique_token_ids = [3] * 7
606 all_token_ids = full_block_token_ids + unique_token_ids
607 req0 = make_request("0", all_token_ids)
606 - blocks = manager.allocate_slots(req0, 55, [])
608 + blocks = manager.allocate_slots(req0, 55)
609 assert [b.block_id for b in blocks] == [0, 1, 2, 3]
610
611 unique_token_ids = [4] * 7
@@ -639,7 +641,7 @@ def test_uncache_blocks():
641 )
642
643 req0 = make_request("0", list(range(30)))
642 - blocks = manager.allocate_slots(req0, 30, [])
644 + blocks = manager.allocate_slots(req0, 30)
645 assert [b.block_id for b in blocks] == [0, 1]
646 assert len(manager.cached_block_hash_to_block) == 1
647
@@ -648,7 +650,7 @@ def test_uncache_blocks():
650 # Simulate speculative tokens.
651 for _ in range(5):
652 req0.append_output_token_ids(8)
651 - manager.append_slots(req0, 5)
653 + manager.allocate_slots(req0, 5)
654 assert len(manager.cached_block_hash_to_block) == 2
655
656 # After sampling, assuming only 1 token is accepted.
vllm/v1/core/kv_cache_manager.py
+64 -104
@@ -1,5 +1,5 @@
1 from collections import defaultdict
2 -from typing import Dict, Iterable, List, Optional, Tuple
2 +from typing import DefaultDict, Dict, Iterable, List, Optional, Tuple
3
4 from vllm.logger import init_logger
5 from vllm.utils import cdiv
@@ -67,7 +67,8 @@ class KVCacheManager:
67 # Mapping from request ID to blocks to track the blocks allocated
68 # for each request, so that we can free the blocks when the request
69 # is finished.
70 - self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
70 + self.req_to_blocks: DefaultDict[str,
71 + List[KVCacheBlock]] = defaultdict(list)
72
73 @property
74 def usage(self) -> float:
@@ -115,33 +116,75 @@ class KVCacheManager:
116 num_computed_tokens = len(computed_blocks) * self.block_size
117 return computed_blocks, num_computed_tokens
118
118 - def append_slots(
119 + def allocate_slots(
120 self,
121 request: Request,
122 num_tokens: int,
123 + new_computed_blocks: Optional[List[KVCacheBlock]] = None
124 ) -> Optional[List[KVCacheBlock]]:
123 - """Append slots to the block table of the request.
124 - We first append slots to already allocated blocks. If the allocated
125 - blocks are not enough, we allocate new blocks.
125 + """Add slots for a request with new tokens to append.
126
127 Args:
128 - request: The request to append slots.
129 - num_tokens: The number of tokens to append.
128 + request: The request to allocate slots.
129 + num_tokens: The number of tokens to allocate. Note that this does
130 + not include the tokens that have already been computed.
131 + new_computed_blocks: A list of new computed blocks just hitting the
132 + prefix caching.
133 +
134 + Blocks layout:
135 + -----------------------------------------------------------------------
136 + | < computed > | < new computed > | < new > | < pre-allocated > |
137 + -----------------------------------------------------------------------
138 + | < required > |
139 + --------------------------------------------------
140 + | < full > |
141 + ------------------------------------------------
142 + | <new full> |
143 + --------------
144 + The following *_blocks are illustrated in this layout.
145
146 Returns:
132 - A list of new blocks if new blocks are allocated, or None
133 - if new blocks are required but cannot be allocated.
147 + A list of new allocated blocks.
148 """
135 - num_required_blocks = cdiv(request.num_computed_tokens + num_tokens,
149 + if num_tokens == 0:
150 + raise ValueError("num_tokens must be greater than 0")
151 +
152 + new_computed_blocks = new_computed_blocks or []
153 +
154 + # The number of computed tokens is the number of computed tokens plus
155 + # the new prefix caching hits
156 + num_computed_tokens = (request.num_computed_tokens +
157 + len(new_computed_blocks) * self.block_size)
158 + num_required_blocks = cdiv(num_computed_tokens + num_tokens,
159 self.block_size)
160 req_blocks = self.req_to_blocks[request.request_id]
161 + num_new_blocks = (num_required_blocks - len(req_blocks) -
162 + len(new_computed_blocks))
163
139 - num_new_blocks = num_required_blocks - len(req_blocks)
140 - if num_new_blocks > self.free_block_queue.num_free_blocks:
141 - # Need to allocate new blocks due to insufficient pre-allocated
142 - # slots, but we cannot allocate new blocks due to the limit.
164 + # If a computed block of a request is an eviction candidate (in the
165 + # free queue and ref_cnt == 0), it cannot be counted as a free block
166 + # when allocating this request.
167 + num_evictable_computed_blocks = sum(1 for blk in new_computed_blocks
168 + if blk.ref_cnt == 0)
169 + if (num_new_blocks > self.free_block_queue.num_free_blocks -
170 + num_evictable_computed_blocks):
171 + # Cannot allocate new blocks
172 return None
173
174 + # Touch the computed blocks to make sure they won't be evicted.
175 + if self.enable_caching:
176 + self._touch(new_computed_blocks)
177 + else:
178 + assert not new_computed_blocks, (
179 + "Computed blocks should be empty when "
180 + "prefix caching is disabled")
181 +
182 + # Append the new computed blocks to the request blocks until now to
183 + # avoid the case where the new blocks cannot be allocated.
184 + req_blocks.extend(new_computed_blocks)
185 +
186 + # Start to handle new blocks
187 +
188 if num_new_blocks <= 0:
189 # No new block is needed.
190 new_blocks = []
@@ -160,112 +203,29 @@ class KVCacheManager:
203 )
204 assert num_new_blocks > 0
205
206 + # Concatenate the computed block IDs and the new block IDs.
207 new_blocks = self._get_new_blocks(num_new_blocks)
208 req_blocks.extend(new_blocks)
209
210 if not self.enable_caching:
211 return new_blocks
212
169 - num_computed_full_blocks = (request.num_computed_tokens //
170 - self.block_size)
171 -
213 # NOTE(rickyx): We are assuming the `num_tokens` are actual
214 # tokens rather than lookahead slots (e.g. for speculative decoding).
215 # TODO(rickyx): When supporting speculative decoding, we will need to
216 # differentiate between them so that we can know how many blocks are
217 # full after appending the actual tokens.
177 - num_full_blocks_after_append = (request.num_computed_tokens +
178 - num_tokens) // self.block_size
179 - assert num_full_blocks_after_append <= len(req_blocks)
180 -
181 - new_full_blocks = req_blocks[
182 - num_computed_full_blocks:num_full_blocks_after_append]
183 - if new_full_blocks:
184 - self._cache_full_blocks(
185 - request=request,
186 - blk_start_idx=num_computed_full_blocks,
187 - full_blocks=new_full_blocks,
188 - prev_block=req_blocks[num_computed_full_blocks - 1]
189 - if num_computed_full_blocks >= 1 else None,
190 - )
191 -
192 - return new_blocks
193 -
194 - def allocate_slots(
195 - self,
196 - request: Request,
197 - num_tokens: int,
198 - computed_blocks: List[KVCacheBlock],
199 - ) -> Optional[List[KVCacheBlock]]:
200 - """Allocate slots for a new request.
201 -
202 - Args:
203 - request: The request to allocate slots.
204 - num_tokens: The number of tokens to allocate. Note that this does
205 - not include the tokens that have already been computed.
206 - computed_blocks: A list of computed blocks.
207 -
208 - Returns:
209 - A list of new allocated blocks.
210 - """
211 - if num_tokens == 0:
212 - raise ValueError(
213 - f"num_tokens must be greater than 0, got {num_tokens}")
214 -
215 - # If a computed block of a request is an eviction candidate (in the
216 - # free queue and ref_cnt == 0), it cannot be counted as a free block
217 - # when allocating this request.
218 - num_evictable_computed_blocks = sum(1 for blk in computed_blocks
219 - if blk.ref_cnt == 0)
220 -
221 - num_required_blocks = cdiv(num_tokens, self.block_size)
222 - if (num_required_blocks > self.free_block_queue.num_free_blocks -
223 - num_evictable_computed_blocks):
224 - # Cannot allocate new blocks.
225 - return None
226 -
227 - # Touch the computed blocks to make sure they won't be evicted.
228 - if self.enable_caching:
229 - self._touch(computed_blocks)
230 - else:
231 - assert not computed_blocks, (
232 - "Computed blocks should be empty when "
233 - "prefix caching is disabled")
234 -
235 - # Determine the number of new blocks to allocate considering
236 - # preallocated blocks.
237 - num_new_blocks = min(
238 - num_required_blocks + self.num_preallocate_blocks,
239 - self.free_block_queue.num_free_blocks,
240 - # Should not exceed the maximum number of blocks per request.
241 - # This is especially because the block table has the shape
242 - # [..., max_num_blocks_per_req].
243 - # TODO(woosuk): Check and reject requests if
244 - # num_prompt_tokens + max_tokens > max_model_len.
245 - self.max_num_blocks_per_req - len(computed_blocks),
246 - )
247 - assert num_new_blocks > 0
248 -
249 - # Concatenate the computed block IDs and the new block IDs.
250 - new_blocks = self._get_new_blocks(num_new_blocks)
251 - self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
252 -
253 - if not self.enable_caching:
254 - return new_blocks
255 -
256 - num_computed_tokens = len(computed_blocks) * self.block_size
218 num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
258 -
259 - new_full_blocks = self.req_to_blocks[
260 - request.request_id][len(computed_blocks):num_full_blocks]
219 + num_computed_full_blocks = num_computed_tokens // self.block_size
220 + new_full_blocks = req_blocks[num_computed_full_blocks:num_full_blocks]
221 if new_full_blocks:
222 self._cache_full_blocks(
223 request=request,
264 - blk_start_idx=len(computed_blocks),
224 + blk_start_idx=num_computed_full_blocks,
225 # The new full blocks are the full blocks that are not computed.
226 full_blocks=new_full_blocks,
267 - prev_block=computed_blocks[-1] if computed_blocks else None,
268 - )
227 + prev_block=(req_blocks[num_computed_full_blocks - 1]
228 + if num_computed_full_blocks > 0 else None))
229
230 return new_blocks
231
vllm/v1/core/scheduler.py
+1 -1
@@ -138,7 +138,7 @@ class Scheduler:
138 assert num_new_tokens > 0
139
140 while True:
141 - new_blocks = self.kv_cache_manager.append_slots(
141 + new_blocks = self.kv_cache_manager.allocate_slots(
142 request, num_new_tokens)
143 if new_blocks is None:
144 # The request cannot be scheduled.