@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97814fbf0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[v1] Refactor KVCacheManager for more hash input than token ids (#10507)

Signed-off-by: rickyx <rickyx@anyscale.com> Signed-off-by: Cody Yu <hao.yu.cody@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Ricky Xu committed Nov 22, 2024 at 15:27 UTC 97814fbf0f847a11d2e0eb339e3e7572ca69379d
3 files changed +365 -186
tests/v1/core/test_prefix_caching.py
+206 -19
@@ -1,8 +1,11 @@
1 """Compare the with and without prefix caching."""
2 +import pytest
3 +
4 from vllm.inputs import token_inputs
5 from vllm.sampling_params import SamplingParams
6 +from vllm.utils import cdiv
7 from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5 -from vllm.v1.core.kv_cache_utils import hash_block_tokens
8 +from vllm.v1.core.kv_cache_utils import KVCacheBlock, hash_block_tokens
9
10
11 def make_request(request_id, prompt_token_ids):
@@ -31,7 +34,8 @@ def test_prefill():
34 # Fully cache miss
35 # Incomplete 1 block (7 tokens)
36 unique_token_ids = [3] * 7
34 - req0 = make_request("0", common_token_ids + unique_token_ids)
37 + all_token_ids = common_token_ids + unique_token_ids
38 + req0 = make_request("0", all_token_ids)
39 computed_blocks = manager.get_computed_blocks(req0)
40 assert not computed_blocks
41 blocks = manager.allocate_slots(req0, 55, computed_blocks)
@@ -40,24 +44,16 @@ def test_prefill():
44 # Check full block metadata
45 parent_block_hash = None
46 for block_id in (0, 1, 2):
43 - block_hash = hash_block_tokens(parent_block_hash,
44 - manager.block_pool[block_id].token_ids)
47 + block_tokens = tuple(all_token_ids[block_id * 16:(block_id + 1) * 16])
48 + block_hash = hash_block_tokens(parent_block_hash, block_tokens)
49 assert manager.block_pool[block_id].block_hash == block_hash
50 assert manager.block_pool[block_id].ref_cnt == 1
47 - assert manager.block_pool[block_id].num_hashed_tokens == 16 * (
48 - block_id + 1)
49 - assert manager.block_pool[block_id].token_ids == tuple([block_id] * 16)
51 parent_block_hash = block_hash
52
53 # Check partial/preallocated block metadata
54 for block_id in (3, 4):
55 assert manager.block_pool[block_id].block_hash is None
56 assert manager.block_pool[block_id].ref_cnt == 1
56 - assert manager.block_pool[block_id].num_hashed_tokens == 0
57 - if block_id == 3:
58 - assert manager.block_pool[block_id].token_ids == [3] * 7
59 - else:
60 - assert not manager.block_pool[block_id].token_ids
57
58 # Cache hit in the common prefix when the original block is still in use.
59 # Incomplete 1 block (5 tokens)
@@ -113,7 +109,7 @@ def test_prefill():
109 req3 = make_request("3", [99] * (16 * 9))
110 computed_blocks = manager.get_computed_blocks(req3)
111 assert not computed_blocks
116 - blocks = manager.allocate_slots(req2, 16 * 9, computed_blocks)
112 + blocks = manager.allocate_slots(req3, 16 * 9, computed_blocks)
113 # This block ID order also checks the eviction order.
114 assert [b.block_id for b in blocks] == [9, 4, 3, 6, 5, 8, 7, 2, 1, 0]
115 assert manager.free_block_queue.num_free_blocks == 0
@@ -148,7 +144,7 @@ def test_decode():
144 req0.append_output_token_ids(8)
145 new_blocks = manager.append_slots(req0, 4)
146 assert new_blocks is not None and len(new_blocks) == 0
151 - assert len(manager.block_pool[3].token_ids) == 11
147 + assert manager.req_to_blocks[req0.request_id][-2].block_hash is None
148
149 # Append slots without allocating a new block, but start using the
150 # preallocated block.
@@ -159,8 +155,7 @@ def test_decode():
155 req0.append_output_token_ids(7)
156 new_blocks = manager.append_slots(req0, 15)
157 assert new_blocks is not None and len(new_blocks) == 0
162 - assert len(manager.block_pool[3].token_ids) == 16
163 - assert len(manager.block_pool[4].token_ids) == 10
158 + assert manager.req_to_blocks[req0.request_id][-2].block_hash is not None
159
160 # Append slots with allocating a new block.
161 req0.num_computed_tokens = 74
@@ -171,9 +166,6 @@ def test_decode():
166 new_blocks = manager.append_slots(req0, 17)
167 # Plus one preallocated block.
168 assert new_blocks is not None and len(new_blocks) == 2
174 - assert len(manager.block_pool[4].token_ids) == 16
175 - assert len(manager.block_pool[5].token_ids) == 11
176 - assert len(manager.block_pool[6].token_ids) == 0
169
170
171 def test_evict():
@@ -217,3 +209,198 @@ def test_evict():
209 blocks = manager.allocate_slots(req2, 3, computed_blocks)
210 assert [b.block_id for b in blocks] == [6, 5]
211 assert manager.free_block_queue.num_free_blocks == 6
212 +
213 +
214 +def test_hash_block_correct_reuse():
215 + """
216 + This tests when a previously cached block is reused as a new block,
217 + its hash metadata should be correctly reset.
218 + """
219 + block_size = 16
220 + manager = KVCacheManager(
221 + block_size=block_size,
222 + num_gpu_blocks=1,
223 + sliding_window=False,
224 + enable_caching=True,
225 + num_preallocate_tokens=0,
226 + )
227 +
228 + # Allocate 1 block and cache it.
229 + num_tokens = block_size * 1
230 + req = make_request("0", list(range(num_tokens)))
231 + computed_blocks = manager.get_computed_blocks(req)
232 + assert not computed_blocks
233 + blocks = manager.allocate_slots(req, num_tokens, computed_blocks)
234 + assert len(blocks) == 1
235 +
236 + # Deallocate the block.
237 + manager.free(req)
238 +
239 + # Allocate a new block that's not full, make sure hash info on the
240 + # block is cleared.
241 + req = make_request("1", list(range(num_tokens - 1)))
242 + computed_blocks = manager.get_computed_blocks(req)
243 + assert not computed_blocks
244 + blocks = manager.allocate_slots(req, num_tokens - 1, computed_blocks)
245 + assert len(blocks) == 1
246 +
247 + assert manager.block_pool[blocks[0].block_id].block_hash is None
248 +
249 +
250 +def test_computed_blocks_not_evicted():
251 + """
252 + Test that the computed blocks are not evicted when getting new blocks
253 + for a request if there are any other free blocks.
254 + """
255 + block_size = 16
256 + manager = KVCacheManager(
257 + block_size=block_size,
258 + num_gpu_blocks=2,
259 + sliding_window=False,
260 + enable_caching=True,
261 + num_preallocate_tokens=0,
262 + )
263 +
264 + # Allocate a block and cache it.
265 + num_tokens = block_size * 1
266 + req0 = make_request("0", list(range(num_tokens)))
267 + computed_blocks = manager.get_computed_blocks(req0)
268 + assert not computed_blocks
269 + blocks = manager.allocate_slots(req0, num_tokens, computed_blocks)
270 + assert len(blocks) == 1
271 + assert blocks[0].block_id == 0
272 +
273 + # Allocate another block.
274 + req1 = make_request("1", list(range(num_tokens, num_tokens * 2)))
275 + computed_blocks = manager.get_computed_blocks(req1)
276 + assert not computed_blocks
277 + blocks = manager.allocate_slots(req1, num_tokens, computed_blocks)
278 + assert len(blocks) == 1
279 + assert blocks[0].block_id == 1
280 +
281 + # Free the blocks.
282 + manager.free(req0)
283 + manager.free(req1)
284 +
285 + # Now if we have a cache hit on the first block, we should evict the second
286 + # cached block rather than the first one.
287 + req2 = make_request("2", list(range(num_tokens * 2)))
288 + computed_blocks = manager.get_computed_blocks(req2)
289 + assert len(computed_blocks) == 1
290 + assert computed_blocks[0].block_id == 0
291 +
292 + blocks = manager.allocate_slots(req2, num_tokens * 2 - num_tokens,
293 + computed_blocks)
294 + assert len(blocks) == 1
295 + assert blocks[0].block_id == 1
296 +
297 +
298 +def test_basic_prefix_caching_disabled():
299 + """
300 + This tests that the prefix caching is disabled.
301 + """
302 + block_size = 4
303 + manager = KVCacheManager(
304 + block_size=block_size,
305 + num_gpu_blocks=4,
306 + sliding_window=False,
307 + enable_caching=False,
308 + num_preallocate_tokens=0,
309 + )
310 +
311 + req1 = make_request("1", list(range(10))) # 2 blocks and some more
312 +
313 + computed_blocks = manager.get_computed_blocks(req1)
314 + assert not computed_blocks
315 + blocks = manager.allocate_slots(req1, 10, computed_blocks)
316 + assert len(blocks) == 3
317 +
318 + # Free the blocks.
319 + manager.free(req1)
320 +
321 + # No caching.
322 + req2 = make_request("2", list(range(16))) # shared prefix
323 + computed_blocks = manager.get_computed_blocks(req2)
324 + assert not computed_blocks
325 + blocks = manager.allocate_slots(req2, 16, computed_blocks)
326 + assert len(blocks) == 4
327 +
328 + # New requests should not have any blocks.
329 + req3 = make_request("3", list(range(4)))
330 + computed_blocks = manager.get_computed_blocks(req3)
331 + assert not computed_blocks
332 + blocks = manager.allocate_slots(req3, 4, computed_blocks)
333 + assert not blocks
334 +
335 +
336 +@pytest.mark.parametrize("num_preallocate_tokens", list(range(0, 8)))
337 +@pytest.mark.parametrize("block_size", [4])
338 +def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
339 + """
340 + This tests that the preallocated blocks are correctly added.
341 + """
342 + manager = KVCacheManager(
343 + block_size=block_size,
344 + num_gpu_blocks=10,
345 + sliding_window=False,
346 + enable_caching=True,
347 + num_preallocate_tokens=num_preallocate_tokens,
348 + )
349 + num_preallocated_blocks = cdiv(num_preallocate_tokens, block_size)
350 +
351 + req = make_request("0", list(range(block_size * 30)))
352 + computed_blocks = manager.get_computed_blocks(req)
353 + assert not computed_blocks
354 + # Just ask for 1 block.
355 + blocks = manager.allocate_slots(req, block_size, computed_blocks)
356 + assert len(blocks) == 1 + num_preallocated_blocks
357 +
358 + # Append slots to the block.
359 + req.num_computed_tokens = block_size * len(blocks) # Assume all used.
360 + blocks = manager.append_slots(req, block_size) # Append 1 block.
361 + assert len(blocks) == 1 + num_preallocated_blocks
362 +
363 +
364 +def test_cache_blocks():
365 + """
366 + This is a unit test that tests the correctness of the _cache_full_blocks
367 + function of KVCacheManager.
368 + """
369 + block_size = 4
370 + manager = KVCacheManager(
371 + block_size=block_size,
372 + num_gpu_blocks=5,
373 + sliding_window=False,
374 + enable_caching=True,
375 + num_preallocate_tokens=0,
376 + )
377 + # Req:
378 + # Block 0: [0, 1, 2, 3]
379 + # Block 1: [4, 5, 6, 7]
380 + # Block 2: [8, 9, 10, 11]
381 + # Block 3: [12, 13]
382 + req = make_request("0", list(range(14)))
383 +
384 + # Test that blocks are cached correctly for 2 full blocks from the start.
385 + blocks = [KVCacheBlock(block_id=i) for i in range(2)]
386 +
387 + manager._cache_full_blocks(
388 + request=req,
389 + blk_start_idx=0,
390 + full_blocks=blocks,
391 + prev_block=None,
392 + )
393 +
394 + assert len(manager.cached_block_hash_to_block) == 2
395 + assert all([block.block_hash is not None for block in blocks])
396 +
397 + # Test that blocks that don't start from the beginning are cached correctly.
398 + blocks = [KVCacheBlock(block_id=2)]
399 + manager._cache_full_blocks(
400 + request=req,
401 + blk_start_idx=2,
402 + full_blocks=blocks,
403 + prev_block=None,
404 + )
405 + assert len(manager.cached_block_hash_to_block) == 3
406 + assert blocks[0].block_hash is not None
vllm/v1/core/kv_cache_manager.py
+137 -152
@@ -79,6 +79,9 @@ class KVCacheManager:
79 return []
80
81 computed_blocks = []
82 +
83 + # TODO(rickyx): potentially we could cache this so we don't have to
84 + # recompute it every time.
85 block_hashes = hash_request_tokens(self.block_size,
86 request.all_token_ids)
87
@@ -120,47 +123,45 @@ class KVCacheManager:
123 # slots, but we cannot allocate new blocks due to the limit.
124 return None
125
123 - # When caching is enabled, assign token IDs to already allocated blocks.
124 - new_token_ids = None
125 - parent_block = None
126 - if self.enable_caching:
127 - # Figure out the token IDs to add to the blocks.
128 - new_token_ids = request.all_token_ids[
129 - request.num_computed_tokens:request.num_computed_tokens +
130 - num_tokens]
131 -
132 - # Find the last full block index.
133 - # TODO: This may be optimized by calculating the computed tokens.
134 - last_full_block_idx = len(req_blocks) - 1
135 - while (last_full_block_idx >= 0
136 - and req_blocks[last_full_block_idx].block_hash is None):
137 - last_full_block_idx -= 1
138 -
139 - parent_block = (req_blocks[last_full_block_idx]
140 - if last_full_block_idx >= 0 else None)
141 - token_id_idx = self._add_token_ids_to_blocks(
142 - blocks=req_blocks[last_full_block_idx + 1:],
143 - token_ids=new_token_ids,
144 - parent_block=parent_block)
145 -
146 - new_token_ids = new_token_ids[token_id_idx:]
147 - parent_block = req_blocks[-1]
148 -
149 - # No new block is needed. When caching is enabled, we make sure
150 - # token_id_idx is equal to len(new_token_ids), meaning that all tokens
151 - # are added to allocated blocks.
152 - if num_required_blocks <= len(req_blocks):
153 - assert not self.enable_caching or token_id_idx == num_tokens, \
154 - f"{token_id_idx=} != {num_tokens=}"
155 - return []
126 + if num_new_blocks <= 0:
127 + # No new block is needed.
128 + new_blocks = []
129 + else:
130 + # Get new blocks from the free block pool considering
131 + # preallocated blocks.
132 + num_new_blocks = min(
133 + num_new_blocks + self.num_preallocate_blocks,
134 + self.free_block_queue.num_free_blocks,
135 + )
136 +
137 + new_blocks = self._get_new_blocks(num_new_blocks)
138 + req_blocks.extend(new_blocks)
139 +
140 + if not self.enable_caching:
141 + return new_blocks
142 +
143 + num_computed_full_blocks = (request.num_computed_tokens //
144 + self.block_size)
145 +
146 + # NOTE(rickyx): We are assuming the `num_tokens` are actual
147 + # tokens rather than lookahead slots (e.g. for speculative decoding).
148 + # TODO(rickyx): When supporting speculative decoding, we will need to
149 + # differentiate between them so that we can know how many blocks are
150 + # full after appending the actual tokens.
151 + num_full_blocks_after_append = (request.num_computed_tokens +
152 + num_tokens) // self.block_size
153 + assert num_full_blocks_after_append <= len(req_blocks)
154 +
155 + new_full_blocks = req_blocks[
156 + num_computed_full_blocks:num_full_blocks_after_append]
157 + self._cache_full_blocks(
158 + request=request,
159 + blk_start_idx=num_computed_full_blocks,
160 + full_blocks=new_full_blocks,
161 + prev_block=req_blocks[num_computed_full_blocks - 1]
162 + if num_computed_full_blocks >= 1 else None,
163 + )
164
157 - # Allocate new blocks considering preallocated blocks, and
158 - # add token IDs to them if caching is enabled.
159 - num_new_blocks = min(num_new_blocks + self.num_preallocate_blocks,
160 - self.free_block_queue.num_free_blocks)
161 - new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
162 - parent_block)
163 - req_blocks.extend(new_blocks)
165 return new_blocks
166
167 def allocate_slots(
@@ -184,11 +185,20 @@ class KVCacheManager:
185 raise ValueError(
186 f"num_tokens must be greater than 0, got {num_tokens}")
187
187 - # If a computed block of a request is an eviction candidate (in the
188 - # free queue and ref_cnt == 0), it cannot be counted as a free block
189 - # when allocating this request.
190 - num_evictable_computed_blocks = len(
191 - [blk for blk in computed_blocks if blk.ref_cnt == 0])
188 + # Touch the computed blocks to make sure they won't be evicted.
189 + num_evictable_computed_blocks = 0
190 + if self.enable_caching:
191 + self._touch(computed_blocks)
192 +
193 + # If a computed block of a request is an eviction candidate (in the
194 + # free queue and ref_cnt == 0), it cannot be counted as a free block
195 + # when allocating this request.
196 + num_evictable_computed_blocks = len(
197 + [blk for blk in computed_blocks if blk.ref_cnt == 0])
198 + else:
199 + assert not computed_blocks, (
200 + "Computed blocks should be empty when "
201 + "prefix caching is disabled")
202
203 num_required_blocks = cdiv(num_tokens, self.block_size)
204 if (num_required_blocks > self.free_block_queue.num_free_blocks -
@@ -201,35 +211,28 @@ class KVCacheManager:
211 num_new_blocks = min(
212 num_required_blocks + self.num_preallocate_blocks,
213 self.free_block_queue.num_free_blocks -
204 - num_evictable_computed_blocks)
205 -
206 - num_computed_tokens = len(computed_blocks) * self.block_size
214 + num_evictable_computed_blocks,
215 + )
216
208 - # When caching is enabled, get the new token IDs and the parent block
209 - # ID to generate cache keys.
210 - new_token_ids = None
211 - parent_block = None
212 - if self.enable_caching:
213 - # Touch the computed blocks to make sure they won't be evicted.
214 - self._touch(computed_blocks)
217 + # Concatenate the computed block IDs and the new block IDs.
218 + new_blocks = self._get_new_blocks(num_new_blocks)
219 + self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
220
216 - # Get the token IDs for the blocks being allocated for hashing.
217 - new_token_ids = request.all_token_ids[
218 - num_computed_tokens:num_computed_tokens + num_tokens]
219 - if not new_token_ids:
220 - raise RuntimeError(
221 - "Failed to infer the token IDs for allocation. "
222 - f"#all_tokens={len(request.all_token_ids)} < "
223 - f"#computed_tokens={num_computed_tokens}")
221 + if not self.enable_caching:
222 + return new_blocks
223
225 - # Get the parent block ID to construct the block chain.
226 - parent_block = computed_blocks[-1] if computed_blocks else None
224 + num_computed_tokens = len(computed_blocks) * self.block_size
225 + num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
226
228 - new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
229 - parent_block)
227 + self._cache_full_blocks(
228 + request=request,
229 + blk_start_idx=len(computed_blocks),
230 + # The new full blocks are the full blocks that are not computed.
231 + full_blocks=self.req_to_blocks[request.request_id]
232 + [len(computed_blocks):num_full_blocks],
233 + prev_block=computed_blocks[-1] if computed_blocks else None,
234 + )
235
231 - # Concatenate the computed block IDs and the new block IDs.
232 - self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
236 return new_blocks
237
238 def free(self, request: Request) -> None:
@@ -248,24 +251,17 @@ class KVCacheManager:
251 blocks = reversed(blocks)
252
253 for block in blocks:
251 - block.ref_cnt -= 1
254 + block.decr_ref()
255 if block.ref_cnt == 0:
256 self.free_block_queue.append(block)
257
255 - def _get_new_blocks(
256 - self,
257 - num_blocks: int,
258 - token_ids: Optional[List[int]] = None,
259 - parent_block: Optional[int] = None) -> List[KVCacheBlock]:
260 - """Get new blocks from the free block pool, and add token IDs to
261 - allocated blocks if caching is enabled.
258 + def _get_new_blocks(self, num_blocks: int) -> List[KVCacheBlock]:
259 + """Get new blocks from the free block pool.
260 +
261 Note that we do not check block cache in this function.
262
263 Args:
264 num_blocks: The number of blocks to allocate.
266 - token_ids: The token IDs in the blocks. None if caching is disabled.
267 - parent_block: The parent block. Used to include block chain
268 - in the block hash.
265
266 Returns:
267 A list of new block.
@@ -274,56 +270,38 @@ class KVCacheManager:
270 raise ValueError(
271 f"Cannot get {num_blocks} free blocks from the pool")
272
277 - # First allocate blocks.
273 ret: List[KVCacheBlock] = []
274 idx = 0
275 while idx < num_blocks:
276 + # First allocate blocks.
277 curr_block = self.free_block_queue.popleft()
278 assert curr_block.ref_cnt == 0
279
284 - # Evict blocks from the cache.
280 + # If the block is cached, evict it.
281 if self.enable_caching:
286 - block_hash = curr_block.block_hash
287 - if (block_hash is not None
288 - and block_hash in self.cached_block_hash_to_block):
289 - if len(self.cached_block_hash_to_block[block_hash]) == 1:
290 - del self.cached_block_hash_to_block[block_hash]
291 - else:
292 - del self.cached_block_hash_to_block[block_hash][
293 - curr_block.block_id]
294 - curr_block.reset()
295 -
296 - curr_block.ref_cnt = 1
282 + self._evict_cached_block(curr_block)
283 +
284 + curr_block.incr_ref()
285 ret.append(curr_block)
286 idx += 1
287
300 - # Then assign token IDs to the allocated blocks.
301 - if self.enable_caching:
302 - assert token_ids is not None
303 - token_id_idx = self._add_token_ids_to_blocks(
304 - blocks=ret, token_ids=token_ids, parent_block=parent_block)
305 - assert token_id_idx == len(token_ids)
306 -
288 return ret
289
309 - def _cache_full_block(self,
310 - block: KVCacheBlock,
311 - parent_block: Optional[KVCacheBlock] = None) -> None:
312 - """Cache a full block for prefix caching.
290 + def _evict_cached_block(self, block: KVCacheBlock) -> None:
291 + """
292 + If a block is cached in `cached_block_hash_to_block`, we reset its hash
293 + metadata and evict it from the cache.
294
295 Args:
315 - block: The block to cache.
316 - parent_block: The parent block. None if this is the first block.
296 + block: The block to evict.
297 """
318 - parent_block_hash = (parent_block.block_hash
319 - if parent_block is not None else None)
320 - assert len(block.token_ids) == self.block_size
321 - block.token_ids = tuple(block.token_ids)
322 - block_hash = hash_block_tokens(parent_block_hash, block.token_ids)
323 - block.block_hash = block_hash
324 - block.num_hashed_tokens = self.block_size + (
325 - parent_block.num_hashed_tokens if parent_block is not None else 0)
326 - self.cached_block_hash_to_block[block_hash][block.block_id] = block
298 + block_hash = block.block_hash
299 + if block_hash and block_hash in self.cached_block_hash_to_block:
300 + block.reset_hash()
301 + del self.cached_block_hash_to_block[block_hash][block.block_id]
302 +
303 + if len(self.cached_block_hash_to_block[block_hash]) == 0:
304 + del self.cached_block_hash_to_block[block_hash]
305
306 def _get_cached_block(self,
307 block_hash: BlockHashType) -> Optional[KVCacheBlock]:
@@ -355,43 +333,50 @@ class KVCacheManager:
333 # candidate), so remove it.
334 if block.ref_cnt == 0:
335 self.free_block_queue.remove(block)
358 - block.ref_cnt += 1
359 -
360 - def _add_token_ids_to_blocks(
361 - self,
362 - blocks: List[KVCacheBlock],
363 - token_ids: List[int],
364 - parent_block: Optional[KVCacheBlock] = None) -> int:
365 - """Add token IDs to a list of allocated blocks.
366 - If a block becomes full after adding token IDs, cache it.
367 - Return the token ID index that has not been added to the blocks
368 - if the blocks are not enough to hold all the token IDs.
336 + block.incr_ref()
337
370 - Args:
371 - blocks: A list of blocks to add token IDs.
372 - token_ids: A list of token IDs to add.
373 - parent_block: The parent block. None if this is the
374 - first block.
338 + def _cache_full_blocks(
339 + self,
340 + request: Request,
341 + blk_start_idx: int,
342 + full_blocks: List[KVCacheBlock],
343 + prev_block: Optional[KVCacheBlock],
344 + ) -> None:
345 + """Cache a list of full blocks for prefix caching.
346
376 - Returns:
377 - The starting token ID index that has not been added to the blocks
378 - due to insufficient given blocks.
347 + This function takes a list of blocks that will have their block hash
348 + metadata to be updated and cached. Given a request, it computes the
349 + block hashes for the blocks starting from `blk_start_idx` to the end
350 + of the request's full blocks, updating the metadata for each block
351 + and caching them in the `cached_block_hash_to_block`.
352 +
353 + Args:
354 + request: The request to cache the blocks.
355 + blk_start_idx: The index of the first block in the request's blocks
356 + to cache.
357 + full_blocks: The list of blocks to update hash metadata.
358 + prev_block: The previous block in the chain.
359 """
380 - token_id_start = 0
381 - for curr_block in blocks:
382 - # If all token IDs are added, then the rest of the blocks are
383 - # preallocated blocks, so we only need to update the
384 - # parent_block_id. FIXME
385 - if token_id_start == len(token_ids):
386 - continue
387 -
388 - # Add token IDs to the empty slots in the block.
389 - empty_slots = self.block_size - len(curr_block.token_ids)
390 - token_id_end = min(token_id_start + empty_slots, len(token_ids))
391 - curr_block.token_ids.extend(token_ids[token_id_start:token_id_end])
392 - # Cache the block if it becomes full.
393 - if len(curr_block.token_ids) == self.block_size:
394 - self._cache_full_block(curr_block, parent_block)
395 - parent_block = curr_block
396 - token_id_start = token_id_end
397 - return token_id_start
360 + # Update the new blocks with the block hashes through the chain.
361 + prev_block_hash = (prev_block.block_hash
362 + if prev_block is not None else None)
363 + for i, blk in enumerate(full_blocks):
364 + blk_idx = blk_start_idx + i
365 +
366 + block_tokens = request.all_token_ids[blk_idx *
367 + self.block_size:(blk_idx +
368 + 1) *
369 + self.block_size]
370 + assert len(block_tokens) == self.block_size, (
371 + f"Expected {self.block_size} tokens, got {len(block_tokens)} "
372 + f"at {blk_idx}th block for request "
373 + f"{request.request_id}({request})")
374 +
375 + # Compute the hash of the current block.
376 + block_hash = hash_block_tokens(prev_block_hash,
377 + tuple(block_tokens))
378 +
379 + # Update and added the full block to the cache.
380 + blk.block_hash = block_hash
381 + self.cached_block_hash_to_block[block_hash][blk.block_id] = blk
382 + prev_block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+22 -15
@@ -1,6 +1,6 @@
1 """KV-Cache Utilities."""
2 -from dataclasses import dataclass, field
3 -from typing import List, Optional, Tuple, Union
2 +from dataclasses import dataclass
3 +from typing import List, Optional, Tuple
4
5 from vllm.logger import init_logger
6
@@ -16,27 +16,34 @@ class KVCacheBlock:
16 block_id: int
17 # Reference count.
18 ref_cnt: int = 0
19 - # Token IDs in the block. When the block is full, the type of token_ids
20 - # should be Tuple[int] for fast matching.
21 - token_ids: Union[List[int], Tuple[int]] = field(default_factory=list)
19 # The hash of the block composed of (block hash, tuple of token IDs).
20 # It is only available when the block is full.
24 - block_hash: Optional[BlockHashType] = None
25 - # The number of hashed tokens. More hashed tokens means the block
26 - # is closer to the end of a prompt and more likely to be evicted.
27 - num_hashed_tokens: int = 0
21 + _block_hash: Optional[BlockHashType] = None
22
23 # Used to construct a doubly linked list for free blocks.
24 # These two attributes should only be manipulated by FreeKVCacheBlockQueue.
25 prev_free_block: Optional["KVCacheBlock"] = None
26 next_free_block: Optional["KVCacheBlock"] = None
27
34 - def reset(self):
35 - """Reset the block metadata."""
36 - self.ref_cnt = 0
37 - self.token_ids = []
38 - self.block_hash = None
39 - self.num_hashed_tokens = 0
28 + def incr_ref(self):
29 + self.ref_cnt += 1
30 +
31 + def decr_ref(self):
32 + self.ref_cnt -= 1
33 +
34 + @property
35 + def block_hash(self) -> Optional[BlockHashType]:
36 + return self._block_hash
37 +
38 + @block_hash.setter
39 + def block_hash(self, block_hash: BlockHashType):
40 + assert self.block_hash is None, (
41 + "The block already has a hash. This should not happen.")
42 + self._block_hash = block_hash
43 +
44 + def reset_hash(self):
45 + """Reset the block hash when the block is evicted."""
46 + self._block_hash = None
47
48
49 class FreeKVCacheBlockQueue: