A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[v1] Refactor KVCacheManager for more hash input than token ids (#10507)
Signed-off-by: rickyx <rickyx@anyscale.com> Signed-off-by: Cody Yu <hao.yu.cody@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Ricky Xu committed
Nov 22, 2024 at 15:27 UTC
97814fbf0f847a11d2e0eb339e3e7572ca69379d
3 files changed
+365
-186
tests/v1/core/test_prefix_caching.py
+206
-19
@@ -1,8 +1,11 @@
1
"""Compare the with and without prefix caching."""
2
+import pytest
3
+
4
from vllm.inputs import token_inputs
5
from vllm.sampling_params import SamplingParams
6
+from vllm.utils import cdiv
7
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
5
-from vllm.v1.core.kv_cache_utils import hash_block_tokens
8
+from vllm.v1.core.kv_cache_utils import KVCacheBlock, hash_block_tokens
9
10
11
def make_request(request_id, prompt_token_ids):
@@ -31,7 +34,8 @@ def test_prefill():
34
# Fully cache miss
35
# Incomplete 1 block (7 tokens)
36
unique_token_ids = [3] * 7
34
- req0 = make_request("0", common_token_ids + unique_token_ids)
37
+ all_token_ids = common_token_ids + unique_token_ids
38
+ req0 = make_request("0", all_token_ids)
39
computed_blocks = manager.get_computed_blocks(req0)
40
assert not computed_blocks
41
blocks = manager.allocate_slots(req0, 55, computed_blocks)
@@ -40,24 +44,16 @@ def test_prefill():
44
# Check full block metadata
45
parent_block_hash = None
46
for block_id in (0, 1, 2):
43
- block_hash = hash_block_tokens(parent_block_hash,
44
- manager.block_pool[block_id].token_ids)
47
+ block_tokens = tuple(all_token_ids[block_id * 16:(block_id + 1) * 16])
48
+ block_hash = hash_block_tokens(parent_block_hash, block_tokens)
49
assert manager.block_pool[block_id].block_hash == block_hash
50
assert manager.block_pool[block_id].ref_cnt == 1
47
- assert manager.block_pool[block_id].num_hashed_tokens == 16 * (
48
- block_id + 1)
49
- assert manager.block_pool[block_id].token_ids == tuple([block_id] * 16)
51
parent_block_hash = block_hash
52
53
# Check partial/preallocated block metadata
54
for block_id in (3, 4):
55
assert manager.block_pool[block_id].block_hash is None
56
assert manager.block_pool[block_id].ref_cnt == 1
56
- assert manager.block_pool[block_id].num_hashed_tokens == 0
57
- if block_id == 3:
58
- assert manager.block_pool[block_id].token_ids == [3] * 7
59
- else:
60
- assert not manager.block_pool[block_id].token_ids
57
58
# Cache hit in the common prefix when the original block is still in use.
59
# Incomplete 1 block (5 tokens)
@@ -113,7 +109,7 @@ def test_prefill():
109
req3 = make_request("3", [99] * (16 * 9))
110
computed_blocks = manager.get_computed_blocks(req3)
111
assert not computed_blocks
116
- blocks = manager.allocate_slots(req2, 16 * 9, computed_blocks)
112
+ blocks = manager.allocate_slots(req3, 16 * 9, computed_blocks)
113
# This block ID order also checks the eviction order.
114
assert [b.block_id for b in blocks] == [9, 4, 3, 6, 5, 8, 7, 2, 1, 0]
115
assert manager.free_block_queue.num_free_blocks == 0
@@ -148,7 +144,7 @@ def test_decode():
144
req0.append_output_token_ids(8)
145
new_blocks = manager.append_slots(req0, 4)
146
assert new_blocks is not None and len(new_blocks) == 0
151
- assert len(manager.block_pool[3].token_ids) == 11
147
+ assert manager.req_to_blocks[req0.request_id][-2].block_hash is None
148
149
# Append slots without allocating a new block, but start using the
150
# preallocated block.
@@ -159,8 +155,7 @@ def test_decode():
155
req0.append_output_token_ids(7)
156
new_blocks = manager.append_slots(req0, 15)
157
assert new_blocks is not None and len(new_blocks) == 0
162
- assert len(manager.block_pool[3].token_ids) == 16
163
- assert len(manager.block_pool[4].token_ids) == 10
158
+ assert manager.req_to_blocks[req0.request_id][-2].block_hash is not None
159
160
# Append slots with allocating a new block.
161
req0.num_computed_tokens = 74
@@ -171,9 +166,6 @@ def test_decode():
166
new_blocks = manager.append_slots(req0, 17)
167
# Plus one preallocated block.
168
assert new_blocks is not None and len(new_blocks) == 2
174
- assert len(manager.block_pool[4].token_ids) == 16
175
- assert len(manager.block_pool[5].token_ids) == 11
176
- assert len(manager.block_pool[6].token_ids) == 0
169
170
171
def test_evict():
@@ -217,3 +209,198 @@ def test_evict():
209
blocks = manager.allocate_slots(req2, 3, computed_blocks)
210
assert [b.block_id for b in blocks] == [6, 5]
211
assert manager.free_block_queue.num_free_blocks == 6
212
+
213
+
214
+def test_hash_block_correct_reuse():
215
+ """
216
+ This tests when a previously cached block is reused as a new block,
217
+ its hash metadata should be correctly reset.
218
+ """
219
+ block_size = 16
220
+ manager = KVCacheManager(
221
+ block_size=block_size,
222
+ num_gpu_blocks=1,
223
+ sliding_window=False,
224
+ enable_caching=True,
225
+ num_preallocate_tokens=0,
226
+ )
227
+
228
+ # Allocate 1 block and cache it.
229
+ num_tokens = block_size * 1
230
+ req = make_request("0", list(range(num_tokens)))
231
+ computed_blocks = manager.get_computed_blocks(req)
232
+ assert not computed_blocks
233
+ blocks = manager.allocate_slots(req, num_tokens, computed_blocks)
234
+ assert len(blocks) == 1
235
+
236
+ # Deallocate the block.
237
+ manager.free(req)
238
+
239
+ # Allocate a new block that's not full, make sure hash info on the
240
+ # block is cleared.
241
+ req = make_request("1", list(range(num_tokens - 1)))
242
+ computed_blocks = manager.get_computed_blocks(req)
243
+ assert not computed_blocks
244
+ blocks = manager.allocate_slots(req, num_tokens - 1, computed_blocks)
245
+ assert len(blocks) == 1
246
+
247
+ assert manager.block_pool[blocks[0].block_id].block_hash is None
248
+
249
+
250
+def test_computed_blocks_not_evicted():
251
+ """
252
+ Test that the computed blocks are not evicted when getting new blocks
253
+ for a request if there are any other free blocks.
254
+ """
255
+ block_size = 16
256
+ manager = KVCacheManager(
257
+ block_size=block_size,
258
+ num_gpu_blocks=2,
259
+ sliding_window=False,
260
+ enable_caching=True,
261
+ num_preallocate_tokens=0,
262
+ )
263
+
264
+ # Allocate a block and cache it.
265
+ num_tokens = block_size * 1
266
+ req0 = make_request("0", list(range(num_tokens)))
267
+ computed_blocks = manager.get_computed_blocks(req0)
268
+ assert not computed_blocks
269
+ blocks = manager.allocate_slots(req0, num_tokens, computed_blocks)
270
+ assert len(blocks) == 1
271
+ assert blocks[0].block_id == 0
272
+
273
+ # Allocate another block.
274
+ req1 = make_request("1", list(range(num_tokens, num_tokens * 2)))
275
+ computed_blocks = manager.get_computed_blocks(req1)
276
+ assert not computed_blocks
277
+ blocks = manager.allocate_slots(req1, num_tokens, computed_blocks)
278
+ assert len(blocks) == 1
279
+ assert blocks[0].block_id == 1
280
+
281
+ # Free the blocks.
282
+ manager.free(req0)
283
+ manager.free(req1)
284
+
285
+ # Now if we have a cache hit on the first block, we should evict the second
286
+ # cached block rather than the first one.
287
+ req2 = make_request("2", list(range(num_tokens * 2)))
288
+ computed_blocks = manager.get_computed_blocks(req2)
289
+ assert len(computed_blocks) == 1
290
+ assert computed_blocks[0].block_id == 0
291
+
292
+ blocks = manager.allocate_slots(req2, num_tokens * 2 - num_tokens,
293
+ computed_blocks)
294
+ assert len(blocks) == 1
295
+ assert blocks[0].block_id == 1
296
+
297
+
298
+def test_basic_prefix_caching_disabled():
299
+ """
300
+ This tests that the prefix caching is disabled.
301
+ """
302
+ block_size = 4
303
+ manager = KVCacheManager(
304
+ block_size=block_size,
305
+ num_gpu_blocks=4,
306
+ sliding_window=False,
307
+ enable_caching=False,
308
+ num_preallocate_tokens=0,
309
+ )
310
+
311
+ req1 = make_request("1", list(range(10))) # 2 blocks and some more
312
+
313
+ computed_blocks = manager.get_computed_blocks(req1)
314
+ assert not computed_blocks
315
+ blocks = manager.allocate_slots(req1, 10, computed_blocks)
316
+ assert len(blocks) == 3
317
+
318
+ # Free the blocks.
319
+ manager.free(req1)
320
+
321
+ # No caching.
322
+ req2 = make_request("2", list(range(16))) # shared prefix
323
+ computed_blocks = manager.get_computed_blocks(req2)
324
+ assert not computed_blocks
325
+ blocks = manager.allocate_slots(req2, 16, computed_blocks)
326
+ assert len(blocks) == 4
327
+
328
+ # New requests should not have any blocks.
329
+ req3 = make_request("3", list(range(4)))
330
+ computed_blocks = manager.get_computed_blocks(req3)
331
+ assert not computed_blocks
332
+ blocks = manager.allocate_slots(req3, 4, computed_blocks)
333
+ assert not blocks
334
+
335
+
336
+@pytest.mark.parametrize("num_preallocate_tokens", list(range(0, 8)))
337
+@pytest.mark.parametrize("block_size", [4])
338
+def test_preallocate_blocks(num_preallocate_tokens: int, block_size: int):
339
+ """
340
+ This tests that the preallocated blocks are correctly added.
341
+ """
342
+ manager = KVCacheManager(
343
+ block_size=block_size,
344
+ num_gpu_blocks=10,
345
+ sliding_window=False,
346
+ enable_caching=True,
347
+ num_preallocate_tokens=num_preallocate_tokens,
348
+ )
349
+ num_preallocated_blocks = cdiv(num_preallocate_tokens, block_size)
350
+
351
+ req = make_request("0", list(range(block_size * 30)))
352
+ computed_blocks = manager.get_computed_blocks(req)
353
+ assert not computed_blocks
354
+ # Just ask for 1 block.
355
+ blocks = manager.allocate_slots(req, block_size, computed_blocks)
356
+ assert len(blocks) == 1 + num_preallocated_blocks
357
+
358
+ # Append slots to the block.
359
+ req.num_computed_tokens = block_size * len(blocks) # Assume all used.
360
+ blocks = manager.append_slots(req, block_size) # Append 1 block.
361
+ assert len(blocks) == 1 + num_preallocated_blocks
362
+
363
+
364
+def test_cache_blocks():
365
+ """
366
+ This is a unit test that tests the correctness of the _cache_full_blocks
367
+ function of KVCacheManager.
368
+ """
369
+ block_size = 4
370
+ manager = KVCacheManager(
371
+ block_size=block_size,
372
+ num_gpu_blocks=5,
373
+ sliding_window=False,
374
+ enable_caching=True,
375
+ num_preallocate_tokens=0,
376
+ )
377
+ # Req:
378
+ # Block 0: [0, 1, 2, 3]
379
+ # Block 1: [4, 5, 6, 7]
380
+ # Block 2: [8, 9, 10, 11]
381
+ # Block 3: [12, 13]
382
+ req = make_request("0", list(range(14)))
383
+
384
+ # Test that blocks are cached correctly for 2 full blocks from the start.
385
+ blocks = [KVCacheBlock(block_id=i) for i in range(2)]
386
+
387
+ manager._cache_full_blocks(
388
+ request=req,
389
+ blk_start_idx=0,
390
+ full_blocks=blocks,
391
+ prev_block=None,
392
+ )
393
+
394
+ assert len(manager.cached_block_hash_to_block) == 2
395
+ assert all([block.block_hash is not None for block in blocks])
396
+
397
+ # Test that blocks that don't start from the beginning are cached correctly.
398
+ blocks = [KVCacheBlock(block_id=2)]
399
+ manager._cache_full_blocks(
400
+ request=req,
401
+ blk_start_idx=2,
402
+ full_blocks=blocks,
403
+ prev_block=None,
404
+ )
405
+ assert len(manager.cached_block_hash_to_block) == 3
406
+ assert blocks[0].block_hash is not None
vllm/v1/core/kv_cache_manager.py
+137
-152
@@ -79,6 +79,9 @@ class KVCacheManager:
79
return []
80
81
computed_blocks = []
82
+
83
+ # TODO(rickyx): potentially we could cache this so we don't have to
84
+ # recompute it every time.
85
block_hashes = hash_request_tokens(self.block_size,
86
request.all_token_ids)
87
@@ -120,47 +123,45 @@ class KVCacheManager:
123
# slots, but we cannot allocate new blocks due to the limit.
124
return None
125
123
- # When caching is enabled, assign token IDs to already allocated blocks.
124
- new_token_ids = None
125
- parent_block = None
126
- if self.enable_caching:
127
- # Figure out the token IDs to add to the blocks.
128
- new_token_ids = request.all_token_ids[
129
- request.num_computed_tokens:request.num_computed_tokens +
130
- num_tokens]
131
-
132
- # Find the last full block index.
133
- # TODO: This may be optimized by calculating the computed tokens.
134
- last_full_block_idx = len(req_blocks) - 1
135
- while (last_full_block_idx >= 0
136
- and req_blocks[last_full_block_idx].block_hash is None):
137
- last_full_block_idx -= 1
138
-
139
- parent_block = (req_blocks[last_full_block_idx]
140
- if last_full_block_idx >= 0 else None)
141
- token_id_idx = self._add_token_ids_to_blocks(
142
- blocks=req_blocks[last_full_block_idx + 1:],
143
- token_ids=new_token_ids,
144
- parent_block=parent_block)
145
-
146
- new_token_ids = new_token_ids[token_id_idx:]
147
- parent_block = req_blocks[-1]
148
-
149
- # No new block is needed. When caching is enabled, we make sure
150
- # token_id_idx is equal to len(new_token_ids), meaning that all tokens
151
- # are added to allocated blocks.
152
- if num_required_blocks <= len(req_blocks):
153
- assert not self.enable_caching or token_id_idx == num_tokens, \
154
- f"{token_id_idx=} != {num_tokens=}"
155
- return []
126
+ if num_new_blocks <= 0:
127
+ # No new block is needed.
128
+ new_blocks = []
129
+ else:
130
+ # Get new blocks from the free block pool considering
131
+ # preallocated blocks.
132
+ num_new_blocks = min(
133
+ num_new_blocks + self.num_preallocate_blocks,
134
+ self.free_block_queue.num_free_blocks,
135
+ )
136
+
137
+ new_blocks = self._get_new_blocks(num_new_blocks)
138
+ req_blocks.extend(new_blocks)
139
+
140
+ if not self.enable_caching:
141
+ return new_blocks
142
+
143
+ num_computed_full_blocks = (request.num_computed_tokens //
144
+ self.block_size)
145
+
146
+ # NOTE(rickyx): We are assuming the `num_tokens` are actual
147
+ # tokens rather than lookahead slots (e.g. for speculative decoding).
148
+ # TODO(rickyx): When supporting speculative decoding, we will need to
149
+ # differentiate between them so that we can know how many blocks are
150
+ # full after appending the actual tokens.
151
+ num_full_blocks_after_append = (request.num_computed_tokens +
152
+ num_tokens) // self.block_size
153
+ assert num_full_blocks_after_append <= len(req_blocks)
154
+
155
+ new_full_blocks = req_blocks[
156
+ num_computed_full_blocks:num_full_blocks_after_append]
157
+ self._cache_full_blocks(
158
+ request=request,
159
+ blk_start_idx=num_computed_full_blocks,
160
+ full_blocks=new_full_blocks,
161
+ prev_block=req_blocks[num_computed_full_blocks - 1]
162
+ if num_computed_full_blocks >= 1 else None,
163
+ )
164
157
- # Allocate new blocks considering preallocated blocks, and
158
- # add token IDs to them if caching is enabled.
159
- num_new_blocks = min(num_new_blocks + self.num_preallocate_blocks,
160
- self.free_block_queue.num_free_blocks)
161
- new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
162
- parent_block)
163
- req_blocks.extend(new_blocks)
165
return new_blocks
166
167
def allocate_slots(
@@ -184,11 +185,20 @@ class KVCacheManager:
185
raise ValueError(
186
f"num_tokens must be greater than 0, got {num_tokens}")
187
187
- # If a computed block of a request is an eviction candidate (in the
188
- # free queue and ref_cnt == 0), it cannot be counted as a free block
189
- # when allocating this request.
190
- num_evictable_computed_blocks = len(
191
- [blk for blk in computed_blocks if blk.ref_cnt == 0])
188
+ # Touch the computed blocks to make sure they won't be evicted.
189
+ num_evictable_computed_blocks = 0
190
+ if self.enable_caching:
191
+ self._touch(computed_blocks)
192
+
193
+ # If a computed block of a request is an eviction candidate (in the
194
+ # free queue and ref_cnt == 0), it cannot be counted as a free block
195
+ # when allocating this request.
196
+ num_evictable_computed_blocks = len(
197
+ [blk for blk in computed_blocks if blk.ref_cnt == 0])
198
+ else:
199
+ assert not computed_blocks, (
200
+ "Computed blocks should be empty when "
201
+ "prefix caching is disabled")
202
203
num_required_blocks = cdiv(num_tokens, self.block_size)
204
if (num_required_blocks > self.free_block_queue.num_free_blocks -
@@ -201,35 +211,28 @@ class KVCacheManager:
211
num_new_blocks = min(
212
num_required_blocks + self.num_preallocate_blocks,
213
self.free_block_queue.num_free_blocks -
204
- num_evictable_computed_blocks)
205
-
206
- num_computed_tokens = len(computed_blocks) * self.block_size
214
+ num_evictable_computed_blocks,
215
+ )
216
208
- # When caching is enabled, get the new token IDs and the parent block
209
- # ID to generate cache keys.
210
- new_token_ids = None
211
- parent_block = None
212
- if self.enable_caching:
213
- # Touch the computed blocks to make sure they won't be evicted.
214
- self._touch(computed_blocks)
217
+ # Concatenate the computed block IDs and the new block IDs.
218
+ new_blocks = self._get_new_blocks(num_new_blocks)
219
+ self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
220
216
- # Get the token IDs for the blocks being allocated for hashing.
217
- new_token_ids = request.all_token_ids[
218
- num_computed_tokens:num_computed_tokens + num_tokens]
219
- if not new_token_ids:
220
- raise RuntimeError(
221
- "Failed to infer the token IDs for allocation. "
222
- f"#all_tokens={len(request.all_token_ids)} < "
223
- f"#computed_tokens={num_computed_tokens}")
221
+ if not self.enable_caching:
222
+ return new_blocks
223
225
- # Get the parent block ID to construct the block chain.
226
- parent_block = computed_blocks[-1] if computed_blocks else None
224
+ num_computed_tokens = len(computed_blocks) * self.block_size
225
+ num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
226
228
- new_blocks = self._get_new_blocks(num_new_blocks, new_token_ids,
229
- parent_block)
227
+ self._cache_full_blocks(
228
+ request=request,
229
+ blk_start_idx=len(computed_blocks),
230
+ # The new full blocks are the full blocks that are not computed.
231
+ full_blocks=self.req_to_blocks[request.request_id]
232
+ [len(computed_blocks):num_full_blocks],
233
+ prev_block=computed_blocks[-1] if computed_blocks else None,
234
+ )
235
231
- # Concatenate the computed block IDs and the new block IDs.
232
- self.req_to_blocks[request.request_id] = computed_blocks + new_blocks
236
return new_blocks
237
238
def free(self, request: Request) -> None:
@@ -248,24 +251,17 @@ class KVCacheManager:
251
blocks = reversed(blocks)
252
253
for block in blocks:
251
- block.ref_cnt -= 1
254
+ block.decr_ref()
255
if block.ref_cnt == 0:
256
self.free_block_queue.append(block)
257
255
- def _get_new_blocks(
256
- self,
257
- num_blocks: int,
258
- token_ids: Optional[List[int]] = None,
259
- parent_block: Optional[int] = None) -> List[KVCacheBlock]:
260
- """Get new blocks from the free block pool, and add token IDs to
261
- allocated blocks if caching is enabled.
258
+ def _get_new_blocks(self, num_blocks: int) -> List[KVCacheBlock]:
259
+ """Get new blocks from the free block pool.
260
+
261
Note that we do not check block cache in this function.
262
263
Args:
264
num_blocks: The number of blocks to allocate.
266
- token_ids: The token IDs in the blocks. None if caching is disabled.
267
- parent_block: The parent block. Used to include block chain
268
- in the block hash.
265
266
Returns:
267
A list of new block.
@@ -274,56 +270,38 @@ class KVCacheManager:
270
raise ValueError(
271
f"Cannot get {num_blocks} free blocks from the pool")
272
277
- # First allocate blocks.
273
ret: List[KVCacheBlock] = []
274
idx = 0
275
while idx < num_blocks:
276
+ # First allocate blocks.
277
curr_block = self.free_block_queue.popleft()
278
assert curr_block.ref_cnt == 0
279
284
- # Evict blocks from the cache.
280
+ # If the block is cached, evict it.
281
if self.enable_caching:
286
- block_hash = curr_block.block_hash
287
- if (block_hash is not None
288
- and block_hash in self.cached_block_hash_to_block):
289
- if len(self.cached_block_hash_to_block[block_hash]) == 1:
290
- del self.cached_block_hash_to_block[block_hash]
291
- else:
292
- del self.cached_block_hash_to_block[block_hash][
293
- curr_block.block_id]
294
- curr_block.reset()
295
-
296
- curr_block.ref_cnt = 1
282
+ self._evict_cached_block(curr_block)
283
+
284
+ curr_block.incr_ref()
285
ret.append(curr_block)
286
idx += 1
287
300
- # Then assign token IDs to the allocated blocks.
301
- if self.enable_caching:
302
- assert token_ids is not None
303
- token_id_idx = self._add_token_ids_to_blocks(
304
- blocks=ret, token_ids=token_ids, parent_block=parent_block)
305
- assert token_id_idx == len(token_ids)
306
-
288
return ret
289
309
- def _cache_full_block(self,
310
- block: KVCacheBlock,
311
- parent_block: Optional[KVCacheBlock] = None) -> None:
312
- """Cache a full block for prefix caching.
290
+ def _evict_cached_block(self, block: KVCacheBlock) -> None:
291
+ """
292
+ If a block is cached in `cached_block_hash_to_block`, we reset its hash
293
+ metadata and evict it from the cache.
294
295
Args:
315
- block: The block to cache.
316
- parent_block: The parent block. None if this is the first block.
296
+ block: The block to evict.
297
"""
318
- parent_block_hash = (parent_block.block_hash
319
- if parent_block is not None else None)
320
- assert len(block.token_ids) == self.block_size
321
- block.token_ids = tuple(block.token_ids)
322
- block_hash = hash_block_tokens(parent_block_hash, block.token_ids)
323
- block.block_hash = block_hash
324
- block.num_hashed_tokens = self.block_size + (
325
- parent_block.num_hashed_tokens if parent_block is not None else 0)
326
- self.cached_block_hash_to_block[block_hash][block.block_id] = block
298
+ block_hash = block.block_hash
299
+ if block_hash and block_hash in self.cached_block_hash_to_block:
300
+ block.reset_hash()
301
+ del self.cached_block_hash_to_block[block_hash][block.block_id]
302
+
303
+ if len(self.cached_block_hash_to_block[block_hash]) == 0:
304
+ del self.cached_block_hash_to_block[block_hash]
305
306
def _get_cached_block(self,
307
block_hash: BlockHashType) -> Optional[KVCacheBlock]:
@@ -355,43 +333,50 @@ class KVCacheManager:
333
# candidate), so remove it.
334
if block.ref_cnt == 0:
335
self.free_block_queue.remove(block)
358
- block.ref_cnt += 1
359
-
360
- def _add_token_ids_to_blocks(
361
- self,
362
- blocks: List[KVCacheBlock],
363
- token_ids: List[int],
364
- parent_block: Optional[KVCacheBlock] = None) -> int:
365
- """Add token IDs to a list of allocated blocks.
366
- If a block becomes full after adding token IDs, cache it.
367
- Return the token ID index that has not been added to the blocks
368
- if the blocks are not enough to hold all the token IDs.
336
+ block.incr_ref()
337
370
- Args:
371
- blocks: A list of blocks to add token IDs.
372
- token_ids: A list of token IDs to add.
373
- parent_block: The parent block. None if this is the
374
- first block.
338
+ def _cache_full_blocks(
339
+ self,
340
+ request: Request,
341
+ blk_start_idx: int,
342
+ full_blocks: List[KVCacheBlock],
343
+ prev_block: Optional[KVCacheBlock],
344
+ ) -> None:
345
+ """Cache a list of full blocks for prefix caching.
346
376
- Returns:
377
- The starting token ID index that has not been added to the blocks
378
- due to insufficient given blocks.
347
+ This function takes a list of blocks that will have their block hash
348
+ metadata to be updated and cached. Given a request, it computes the
349
+ block hashes for the blocks starting from `blk_start_idx` to the end
350
+ of the request's full blocks, updating the metadata for each block
351
+ and caching them in the `cached_block_hash_to_block`.
352
+
353
+ Args:
354
+ request: The request to cache the blocks.
355
+ blk_start_idx: The index of the first block in the request's blocks
356
+ to cache.
357
+ full_blocks: The list of blocks to update hash metadata.
358
+ prev_block: The previous block in the chain.
359
"""
380
- token_id_start = 0
381
- for curr_block in blocks:
382
- # If all token IDs are added, then the rest of the blocks are
383
- # preallocated blocks, so we only need to update the
384
- # parent_block_id. FIXME
385
- if token_id_start == len(token_ids):
386
- continue
387
-
388
- # Add token IDs to the empty slots in the block.
389
- empty_slots = self.block_size - len(curr_block.token_ids)
390
- token_id_end = min(token_id_start + empty_slots, len(token_ids))
391
- curr_block.token_ids.extend(token_ids[token_id_start:token_id_end])
392
- # Cache the block if it becomes full.
393
- if len(curr_block.token_ids) == self.block_size:
394
- self._cache_full_block(curr_block, parent_block)
395
- parent_block = curr_block
396
- token_id_start = token_id_end
397
- return token_id_start
360
+ # Update the new blocks with the block hashes through the chain.
361
+ prev_block_hash = (prev_block.block_hash
362
+ if prev_block is not None else None)
363
+ for i, blk in enumerate(full_blocks):
364
+ blk_idx = blk_start_idx + i
365
+
366
+ block_tokens = request.all_token_ids[blk_idx *
367
+ self.block_size:(blk_idx +
368
+ 1) *
369
+ self.block_size]
370
+ assert len(block_tokens) == self.block_size, (
371
+ f"Expected {self.block_size} tokens, got {len(block_tokens)} "
372
+ f"at {blk_idx}th block for request "
373
+ f"{request.request_id}({request})")
374
+
375
+ # Compute the hash of the current block.
376
+ block_hash = hash_block_tokens(prev_block_hash,
377
+ tuple(block_tokens))
378
+
379
+ # Update and added the full block to the cache.
380
+ blk.block_hash = block_hash
381
+ self.cached_block_hash_to_block[block_hash][blk.block_id] = blk
382
+ prev_block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+22
-15
@@ -1,6 +1,6 @@
1
"""KV-Cache Utilities."""
2
-from dataclasses import dataclass, field
3
-from typing import List, Optional, Tuple, Union
2
+from dataclasses import dataclass
3
+from typing import List, Optional, Tuple
4
5
from vllm.logger import init_logger
6
@@ -16,27 +16,34 @@ class KVCacheBlock:
16
block_id: int
17
# Reference count.
18
ref_cnt: int = 0
19
- # Token IDs in the block. When the block is full, the type of token_ids
20
- # should be Tuple[int] for fast matching.
21
- token_ids: Union[List[int], Tuple[int]] = field(default_factory=list)
19
# The hash of the block composed of (block hash, tuple of token IDs).
20
# It is only available when the block is full.
24
- block_hash: Optional[BlockHashType] = None
25
- # The number of hashed tokens. More hashed tokens means the block
26
- # is closer to the end of a prompt and more likely to be evicted.
27
- num_hashed_tokens: int = 0
21
+ _block_hash: Optional[BlockHashType] = None
22
23
# Used to construct a doubly linked list for free blocks.
24
# These two attributes should only be manipulated by FreeKVCacheBlockQueue.
25
prev_free_block: Optional["KVCacheBlock"] = None
26
next_free_block: Optional["KVCacheBlock"] = None
27
34
- def reset(self):
35
- """Reset the block metadata."""
36
- self.ref_cnt = 0
37
- self.token_ids = []
38
- self.block_hash = None
39
- self.num_hashed_tokens = 0
28
+ def incr_ref(self):
29
+ self.ref_cnt += 1
30
+
31
+ def decr_ref(self):
32
+ self.ref_cnt -= 1
33
+
34
+ @property
35
+ def block_hash(self) -> Optional[BlockHashType]:
36
+ return self._block_hash
37
+
38
+ @block_hash.setter
39
+ def block_hash(self, block_hash: BlockHashType):
40
+ assert self.block_hash is None, (
41
+ "The block already has a hash. This should not happen.")
42
+ self._block_hash = block_hash
43
+
44
+ def reset_hash(self):
45
+ """Reset the block hash when the block is evicted."""
46
+ self._block_hash = None
47
48
49
class FreeKVCacheBlockQueue: