A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core][Bugfix]: fix prefix caching for blockv2 (#4764)
Co-authored-by: Lei Wen <wenlei03@qiyi.com>
leiwen83 committed
May 25, 2024 at 01:07 UTC
e64fde4b013cb8bb2321f59ba78aca50b02071cb
2 files changed
+141
-17
tests/core/block/test_prefix_caching_block.py
+117
@@ -410,6 +410,123 @@ class TestPrefixCachingBlockAllocator:
410
411
assert (len(res) == zero_point_blocks)
412
413
+ # Test case that assume those prompted block after first immutable would
414
+ # be freed into hashless allocator, while first immutable block get ref
415
+ # increased.
416
+ @staticmethod
417
+ @pytest.mark.parametrize("num_blocks", [3])
418
+ @pytest.mark.parametrize("block_size", [16])
419
+ @pytest.mark.parametrize("seed", list(range(10)))
420
+ def test_alloc_promotion(num_blocks: int, block_size: int, seed: int):
421
+ random.seed(seed)
422
+
423
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
424
+ block_size=block_size)
425
+ token_ids = list(range(block_size))
426
+
427
+ block = allocator.allocate_immutable(prev_block=None,
428
+ token_ids=token_ids)
429
+
430
+ assert allocator._refcounter.get(block.block_id) == 1
431
+ m = allocator.allocate_mutable(prev_block=None)
432
+
433
+ block_id = m.block_id
434
+ for i in range(block_size):
435
+ m.append_token_ids([i])
436
+ # After block get promoted to immutable from mutable, if there is
437
+ # already same content hash block, then it shall be released into
438
+ # hashless_allocator
439
+ # And first immutable block's ref get increased by 1
440
+ assert m.block_id == block.block_id
441
+ assert block_id in allocator._hashless_allocator._free_block_indices
442
+ assert allocator._refcounter.get(block.block_id) == 2
443
+
444
+ # Test case when eviction and allocation are mixed,
445
+ # make sure they work as expected
446
+ @staticmethod
447
+ @pytest.mark.parametrize("num_blocks", [3])
448
+ @pytest.mark.parametrize("block_size", [16])
449
+ @pytest.mark.parametrize("seed", list(range(10)))
450
+ def test_eviction_alloc_mixed(num_blocks: int, block_size: int, seed: int):
451
+ random.seed(seed)
452
+
453
+ all_blocks_list = [i for i in range(num_blocks)]
454
+ zero_ref = {i: 0 for i in range(num_blocks)}
455
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
456
+ block_size=block_size)
457
+ token_ids = list(range(num_blocks * block_size))
458
+
459
+ # now we have num_blocks free blocks in hashless allocator
460
+ # with internal tracking list _blocks _cached_blocks and evictor
461
+ # empty and block's ref shall be 0
462
+ assert list(allocator._hashless_allocator._free_block_indices
463
+ ) == all_blocks_list
464
+ assert len(allocator._blocks.keys()) == 0
465
+ assert len(allocator._cached_blocks.values()) == 0
466
+ assert len(allocator.evictor.free_table.keys()) == 0
467
+ assert allocator._refcounter._refcounts == zero_ref
468
+
469
+ # Allocate immutable chains with only one block residuled in
470
+ new_block = []
471
+ for i in range(num_blocks):
472
+ block = allocator.allocate_immutable(
473
+ prev_block=None,
474
+ token_ids=token_ids[block_size * i:block_size * (i + 1)])
475
+ new_block.append(block)
476
+
477
+ # Free all blocks, and now all blocks shall be in the evictor
478
+ # there shall be no tracking data left in _blocks
479
+ # all blocks shall be tracked in _cached_blocks
480
+ # all blocks' ref shall be zero
481
+ for block in new_block:
482
+ allocator.free(block)
483
+
484
+ assert len(allocator._blocks.keys()) == 0
485
+ assert len(allocator._hashless_allocator._free_block_indices) == 0
486
+ assert list(allocator._cached_blocks.values()) == all_blocks_list
487
+ assert list(allocator.evictor.free_table.keys()) == all_blocks_list
488
+ assert allocator._refcounter._refcounts == zero_ref
489
+
490
+ # Allocate a mutable block, and the first block shall be evicted
491
+ # and set its content hash into None, ref to 1
492
+ mutable = allocator.allocate_mutable(prev_block=None)
493
+
494
+ assert mutable.block_id == 0
495
+ assert mutable.content_hash is None
496
+ assert 0 in allocator._blocks
497
+ assert allocator._refcounter.get(0) == 1
498
+ assert 0 not in allocator._cached_blocks
499
+ assert 0 not in allocator.evictor
500
+
501
+ # Since this mutable block has no hash yet, it shall be released into
502
+ # hashless allocator
503
+ allocator.free(mutable)
504
+
505
+ assert len(allocator._blocks.keys()) == 0
506
+ assert allocator._refcounter._refcounts == zero_ref
507
+ assert 0 not in allocator._cached_blocks
508
+ assert 0 not in allocator.evictor
509
+ assert 0 in allocator._hashless_allocator._free_block_indices
510
+
511
+ # when allocate immutable with first block_size tokens, we
512
+ # shall get free block from hashless allocator, thus no block left
513
+ # in hashless
514
+ block = allocator.allocate_immutable(prev_block=None,
515
+ token_ids=token_ids[:block_size])
516
+
517
+ assert block.block_id == 0
518
+ assert len(allocator._hashless_allocator._free_block_indices) == 0
519
+ assert 0 in allocator._blocks
520
+ assert 0 in allocator._cached_blocks.values()
521
+ assert allocator._refcounter.get(0) == 1
522
+ assert 0 not in allocator.evictor
523
+
524
+ # allocate mutable block again, it shall be popped from evictor
525
+ mutable = allocator.allocate_mutable(prev_block=None)
526
+ assert len(allocator._hashless_allocator._free_block_indices) == 0
527
+ assert mutable.block_id not in allocator.evictor.free_table
528
+ assert allocator._refcounter.get(mutable.block_id) == 1
529
+
530
# Test case where two last accessed times are equal
531
@staticmethod
532
@pytest.mark.parametrize("num_blocks", [1024])
vllm/core/block/prefix_caching_block.py
+24
-17
@@ -160,21 +160,17 @@ class PrefixCachingBlockAllocator(BlockAllocator):
160
# If the evictor has blocks available for eviction, evict a block
161
# and return it.
162
if self.evictor.num_blocks > 0:
163
+ # here we get an evicted block, which is only added
164
+ # into evictor if its ref counter is 0
165
+ # and since its content would be changed, we need
166
+ # to remove it from _cached_blocks's tracking list
167
block_id, content_hash_to_evict = self.evictor.evict()
168
165
- # Here we may have scenario that several blocks have
166
- # the same content hash, but due to the latter coming block
167
- # is coming from mutable to immutable path, their physical
168
- # block is added into evictor.
169
- # However in this case, we shall not pop the _cached_blocks,
170
- # as the same content is still used by others, which means
171
- # we need to check ref before decide to pop the list.
172
-
169
_block_id = self._cached_blocks[content_hash_to_evict]
174
- refcount = self._refcounter.get(_block_id)
175
- if refcount == 1:
176
- self._cached_blocks.pop(content_hash_to_evict)
177
- assert _block_id == block_id
170
+ assert self._refcounter.get(_block_id) == 0
171
+ assert _block_id == block_id
172
+
173
+ self._cached_blocks.pop(content_hash_to_evict)
174
175
self._refcounter.incr(block_id)
176
@@ -199,7 +195,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
195
196
def _incr_refcount_cached_block(self, block: Block,
197
block_id: BlockId) -> None:
202
- # since block is already computed, mark it
198
+ # now _incr_refcount_cached_block comes from two place
199
+ # allocate_immutable/promote_to_immutable_block where hit
200
+ # _cached_blocks hash key.
201
+ # In both cases, it means that already exists a already
202
+ # computed block which shared with block now
203
block.computed = True
204
205
refcount = self._refcounter.incr(block_id)
@@ -228,13 +228,19 @@ class PrefixCachingBlockAllocator(BlockAllocator):
228
block: Block) -> None:
229
assert isinstance(block, PrefixCachingBlock)
230
231
- if block.content_hash is None:
231
+ # if we comes from promote_to_immutable_block, it means that
232
+ # block.content_hash is never None.
233
+ # However we need to release the same content block, so that
234
+ # physical block could get reused.
235
+ if block.block_id != block_id or block.content_hash is None:
236
refcount = self._refcounter.get(block_id)
237
# We have fork case where block would get more than one ref,
238
# so we cannot free it from tracking if ref cnt large than 1
235
- if refcount <= 1:
236
- assert block.block_id is not None
239
+ assert block.block_id is not None
240
+ refcount = self._refcounter.get(block.block_id)
241
+ if refcount == 1:
242
del self._blocks[block.block_id]
243
+
244
return self._hashless_allocator.free(block)
245
246
refcount = self._refcounter.decr(block_id)
@@ -317,7 +323,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
323
if block.content_hash not in self._cached_blocks:
324
self._cached_blocks[block.content_hash] = block.block_id
325
else:
320
- self._free_block_id_for_block(block.block_id, block)
326
+ self._free_block_id_for_block(
327
+ self._cached_blocks[block.content_hash], block)
328
self._incr_refcount_cached_block(
329
block, self._cached_blocks[block.content_hash])
330