@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e64fde4b01
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core][Bugfix]: fix prefix caching for blockv2 (#4764)

Co-authored-by: Lei Wen <wenlei03@qiyi.com>

leiwen83 committed May 25, 2024 at 01:07 UTC e64fde4b013cb8bb2321f59ba78aca50b02071cb
2 files changed +141 -17
tests/core/block/test_prefix_caching_block.py
+117
@@ -410,6 +410,123 @@ class TestPrefixCachingBlockAllocator:
410
411 assert (len(res) == zero_point_blocks)
412
413 + # Test case that assume those prompted block after first immutable would
414 + # be freed into hashless allocator, while first immutable block get ref
415 + # increased.
416 + @staticmethod
417 + @pytest.mark.parametrize("num_blocks", [3])
418 + @pytest.mark.parametrize("block_size", [16])
419 + @pytest.mark.parametrize("seed", list(range(10)))
420 + def test_alloc_promotion(num_blocks: int, block_size: int, seed: int):
421 + random.seed(seed)
422 +
423 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
424 + block_size=block_size)
425 + token_ids = list(range(block_size))
426 +
427 + block = allocator.allocate_immutable(prev_block=None,
428 + token_ids=token_ids)
429 +
430 + assert allocator._refcounter.get(block.block_id) == 1
431 + m = allocator.allocate_mutable(prev_block=None)
432 +
433 + block_id = m.block_id
434 + for i in range(block_size):
435 + m.append_token_ids([i])
436 + # After block get promoted to immutable from mutable, if there is
437 + # already same content hash block, then it shall be released into
438 + # hashless_allocator
439 + # And first immutable block's ref get increased by 1
440 + assert m.block_id == block.block_id
441 + assert block_id in allocator._hashless_allocator._free_block_indices
442 + assert allocator._refcounter.get(block.block_id) == 2
443 +
444 + # Test case when eviction and allocation are mixed,
445 + # make sure they work as expected
446 + @staticmethod
447 + @pytest.mark.parametrize("num_blocks", [3])
448 + @pytest.mark.parametrize("block_size", [16])
449 + @pytest.mark.parametrize("seed", list(range(10)))
450 + def test_eviction_alloc_mixed(num_blocks: int, block_size: int, seed: int):
451 + random.seed(seed)
452 +
453 + all_blocks_list = [i for i in range(num_blocks)]
454 + zero_ref = {i: 0 for i in range(num_blocks)}
455 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
456 + block_size=block_size)
457 + token_ids = list(range(num_blocks * block_size))
458 +
459 + # now we have num_blocks free blocks in hashless allocator
460 + # with internal tracking list _blocks _cached_blocks and evictor
461 + # empty and block's ref shall be 0
462 + assert list(allocator._hashless_allocator._free_block_indices
463 + ) == all_blocks_list
464 + assert len(allocator._blocks.keys()) == 0
465 + assert len(allocator._cached_blocks.values()) == 0
466 + assert len(allocator.evictor.free_table.keys()) == 0
467 + assert allocator._refcounter._refcounts == zero_ref
468 +
469 + # Allocate immutable chains with only one block residuled in
470 + new_block = []
471 + for i in range(num_blocks):
472 + block = allocator.allocate_immutable(
473 + prev_block=None,
474 + token_ids=token_ids[block_size * i:block_size * (i + 1)])
475 + new_block.append(block)
476 +
477 + # Free all blocks, and now all blocks shall be in the evictor
478 + # there shall be no tracking data left in _blocks
479 + # all blocks shall be tracked in _cached_blocks
480 + # all blocks' ref shall be zero
481 + for block in new_block:
482 + allocator.free(block)
483 +
484 + assert len(allocator._blocks.keys()) == 0
485 + assert len(allocator._hashless_allocator._free_block_indices) == 0
486 + assert list(allocator._cached_blocks.values()) == all_blocks_list
487 + assert list(allocator.evictor.free_table.keys()) == all_blocks_list
488 + assert allocator._refcounter._refcounts == zero_ref
489 +
490 + # Allocate a mutable block, and the first block shall be evicted
491 + # and set its content hash into None, ref to 1
492 + mutable = allocator.allocate_mutable(prev_block=None)
493 +
494 + assert mutable.block_id == 0
495 + assert mutable.content_hash is None
496 + assert 0 in allocator._blocks
497 + assert allocator._refcounter.get(0) == 1
498 + assert 0 not in allocator._cached_blocks
499 + assert 0 not in allocator.evictor
500 +
501 + # Since this mutable block has no hash yet, it shall be released into
502 + # hashless allocator
503 + allocator.free(mutable)
504 +
505 + assert len(allocator._blocks.keys()) == 0
506 + assert allocator._refcounter._refcounts == zero_ref
507 + assert 0 not in allocator._cached_blocks
508 + assert 0 not in allocator.evictor
509 + assert 0 in allocator._hashless_allocator._free_block_indices
510 +
511 + # when allocate immutable with first block_size tokens, we
512 + # shall get free block from hashless allocator, thus no block left
513 + # in hashless
514 + block = allocator.allocate_immutable(prev_block=None,
515 + token_ids=token_ids[:block_size])
516 +
517 + assert block.block_id == 0
518 + assert len(allocator._hashless_allocator._free_block_indices) == 0
519 + assert 0 in allocator._blocks
520 + assert 0 in allocator._cached_blocks.values()
521 + assert allocator._refcounter.get(0) == 1
522 + assert 0 not in allocator.evictor
523 +
524 + # allocate mutable block again, it shall be popped from evictor
525 + mutable = allocator.allocate_mutable(prev_block=None)
526 + assert len(allocator._hashless_allocator._free_block_indices) == 0
527 + assert mutable.block_id not in allocator.evictor.free_table
528 + assert allocator._refcounter.get(mutable.block_id) == 1
529 +
530 # Test case where two last accessed times are equal
531 @staticmethod
532 @pytest.mark.parametrize("num_blocks", [1024])
vllm/core/block/prefix_caching_block.py
+24 -17
@@ -160,21 +160,17 @@ class PrefixCachingBlockAllocator(BlockAllocator):
160 # If the evictor has blocks available for eviction, evict a block
161 # and return it.
162 if self.evictor.num_blocks > 0:
163 + # here we get an evicted block, which is only added
164 + # into evictor if its ref counter is 0
165 + # and since its content would be changed, we need
166 + # to remove it from _cached_blocks's tracking list
167 block_id, content_hash_to_evict = self.evictor.evict()
168
165 - # Here we may have scenario that several blocks have
166 - # the same content hash, but due to the latter coming block
167 - # is coming from mutable to immutable path, their physical
168 - # block is added into evictor.
169 - # However in this case, we shall not pop the _cached_blocks,
170 - # as the same content is still used by others, which means
171 - # we need to check ref before decide to pop the list.
172 -
169 _block_id = self._cached_blocks[content_hash_to_evict]
174 - refcount = self._refcounter.get(_block_id)
175 - if refcount == 1:
176 - self._cached_blocks.pop(content_hash_to_evict)
177 - assert _block_id == block_id
170 + assert self._refcounter.get(_block_id) == 0
171 + assert _block_id == block_id
172 +
173 + self._cached_blocks.pop(content_hash_to_evict)
174
175 self._refcounter.incr(block_id)
176
@@ -199,7 +195,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
195
196 def _incr_refcount_cached_block(self, block: Block,
197 block_id: BlockId) -> None:
202 - # since block is already computed, mark it
198 + # now _incr_refcount_cached_block comes from two place
199 + # allocate_immutable/promote_to_immutable_block where hit
200 + # _cached_blocks hash key.
201 + # In both cases, it means that already exists a already
202 + # computed block which shared with block now
203 block.computed = True
204
205 refcount = self._refcounter.incr(block_id)
@@ -228,13 +228,19 @@ class PrefixCachingBlockAllocator(BlockAllocator):
228 block: Block) -> None:
229 assert isinstance(block, PrefixCachingBlock)
230
231 - if block.content_hash is None:
231 + # if we comes from promote_to_immutable_block, it means that
232 + # block.content_hash is never None.
233 + # However we need to release the same content block, so that
234 + # physical block could get reused.
235 + if block.block_id != block_id or block.content_hash is None:
236 refcount = self._refcounter.get(block_id)
237 # We have fork case where block would get more than one ref,
238 # so we cannot free it from tracking if ref cnt large than 1
235 - if refcount <= 1:
236 - assert block.block_id is not None
239 + assert block.block_id is not None
240 + refcount = self._refcounter.get(block.block_id)
241 + if refcount == 1:
242 del self._blocks[block.block_id]
243 +
244 return self._hashless_allocator.free(block)
245
246 refcount = self._refcounter.decr(block_id)
@@ -317,7 +323,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
323 if block.content_hash not in self._cached_blocks:
324 self._cached_blocks[block.content_hash] = block.block_id
325 else:
320 - self._free_block_id_for_block(block.block_id, block)
326 + self._free_block_id_for_block(
327 + self._cached_blocks[block.content_hash], block)
328 self._incr_refcount_cached_block(
329 block, self._cached_blocks[block.content_hash])
330