@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2deb029d11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Performance][BlockManagerV2] Mark prefix cache block as computed after schedule (#7822)

Cody Yu committed Aug 26, 2024 at 11:24 UTC 2deb029d115dadd012ce5ea70487a207cb025493
3 files changed +53 -10
tests/core/block/test_prefix_caching_block.py
+31
@@ -708,6 +708,37 @@ class TestPrefixCachingBlockAllocator:
708 token_ids=token_ids)
709 assert allocator.get_prefix_cache_hit_rate() > 0.99
710
711 + # Test case for marking cache hit blocks as computed right after
712 + # a batch of prefill sequences are scheduled.
713 + @staticmethod
714 + def test_touch_block():
715 + block_size = 16
716 + common_blocks = 4
717 + allocator = PrefixCachingBlockAllocator(num_blocks=8,
718 + block_size=block_size)
719 +
720 + common_token_ids = list(range(block_size * common_blocks))
721 +
722 + # Mimic the behavior of allocating the same block chain
723 + # (i.e., common prefix) for a batch of 3 different prefill sequences.
724 + for _ in range(3):
725 + blocks = TestPrefixCachingBlockAllocator.create_immutable_chain(
726 + block_size=block_size,
727 + token_ids=common_token_ids,
728 + allocator=allocator,
729 + )
730 + block_ids = [block.block_id for block in blocks]
731 + # The allocated blocks should be marked as touched
732 + # but not computed.
733 + computed_block_ids = allocator.get_computed_block_ids(
734 + [], block_ids, skip_last_block_id=False)
735 + assert len(computed_block_ids) == 0
736 +
737 + allocator.mark_blocks_as_computed([])
738 + computed_block_ids = allocator.get_computed_block_ids(
739 + [], block_ids, skip_last_block_id=False)
740 + assert len(computed_block_ids) == common_blocks
741 +
742 @staticmethod
743 def create_immutable_chain(
744 block_size: int,
vllm/core/block/prefix_caching_block.py
+17 -5
@@ -1,6 +1,6 @@
1 """Token blocks."""
2 from os.path import commonprefix
3 -from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
3 +from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple
4
5 from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
6 get_all_blocks_recursively)
@@ -73,6 +73,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
73 # prefix hash will be in this dict, even if they have refcount 0.
74 self._cached_blocks: Dict[PrefixHash, BlockId] = {}
75
76 + # A list of immutable block IDs that have been touched by scheduler
77 + # and should be marked as computed after an entire batch of sequences
78 + # are scheduled.
79 + self._touched_blocks: Set[BlockId] = set()
80 +
81 # Used to track status of each physical block id
82 self._block_tracker: Dict[BlockId, BlockTracker] = {}
83 for block_id in block_ids:
@@ -438,10 +443,14 @@ class PrefixCachingBlockAllocator(BlockAllocator):
443 assert self._refcounter.get(block.block_id) > 0
444
445 if block.content_hash not in self._cached_blocks:
441 - # No cached content hash => Set this block as cached
442 - # (Note that this block is not computed yet =>
443 - # Will be computed after free())
446 + # No cached content hash => Set this block as cached.
447 + # Note that this block cannot be marked as computed yet
448 + # because other sequences in the same batch cannot reuse
449 + # this block.
450 self._cached_blocks[block.content_hash] = block.block_id
451 + # Mark this block as touched so that it can be marked as
452 + # computed after the entire batch of sequences are scheduled.
453 + self._touched_blocks.add(block.block_id)
454 return block.block_id
455
456 # Reuse the cached content hash
@@ -507,7 +516,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
516 "Mark block as accessed which is not belonged to GPU")
517
518 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
510 - raise NotImplementedError("Marking as computed is incremental")
519 + # Mark all touched blocks as computed.
520 + for block_id in self._touched_blocks:
521 + self._block_tracker[block_id].computed = True
522 + self._touched_blocks.clear()
523
524 def _track_block_id(self, block_id: Optional[BlockId],
525 computed: bool) -> None:
vllm/core/block_manager_v2.py
+5 -5
@@ -287,11 +287,11 @@ class BlockSpaceManagerV2(BlockSpaceManager):
287 seq.seq_id, now)
288
289 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
290 - # The only need for mark block as computed is for prefix caching,
291 - # while currently we could determine whether one block is computed
292 - # or not by check whether it has content hash.
293 - # So this function is useless for block_v2.
294 - pass
290 + # If prefix caching is enabled, mark immutable blocks as computed
291 + # right after they have been scheduled (for prefill). This assumes
292 + # the scheduler is synchronous so blocks are actually computed when
293 + # scheduling the next batch.
294 + self.block_allocator.mark_blocks_as_computed([])
295
296 def get_common_computed_block_ids(
297 self, seqs: List[Sequence]) -> GenericSequence[int]: