A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Performance][BlockManagerV2] Mark prefix cache block as computed after schedule (#7822)
Cody Yu committed
Aug 26, 2024 at 11:24 UTC
2deb029d115dadd012ce5ea70487a207cb025493
3 files changed
+53
-10
tests/core/block/test_prefix_caching_block.py
+31
@@ -708,6 +708,37 @@ class TestPrefixCachingBlockAllocator:
708
token_ids=token_ids)
709
assert allocator.get_prefix_cache_hit_rate() > 0.99
710
711
+ # Test case for marking cache hit blocks as computed right after
712
+ # a batch of prefill sequences are scheduled.
713
+ @staticmethod
714
+ def test_touch_block():
715
+ block_size = 16
716
+ common_blocks = 4
717
+ allocator = PrefixCachingBlockAllocator(num_blocks=8,
718
+ block_size=block_size)
719
+
720
+ common_token_ids = list(range(block_size * common_blocks))
721
+
722
+ # Mimic the behavior of allocating the same block chain
723
+ # (i.e., common prefix) for a batch of 3 different prefill sequences.
724
+ for _ in range(3):
725
+ blocks = TestPrefixCachingBlockAllocator.create_immutable_chain(
726
+ block_size=block_size,
727
+ token_ids=common_token_ids,
728
+ allocator=allocator,
729
+ )
730
+ block_ids = [block.block_id for block in blocks]
731
+ # The allocated blocks should be marked as touched
732
+ # but not computed.
733
+ computed_block_ids = allocator.get_computed_block_ids(
734
+ [], block_ids, skip_last_block_id=False)
735
+ assert len(computed_block_ids) == 0
736
+
737
+ allocator.mark_blocks_as_computed([])
738
+ computed_block_ids = allocator.get_computed_block_ids(
739
+ [], block_ids, skip_last_block_id=False)
740
+ assert len(computed_block_ids) == common_blocks
741
+
742
@staticmethod
743
def create_immutable_chain(
744
block_size: int,
vllm/core/block/prefix_caching_block.py
+17
-5
@@ -1,6 +1,6 @@
1
"""Token blocks."""
2
from os.path import commonprefix
3
-from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
3
+from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple
4
5
from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
6
get_all_blocks_recursively)
@@ -73,6 +73,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
73
# prefix hash will be in this dict, even if they have refcount 0.
74
self._cached_blocks: Dict[PrefixHash, BlockId] = {}
75
76
+ # A list of immutable block IDs that have been touched by scheduler
77
+ # and should be marked as computed after an entire batch of sequences
78
+ # are scheduled.
79
+ self._touched_blocks: Set[BlockId] = set()
80
+
81
# Used to track status of each physical block id
82
self._block_tracker: Dict[BlockId, BlockTracker] = {}
83
for block_id in block_ids:
@@ -438,10 +443,14 @@ class PrefixCachingBlockAllocator(BlockAllocator):
443
assert self._refcounter.get(block.block_id) > 0
444
445
if block.content_hash not in self._cached_blocks:
441
- # No cached content hash => Set this block as cached
442
- # (Note that this block is not computed yet =>
443
- # Will be computed after free())
446
+ # No cached content hash => Set this block as cached.
447
+ # Note that this block cannot be marked as computed yet
448
+ # because other sequences in the same batch cannot reuse
449
+ # this block.
450
self._cached_blocks[block.content_hash] = block.block_id
451
+ # Mark this block as touched so that it can be marked as
452
+ # computed after the entire batch of sequences are scheduled.
453
+ self._touched_blocks.add(block.block_id)
454
return block.block_id
455
456
# Reuse the cached content hash
@@ -507,7 +516,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
516
"Mark block as accessed which is not belonged to GPU")
517
518
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
510
- raise NotImplementedError("Marking as computed is incremental")
519
+ # Mark all touched blocks as computed.
520
+ for block_id in self._touched_blocks:
521
+ self._block_tracker[block_id].computed = True
522
+ self._touched_blocks.clear()
523
524
def _track_block_id(self, block_id: Optional[BlockId],
525
computed: bool) -> None:
vllm/core/block_manager_v2.py
+5
-5
@@ -287,11 +287,11 @@ class BlockSpaceManagerV2(BlockSpaceManager):
287
seq.seq_id, now)
288
289
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
290
- # The only need for mark block as computed is for prefix caching,
291
- # while currently we could determine whether one block is computed
292
- # or not by check whether it has content hash.
293
- # So this function is useless for block_v2.
294
- pass
290
+ # If prefix caching is enabled, mark immutable blocks as computed
291
+ # right after they have been scheduled (for prefill). This assumes
292
+ # the scheduler is synchronous so blocks are actually computed when
293
+ # scheduling the next batch.
294
+ self.block_allocator.mark_blocks_as_computed([])
295
296
def get_common_computed_block_ids(
297
self, seqs: List[Sequence]) -> GenericSequence[int]: