@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4634a89d18
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

Prefix Cache Aware Scheduling [1/n] (#10128)

Signed-off-by: rickyx <rickyx@anyscale.com>

Ricky Xu committed Nov 22, 2024 at 21:15 UTC 4634a89d18569ef0ee2d7dd2d535377a1f460188
13 files changed +967 -241
tests/core/block/test_prefix_caching_block.py
+175 -6
@@ -5,9 +5,14 @@ from unittest.mock import MagicMock
5
6 import pytest
7
8 +from tests.core.utils import create_dummy_sequence
9 +from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
10 from vllm.core.block.interfaces import Block, BlockAllocator
9 -from vllm.core.block.prefix_caching_block import (PrefixCachingBlock,
11 +from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
12 + PrefixCachingBlock,
13 PrefixCachingBlockAllocator)
14 +from vllm.sequence import Logprob
15 +from vllm.utils import Device
16
17
18 class TestPrefixCachingBlock:
@@ -726,18 +731,71 @@ class TestPrefixCachingBlockAllocator:
731 token_ids=common_token_ids,
732 allocator=allocator,
733 )
729 - block_ids = [block.block_id for block in blocks]
734 + block_hashes = [block.content_hash for block in blocks]
735 # The allocated blocks should be marked as touched
736 # but not computed.
732 - computed_block_ids = allocator.get_computed_block_ids(
733 - [], block_ids, skip_last_block_id=False)
737 + computed_block_ids = allocator.find_cached_blocks_prefix(
738 + block_hashes)
739 assert len(computed_block_ids) == 0
740
741 allocator.mark_blocks_as_computed([])
737 - computed_block_ids = allocator.get_computed_block_ids(
738 - [], block_ids, skip_last_block_id=False)
742 + computed_block_ids = allocator.find_cached_blocks_prefix(
743 + block_hashes=block_hashes)
744 assert len(computed_block_ids) == common_blocks
745
746 + @staticmethod
747 + def test_find_cached_blocks_prefix():
748 + """
749 + This test verifies the behavior of find_cached_blocks_prefix.
750 + """
751 + block_size = 4
752 + num_blocks = 8
753 + total_test_blocks = 12
754 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
755 + block_size=block_size)
756 +
757 + token_ids = list(range(total_test_blocks * block_size))
758 + block_tokens_seq1 = token_ids[:num_blocks * block_size]
759 + blocks_seq1 = TestPrefixCachingBlockAllocator.create_immutable_chain(
760 + block_size=block_size,
761 + token_ids=block_tokens_seq1,
762 + allocator=allocator,
763 + )
764 + block_hashes_seq1 = [block.content_hash for block in blocks_seq1]
765 + allocator.mark_blocks_as_computed([])
766 +
767 + # All blocks should be cached.
768 + cached_blocks_seq1 = allocator.find_cached_blocks_prefix(
769 + block_hashes=block_hashes_seq1)
770 + assert len(cached_blocks_seq1) == num_blocks
771 +
772 + # Free the first sequence.
773 + for block in blocks_seq1:
774 + allocator.free(block)
775 +
776 + # All blocks should be still be cached if not required to be allocated.
777 + cached_blocks = allocator.find_cached_blocks_prefix(
778 + block_hashes=block_hashes_seq1)
779 + assert len(cached_blocks) == num_blocks
780 +
781 + block_tokens_seq2 = token_ids[num_blocks * block_size:]
782 + blocks_seq2 = TestPrefixCachingBlockAllocator.create_immutable_chain(
783 + block_size=block_size,
784 + token_ids=block_tokens_seq2,
785 + allocator=allocator,
786 + )
787 + block_hashes_seq2 = [block.content_hash for block in blocks_seq2]
788 + allocator.mark_blocks_as_computed([])
789 + cached_blocks = allocator.find_cached_blocks_prefix(
790 + block_hashes=block_hashes_seq2)
791 + assert len(cached_blocks) == len(blocks_seq2)
792 +
793 + # Half of the blocks from seq1 should still be cached.
794 + num_evicted_blocks = len(blocks_seq2)
795 + cached_blocks = allocator.find_cached_blocks_prefix(
796 + block_hashes=block_hashes_seq1)
797 + assert len(cached_blocks) == len(blocks_seq1) - num_evicted_blocks
798 +
799 @staticmethod
800 def create_immutable_chain(
801 block_size: int,
@@ -762,3 +820,114 @@ class TestPrefixCachingBlockAllocator:
820 blocks.append(prev_block)
821
822 return blocks
823 +
824 +
825 +class TestComputedBlocksTracker:
826 +
827 + @staticmethod
828 + def _get_mock_allocator():
829 + return MagicMock(spec=PrefixCachingBlockAllocator)
830 +
831 + @staticmethod
832 + def test_get_num_cached_tokens():
833 + """
834 + Test it correctly computes the number of cached tokens for a given
835 + sequence:
836 +
837 + - The cache token count is derived from the number of cached blocks.
838 + - The cache token count is updated when the allocator is updated.
839 + - When a sequence is removed, the cache token count should be updated
840 + accordingly.
841 +
842 + # TODO(rickyx): This behaviour for prefill sequence is a hack until
843 + we fix the computed blocks tracking.
844 + - The cache token count for prefill sequence doesn't change while
845 + the sequence is in continuous prefill (chunked prefill).
846 + """
847 + block_size = 4
848 + mock_allocator = TestComputedBlocksTracker._get_mock_allocator()
849 + tracker = ComputedBlocksTracker(
850 + allocator=mock_allocator,
851 + block_size=block_size,
852 + enable_caching=True,
853 + )
854 +
855 + # Not yet allocated.
856 + tokens = [0, 1, 2, 3, 4, 5]
857 + seq1 = create_dummy_sequence(request_id=0,
858 + token_ids=tokens,
859 + block_size=block_size)
860 + mock_allocator.find_cached_blocks_prefix.return_value = []
861 + assert tracker.get_num_cached_tokens(seq1) == 0
862 +
863 + mock_allocator.find_cached_blocks_prefix.return_value = [
864 + None
865 + ] # 1 block cached.
866 + # Result is cached for prefill sequence.
867 + assert tracker.get_num_cached_tokens(seq1) == 0
868 +
869 + # Mark the sequence as non-prefill.
870 + seq1.data.update_num_computed_tokens(len(tokens)) # 6 tokens computed.
871 + assert not seq1.is_prefill()
872 +
873 + # Recomputes for decoding sequence.
874 + assert tracker.get_num_cached_tokens(seq1) == 4
875 +
876 + # Append new tokens to the sequence.
877 + num_new_tokens = 3
878 + for i in range(num_new_tokens):
879 + seq1.append_token_id(i, {i: Logprob(logprob=0.0)})
880 +
881 + assert tracker.get_num_cached_tokens(seq1) == 4
882 +
883 + # Update the allocator.
884 + mock_allocator.find_cached_blocks_prefix.return_value = [
885 + None
886 + ] * 2 # 2 blocks cached.
887 + assert tracker.get_num_cached_tokens(seq1) == 8
888 +
889 + # Remove the sequence.
890 + tracker.remove_seq(seq1.seq_id)
891 +
892 + # Re-create the sequence with the same request id to simulate recompute.
893 + seq1 = create_dummy_sequence(request_id=0,
894 + token_ids=tokens,
895 + block_size=block_size)
896 + mock_allocator.find_cached_blocks_prefix.return_value = [
897 + ] # no cached block
898 + assert tracker.get_num_cached_tokens(seq1) == 0
899 +
900 + @staticmethod
901 + def test_correct_block_hash():
902 + """
903 + Test that the block hash is correctly computed for a sequence (should
904 + match the underlying block allocator's block hash). So the number of
905 + cached tokens is correctly retrieved.
906 + """
907 + block_size = 4
908 + allocator = CpuGpuBlockAllocator.create(
909 + allocator_type="prefix_caching",
910 + num_gpu_blocks=16,
911 + num_cpu_blocks=16,
912 + block_size=block_size,
913 + )
914 + gpu_allocator = allocator._allocators[Device.GPU]
915 +
916 + tracker = ComputedBlocksTracker(
917 + allocator=allocator,
918 + block_size=block_size,
919 + enable_caching=True,
920 + )
921 +
922 + tokens = list(range(block_size * 4)) # 4 blocks.
923 + seq = create_dummy_sequence(request_id=0,
924 + token_ids=tokens,
925 + block_size=block_size)
926 + _ = TestPrefixCachingBlockAllocator.create_immutable_chain(
927 + block_size=block_size,
928 + token_ids=tokens,
929 + allocator=gpu_allocator,
930 + )
931 + allocator.mark_blocks_as_computed([])
932 +
933 + assert tracker.get_num_cached_tokens(seq) == len(tokens)
tests/core/test_scheduler.py
+175 -4
@@ -12,9 +12,9 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12 from vllm.lora.request import LoRARequest
13 from vllm.sequence import SequenceGroup
14
15 -from .utils import (append_new_token, append_new_token_seq_group,
16 - create_dummy_prompt, get_sequence_groups,
17 - schedule_and_update_computed_tokens)
15 +from .utils import (append_new_token, append_new_token_seq,
16 + append_new_token_seq_group, create_dummy_prompt,
17 + get_sequence_groups, schedule_and_update_computed_tokens)
18
19
20 def test_scheduler_add_seq_group():
@@ -305,6 +305,8 @@ def initialize_scheduler(
305 block_size=4,
306 num_cpu_blocks=8,
307 num_gpu_blocks=8,
308 + enable_prefix_caching=False,
309 + enable_chunked_prefill=False,
310 ):
311 block_size = block_size
312 scheduler_config = SchedulerConfig(
@@ -312,8 +314,15 @@ def initialize_scheduler(
314 max_num_batched_tokens=max_token_budget,
315 max_num_seqs=max_num_seqs,
316 max_model_len=max_model_len,
317 + enable_chunked_prefill=enable_chunked_prefill,
318 + )
319 + cache_config = CacheConfig(
320 + block_size,
321 + 1.0,
322 + 1,
323 + "auto",
324 + enable_prefix_caching=enable_prefix_caching,
325 )
316 - cache_config = CacheConfig(block_size, 1.0, 1, "auto")
326 cache_config.num_cpu_blocks = num_cpu_blocks
327 cache_config.num_gpu_blocks = num_gpu_blocks
328 scheduler = Scheduler(scheduler_config, cache_config, lora_config)
@@ -800,3 +809,165 @@ def test_scheduling_budget():
809 assert budget.num_curr_seqs == 0
810 budget.subtract_num_seqs(seq_group.request_id, 2)
811 assert budget.num_curr_seqs == 0
812 +
813 +
814 +@pytest.mark.parametrize("enable_prefix_caching", [True, False])
815 +def test_prefix_caching_aware_prefills(enable_prefix_caching):
816 + """
817 + Test the below scenario:
818 +
819 + For 3 sequences, seqA, seqB, seqC, share the first block as prefix.
820 +
821 + The test verifies the below scenarios:
822 + 1. SeqA is first scheduled.
823 + 2. SeqB and SeqC can be prefilled together in a single schedule round
824 + even though there are not enough token budgets to prefill both without
825 + considering prefix caching.
826 + """
827 +
828 + block_size = 4
829 + max_num_batched_tokens = 12
830 + max_seq_group = 3
831 + scheduler = initialize_scheduler(
832 + block_size=block_size,
833 + num_cpu_blocks=16,
834 + num_gpu_blocks=16,
835 + max_token_budget=max_num_batched_tokens,
836 + max_num_seqs=max_seq_group,
837 + max_model_len=max_num_batched_tokens,
838 + enable_prefix_caching=enable_prefix_caching,
839 + )
840 +
841 + seqA_tokens = list(range(8))
842 + num_shared_tokens = 4
843 + seqB_tokens = seqA_tokens[:num_shared_tokens] + list(range(
844 + 12, 16)) # Shared prefix first 4.
845 + seqC_tokens = seqA_tokens[:num_shared_tokens] + list(range(
846 + 16, 20)) # Shared prefix first 4.
847 +
848 + seqA, seqA_group = create_dummy_prompt("0",
849 + prompt_tokens=seqA_tokens,
850 + block_size=block_size)
851 + seqB, seqB_group = create_dummy_prompt("1",
852 + prompt_tokens=seqB_tokens,
853 + block_size=block_size)
854 + seqC, seqC_group = create_dummy_prompt("2",
855 + prompt_tokens=seqC_tokens,
856 + block_size=block_size)
857 +
858 + # Schedule seqA prefill.
859 + scheduler.add_seq_group(seqA_group)
860 + metas, out, _ = scheduler.schedule()
861 + assert (len(out.scheduled_seq_groups) == 1
862 + and out.scheduled_seq_groups[0].seq_group == seqA_group)
863 + assert out.scheduled_seq_groups[0].token_chunk_size == len(seqA_tokens)
864 +
865 + # Schedule seqA decode.
866 + append_new_token_seq_group(len(seqA_tokens), seqA_group, 999)
867 + metas, out, _ = scheduler.schedule()
868 +
869 + assert len(out.scheduled_seq_groups) == 1
870 + assert out.scheduled_seq_groups[0].seq_group == seqA_group
871 + assert out.scheduled_seq_groups[0].token_chunk_size == 1
872 +
873 + # Schedule seqB and seqC prefills should work with prefix caching.
874 + scheduler.add_seq_group(seqB_group)
875 + scheduler.add_seq_group(seqC_group)
876 + metas, out, _ = scheduler.schedule()
877 +
878 + if enable_prefix_caching:
879 + assert len(out.scheduled_seq_groups) == 2
880 + assert set([
881 + out.scheduled_seq_groups[0].seq_group,
882 + out.scheduled_seq_groups[1].seq_group,
883 + ]) == set([seqB_group, seqC_group])
884 + assert len(metas) == 2
885 + for meta in metas:
886 + assert meta.token_chunk_size == 8
887 + assert (len(meta.computed_block_nums) == num_shared_tokens //
888 + block_size) # 1 Block for the 8 tokens.
889 + else:
890 + assert len(out.scheduled_seq_groups) == 1
891 + assert len(metas) == 1
892 + assert metas[0].token_chunk_size == 8
893 + assert len(metas[0].computed_block_nums) == 0 # No blocks computed.
894 +
895 +
896 +def test_no_multiple_partial_prefills_with_chunked_prefill_and_prefix_caching(
897 +):
898 + """
899 + This test verifies that we don't schedule new prefills if there's already
900 + a continuous prefill in progress even though the new prefills with shared
901 + prefix can fit in the token budget:
902 +
903 + - SeqA is being chunked prefill.
904 + - SeqB with the same prompt shouldn't be scheduled for prefill even though
905 + there's enough token budget to prefill the cached tokens.
906 + - Neither should seqC be scheduled.
907 +
908 + - When seqA is in decoding phase, seqB and seqC can be scheduled.
909 + - Entire seqB should be prefilled since it's a full prefix cache hit.
910 + - SeqC would be partially prefilled with the prefix shared, and the
911 + remaining unique tokens would be prefilled (rounded down to be
912 + block-size aligned).
913 + """
914 +
915 + block_size = 2
916 + max_num_batched_tokens = 4
917 + max_seq_group = 3
918 + scheduler = initialize_scheduler(
919 + block_size=block_size,
920 + num_cpu_blocks=16,
921 + num_gpu_blocks=16,
922 + max_token_budget=max_num_batched_tokens,
923 + max_num_seqs=max_seq_group,
924 + max_model_len=100,
925 + enable_prefix_caching=True,
926 + enable_chunked_prefill=True,
927 + )
928 +
929 + seqA_tokens = list(range(8))
930 + seqB_tokens = seqA_tokens
931 + seqC_shared_prefix_len = 4
932 + seqC_tokens = seqA_tokens[:seqC_shared_prefix_len] + list(range(12, 20))
933 +
934 + seqA, seqA_group = create_dummy_prompt("0",
935 + prompt_tokens=seqA_tokens,
936 + block_size=block_size)
937 + seqB, seqB_group = create_dummy_prompt("1",
938 + prompt_tokens=seqB_tokens,
939 + block_size=block_size)
940 +
941 + # Chunked prefill seqA.
942 + scheduler.add_seq_group(seqA_group)
943 + metas, out = schedule_and_update_computed_tokens(scheduler)
944 + assert len(out.scheduled_seq_groups) == 1
945 + assert out.scheduled_seq_groups[0].seq_group == seqA_group
946 + assert out.scheduled_seq_groups[0].token_chunk_size == 4
947 +
948 + # seqB should not be scheduled with ongoing prefills.
949 + scheduler.add_seq_group(seqB_group)
950 + metas, out = schedule_and_update_computed_tokens(scheduler)
951 + assert len(out.scheduled_seq_groups) == 1
952 + assert out.scheduled_seq_groups[0].seq_group == seqA_group
953 + assert out.scheduled_seq_groups[0].token_chunk_size == 4
954 +
955 + # both seqB and seqC can now be scheduled with seqA is over.
956 + # seqA is in decoding phase.
957 + append_new_token_seq(seqA, 999)
958 + seqC, seqC_group = create_dummy_prompt("2",
959 + prompt_tokens=seqC_tokens,
960 + block_size=block_size)
961 + scheduler.add_seq_group(seqC_group)
962 + metas, out = schedule_and_update_computed_tokens(scheduler)
963 + assert len(out.scheduled_seq_groups) == 3
964 +
965 + metas = {meta.request_id: meta for meta in metas}
966 + assert metas[seqA_group.request_id].token_chunk_size == 1 # Decode
967 + assert (metas[seqB_group.request_id].token_chunk_size == 8
968 + ) # Fully cached prefill
969 + assert (
970 + metas[seqC_group.request_id].token_chunk_size == 6
971 + ), "A partial prefix of C (4 tokens) should be prefilled, with the "
972 + "remaining tokens fit into 3 token budget (4-1 from the seqA). It will "
973 + "then be rounded down to 2 tokens on block size, thus 6 tokens in total."
tests/core/utils.py
+46 -5
@@ -1,17 +1,20 @@
1 import time
2 -from typing import List, Optional
2 +from collections import defaultdict
3 +from typing import Any, Dict, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Tuple
6
7 from vllm import SamplingParams
8 +from vllm.core.scheduler import Scheduler, SchedulerOutputs
9 from vllm.inputs import EncoderDecoderInputs, token_inputs
10 from vllm.lora.request import LoRARequest
9 -from vllm.sequence import Logprob, Sequence, SequenceGroup
11 +from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12 + SequenceGroupMetadata)
13
14
15 def create_dummy_prompt(
16 request_id: str,
14 - prompt_length: int,
17 + prompt_length: int = -1,
18 block_size: Optional[int] = None,
19 lora_request: Optional[LoRARequest] = None,
20 best_of: int = 1,
@@ -26,6 +29,7 @@ def create_dummy_prompt(
29 # Create dummy prompt sequence with tokens 0...block_size-1
30 # and prompt "0 ... block_size".
31 prompt_tokens = list(range(prompt_length))
32 +
33 prompt_str = " ".join([str(t) for t in prompt_tokens])
34 prompt = Sequence(int(request_id),
35 inputs=token_inputs(prompt_tokens, prompt=prompt_str),
@@ -42,6 +46,15 @@ def create_dummy_prompt(
46 return prompt, seq_group
47
48
49 +def create_dummy_sequence(request_id: int, token_ids: List[int],
50 + block_size: int) -> Sequence:
51 + return Sequence(
52 + seq_id=request_id,
53 + inputs=token_inputs(token_ids),
54 + block_size=block_size,
55 + )
56 +
57 +
58 def create_dummy_prompt_encoder_decoder(
59 request_id: str,
60 decoder_prompt_length: int,
@@ -194,12 +207,40 @@ def append_new_token(out, token_id: int):
207
208 def schedule_and_update_computed_tokens(scheduler):
209 metas, out, _ = scheduler.schedule()
197 - for s, meta in zip(out.scheduled_seq_groups, metas):
198 - s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
210 + for s in out.scheduled_seq_groups:
211 + s.seq_group.update_num_computed_tokens(s.token_chunk_size)
212 return metas, out
213
214
215 +def append_new_token_seq(seq: Sequence, token_id: int):
216 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
217 +
218 +
219 def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
220 seq_group.update_num_computed_tokens(token_chunk_size)
221 for seq in seq_group.get_seqs():
222 seq.append_token_id(token_id, {token_id: Logprob(token_id)})
223 +
224 +
225 +class SchedulerProxy:
226 + """
227 + A proxy class to forward calls to the scheduler.
228 + """
229 +
230 + def __init__(self, scheduler: Scheduler):
231 + self.scheduler_ = scheduler
232 + self.call_history: Dict[str, List[Any]] = defaultdict(list)
233 +
234 + def __getattr__(self, name: str) -> Any:
235 +
236 + def wrapper(*args, **kwargs):
237 + result = getattr(self.scheduler_, name)(*args, **kwargs)
238 + self.call_history[name].append((args, kwargs, result))
239 + return result
240 +
241 + return wrapper
242 +
243 + def last_schedule_ret(
244 + self, ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, Any]:
245 + _, _, ret = self.call_history["schedule"][-1]
246 + return ret
tests/prefix_caching/test_prefix_caching.py
+100 -6
@@ -2,10 +2,15 @@
2
3 Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 +
6 import pytest
7
8 +from tests.conftest import VllmRunner
9 +from tests.core.utils import SchedulerProxy, create_dummy_prompt
10 from tests.kernels.utils import override_backend_env_variable
11 from vllm import SamplingParams, TokensPrompt
12 +from vllm.core.scheduler import Scheduler
13 +from vllm.engine.llm_engine import LLMEngine
14
15 from ..models.utils import check_outputs_equal
16
@@ -27,6 +32,7 @@ UNSTABLE_PROMPT_SEQUENCE = [
32 @pytest.mark.parametrize("dtype", ["half"])
33 @pytest.mark.parametrize("max_tokens", [5])
34 @pytest.mark.parametrize("cached_position", [0, 1])
35 +@pytest.mark.parametrize("enable_chunked_prefill", [True, False])
36 @pytest.mark.parametrize("block_size", [16])
37 def test_mixed_requests(
38 hf_runner,
@@ -37,6 +43,7 @@ def test_mixed_requests(
43 dtype: str,
44 max_tokens: int,
45 cached_position: int,
46 + enable_chunked_prefill: bool,
47 block_size: int,
48 monkeypatch,
49 ) -> None:
@@ -55,6 +62,7 @@ def test_mixed_requests(
62 model,
63 dtype=dtype,
64 enable_prefix_caching=True,
65 + enable_chunked_prefill=enable_chunked_prefill,
66 block_size=block_size,
67 ) as vllm_model:
68 # Run the first prompt so the cache is populated
@@ -72,13 +80,13 @@ def test_mixed_requests(
80 block_size) * block_size
81 else:
82 expected_num_cached_tokens = 0
75 - assert req_outputs[
76 - i].num_cached_tokens == expected_num_cached_tokens
83 + assert (
84 + req_outputs[i].num_cached_tokens == expected_num_cached_tokens)
85
78 - vllm_outputs = [
79 - (output.prompt_token_ids + list(output.outputs[0].token_ids),
80 - output.prompt + output.outputs[0].text) for output in req_outputs
81 - ]
86 + vllm_outputs = [(
87 + output.prompt_token_ids + list(output.outputs[0].token_ids),
88 + output.prompt + output.outputs[0].text,
89 + ) for output in req_outputs]
90
91 check_outputs_equal(
92 outputs_0_lst=hf_outputs,
@@ -105,3 +113,89 @@ def test_unstable_prompt_sequence(
113 for prompt in UNSTABLE_PROMPT_SEQUENCE:
114 vllm_model.generate(TokensPrompt(prompt_token_ids=prompt),
115 SamplingParams(max_tokens=1))
116 +
117 +
118 +@pytest.mark.parametrize("model", MODELS)
119 +def test_fully_cached_prefill_needs_uncached_token(model):
120 + block_size = 16
121 + max_num_batched_tokens = 16
122 + num_output_tokens = 5
123 + # Make a vllm engine
124 + runner = VllmRunner(
125 + model_name=model,
126 + gpu_memory_utilization=0.7,
127 + enable_chunked_prefill=True,
128 + enforce_eager=True,
129 + enable_prefix_caching=True,
130 + block_size=block_size,
131 + max_num_batched_tokens=max_num_batched_tokens,
132 + max_num_seqs=max_num_batched_tokens,
133 + )
134 + engine: LLMEngine = runner.model.llm_engine
135 +
136 + scheduler: Scheduler = SchedulerProxy(engine.scheduler[0]) # type: ignore
137 + engine.scheduler[0] = scheduler
138 +
139 + # SeqA
140 + seqA_tokens = list(range(2 * block_size))
141 + seqA, seq_groupA = create_dummy_prompt(
142 + request_id="0",
143 + prompt_tokens=seqA_tokens,
144 + max_tokens=num_output_tokens,
145 + block_size=block_size,
146 + )
147 +
148 + scheduler.add_seq_group(seq_groupA)
149 +
150 + assert seqA.data.get_num_computed_tokens() == 0
151 +
152 + # Prefill seqA
153 + while not seqA.is_finished():
154 + engine.step()
155 +
156 + # seqB
157 + seqB_tokens = [t + 1 for t in seqA_tokens] # shift by 1
158 + seqB, seq_groupB = create_dummy_prompt(
159 + request_id="1",
160 + prompt_tokens=seqB_tokens,
161 + max_tokens=num_output_tokens,
162 + block_size=block_size,
163 + )
164 +
165 + # seqC is the same as seqA
166 + seqC, seq_groupC = create_dummy_prompt(
167 + request_id="2",
168 + prompt_tokens=seqA_tokens,
169 + max_tokens=num_output_tokens,
170 + block_size=block_size,
171 + )
172 +
173 + scheduler.add_seq_group(seq_groupB)
174 + scheduler.add_seq_group(seq_groupC)
175 +
176 + # Even seqC is fully cached, it should not be prefilled since we
177 + # require at least 1 uncached token.
178 + engine.step()
179 +
180 + sched_metas, sched_out, _ = scheduler.last_schedule_ret()
181 + assert len(sched_out.scheduled_seq_groups) == 1
182 + assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
183 + seq_groupB.request_id)
184 + assert (sched_out.scheduled_seq_groups[0].token_chunk_size ==
185 + max_num_batched_tokens)
186 +
187 + # When seqB is finished, seqC could be prefilled.
188 + while not seqB.is_finished():
189 + engine.step()
190 + sched_metas, sched_out, _ = scheduler.last_schedule_ret()
191 + assert len(sched_out.scheduled_seq_groups) == 1
192 + assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
193 + seq_groupB.request_id)
194 +
195 + engine.step()
196 + sched_metas, sched_out, _ = scheduler.last_schedule_ret()
197 + assert len(sched_out.scheduled_seq_groups) == 1
198 + assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
199 + seq_groupC.request_id)
200 + assert sched_out.scheduled_seq_groups[0].token_chunk_size == len(
201 + seqA_tokens)
vllm/core/block/cpu_gpu_block_allocator.py
+7 -8
@@ -306,14 +306,6 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
306 device = Device.GPU
307 return self._allocators[device].mark_blocks_as_computed(block_ids)
308
309 - def get_computed_block_ids(self, prev_computed_block_ids: List[int],
310 - block_ids: List[int],
311 - skip_last_block_id: bool) -> List[int]:
312 - # Prefix caching only supported on GPU.
313 - device = Device.GPU
314 - return self._allocators[device].get_computed_block_ids(
315 - prev_computed_block_ids, block_ids, skip_last_block_id)
316 -
309 def get_common_computed_block_ids(
310 self, computed_seq_block_ids: List[List[int]]) -> List[int]:
311 # Prefix caching only supported on GPU.
@@ -342,6 +334,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
334 self._swap_mapping.clear()
335 return list(mapping.items())
336
337 + def find_cached_blocks_prefix(
338 + self,
339 + block_hashes: List[int],
340 + device: Device = Device.GPU,
341 + ) -> List[int]:
342 + return self._allocators[device].find_cached_blocks_prefix(block_hashes)
343 +
344
345 class NullBlock(Block):
346 """
vllm/core/block/interfaces.py
+21 -15
@@ -159,12 +159,6 @@ class BlockAllocator(ABC):
159 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
160 pass
161
162 - @abstractmethod
163 - def get_computed_block_ids(self, prev_computed_block_ids: List[int],
164 - block_ids: List[int],
165 - skip_last_block_id: bool) -> List[int]:
166 - pass
167 -
162 @abstractmethod
163 def get_common_computed_block_ids(
164 self, computed_seq_block_ids: List[List[int]]) -> List[int]:
@@ -192,6 +186,13 @@ class BlockAllocator(ABC):
186 class NoFreeBlocksError(ValueError):
187 pass
188
189 + @abstractmethod
190 + def find_cached_blocks_prefix(
191 + self,
192 + block_hashes: List[int],
193 + ) -> List[int]:
194 + pass
195 +
196
197 class DeviceAwareBlockAllocator(ABC):
198
@@ -207,9 +208,12 @@ class DeviceAwareBlockAllocator(ABC):
208 pass
209
210 @abstractmethod
210 - def allocate_immutable_blocks(self, prev_block: Optional[Block],
211 - block_token_ids: List[List[int]],
212 - device: Device) -> List[Block]:
211 + def allocate_immutable_blocks(
212 + self,
213 + prev_block: Optional[Block],
214 + block_token_ids: List[List[int]],
215 + device: Device,
216 + ) -> List[Block]:
217 pass
218
219 @abstractmethod
@@ -246,12 +250,6 @@ class DeviceAwareBlockAllocator(ABC):
250 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
251 pass
252
249 - @abstractmethod
250 - def get_computed_block_ids(self, prev_computed_block_ids: List[int],
251 - block_ids: List[int],
252 - skip_last_block_id: bool) -> List[int]:
253 - pass
254 -
253 @abstractmethod
254 def get_common_computed_block_ids(
255 self, computed_seq_block_ids: List[List[int]]) -> List[int]:
@@ -284,3 +282,11 @@ class DeviceAwareBlockAllocator(ABC):
282 def get_prefix_cache_hit_rate(self, device: Device) -> float:
283 """Prefix cache hit rate. -1 means not supported or disabled."""
284 pass
285 +
286 + @abstractmethod
287 + def find_cached_blocks_prefix(
288 + self,
289 + block_hashes: List[int],
290 + device: Device = Device.GPU,
291 + ) -> List[int]:
292 + pass
vllm/core/block/naive_block.py
+4 -7
@@ -262,13 +262,6 @@ class NaiveBlockAllocator(BlockAllocator):
262 """
263 pass
264
265 - def get_computed_block_ids(self, prev_computed_block_ids: List[int],
266 - block_ids: List[int],
267 - skip_last_block_id: bool) -> List[int]:
268 - """No prefix caching here => return empty list
269 - """
270 - return []
271 -
265 def get_common_computed_block_ids(
266 self, computed_seq_block_ids: List[List[int]]) -> List[int]:
267 """Determine blocks that can be skipped in prefill.
@@ -329,6 +322,10 @@ class NaiveBlockAllocator(BlockAllocator):
322 def get_prefix_cache_hit_rate(self) -> float:
323 return -1
324
325 + def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
326 + # Not applicable for naive block allocator.
327 + return []
328 +
329
330 class NaiveBlock(Block):
331 """An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+162 -96
@@ -1,13 +1,18 @@
1 """Token blocks."""
2 +import sys
3 +from bisect import bisect_left
4 from os.path import commonprefix
3 -from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple
5 +from typing import (Callable, Dict, FrozenSet, Iterable, List, Optional, Set,
6 + Tuple)
7
8 from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
9 get_all_blocks_recursively)
7 -from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10 +from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId, Device,
11 + DeviceAwareBlockAllocator)
12 from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
13 NaiveBlockAllocator)
14 from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
15 +from vllm.sequence import Sequence
16
17 PrefixHash = int
18
@@ -534,26 +539,6 @@ class PrefixCachingBlockAllocator(BlockAllocator):
539 else:
540 return block_id in self.evictor
541
537 - def get_computed_block_ids(self,
538 - prev_computed_block_ids: List[int],
539 - block_ids: List[int],
540 - skip_last_block_id: bool = True) -> List[int]:
541 - prev_prefix_size = len(prev_computed_block_ids)
542 - cur_size = len(block_ids)
543 - if skip_last_block_id:
544 - cur_size -= 1
545 -
546 - # Sanity checks
547 - assert cur_size >= 0
548 - assert prev_prefix_size <= cur_size
549 -
550 - ret = prev_computed_block_ids
551 - for i in range(prev_prefix_size, cur_size):
552 - block_id = block_ids[i]
553 - if self.block_is_computed(block_id):
554 - ret.append(block_id)
555 - return ret
556 -
542 def get_common_computed_block_ids(
543 self, computed_seq_block_ids: List[List[int]]) -> List[int]:
544 """Return the block ids that are common for a given sequence group.
@@ -634,6 +619,47 @@ class PrefixCachingBlockAllocator(BlockAllocator):
619
620 block.block_id = block_id # Assign block_id
621
622 + def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
623 + """
624 + Given a list of block hashes, return the prefix of the block hashes that
625 + are all cached.
626 +
627 + Since a block's block hash includes the hashes of all previous blocks,
628 + and we only allocate/deallocate blocks in the entire sequence, so if a
629 + block is cached, then all previous blocks are also cached. With this
630 + property, we can use binary search to find the prefix of cached blocks.
631 +
632 + Args:
633 + block_hashes (List[int]): The list of block hashes.
634 +
635 + Returns:
636 + List[int]: The prefix of the `block_hashes` that are cached.
637 + """
638 +
639 + def _block_is_cached(block_hash: PrefixHash) -> bool:
640 + if block_hash not in self._cached_blocks:
641 + return False
642 +
643 + cached_block_id = self._cached_blocks[block_hash]
644 + # We only consider the blocks that are marked as computed.
645 + return self.block_is_computed(cached_block_id)
646 +
647 + def _bisect_left(a, x, key: Callable[[PrefixHash], bool]) -> int:
648 +
649 + # python <= 3.10 don't have the key argument
650 + if sys.version_info < (3, 10):
651 + a = [key(e) for e in a]
652 + return bisect_left(a, x)
653 + else:
654 + return bisect_left(a, x, key=key)
655 +
656 + # Look for the first block that's not cached, and returns the prefix
657 + # i.e. blocks that are cached.
658 + idx = _bisect_left(block_hashes,
659 + True,
660 + key=lambda x: not _block_is_cached(x))
661 + return block_hashes[:idx]
662 +
663
664 class PrefixCachingBlock(Block):
665 """A block implementation that supports prefix caching.
@@ -843,86 +869,126 @@ class PrefixCachingBlock(Block):
869
870
871 class ComputedBlocksTracker:
846 - """Handles caching of per-sequence computed block ids.
847 - When a sequence appears for the first time, it traverses all of the
848 - blocks and detects the prefix of blocks that is computed. On the
849 - subsequent times, it only traverses the new blocks that were added
850 - and updates the already recorded prefix of blocks with the newly
851 - computed blocks.
852 -
853 - To avoid redundant traversals, the algorithm also detects when there
854 - is a "gap" in the computed prefix. For example, if we have blocks =
855 - [1,2,3,4,5], and we have detected [1,2,3] as the computed prefix, then
856 - we won't try to add more computed blocks to [1,2,3] in this sequence
857 - iteration, and will add more computed blocks only after the sequence is
858 - freed and reused again.
859 -
860 - Note that currently, for a given sequence, we also skip the last
861 - block id for caching purposes, to avoid caching of a full sequence
872 """
873 + Tracks the computed blocks for each sequence.
874
864 - def __init__(self, allocator):
865 - self._allocator = allocator
866 - self._cached_computed_seq_blocks: Dict[int, Tuple[List[int],
867 - bool]] = {}
875 + Internally, it maintains a map from sequence id to the list of block hashes
876 + for the sequence. We cache the hashes of the full blocks for each sequence,
877 + and make sure the hash is calculated in the same way as the allocator.
878 + When a sequence is being decoded, we also update the sequence's hash
879 + accordingly and incrementally.
880
869 - def add_seq(self, seq_id: int) -> None:
870 - """Start tracking seq_id
871 - """
872 - assert seq_id not in self._cached_computed_seq_blocks
873 - self._cached_computed_seq_blocks[seq_id] = ([], False)
874 -
875 - def remove_seq(self, seq_id: int) -> None:
876 - """Stop tracking seq_id
877 - """
878 - assert seq_id in self._cached_computed_seq_blocks
879 - del self._cached_computed_seq_blocks[seq_id]
880 -
881 - def get_cached_computed_blocks_and_update(
882 - self, seq_id: int, block_ids: List[int]) -> List[int]:
883 - """ Look at the class documentation for details
884 - """
885 - # Ensure seq_id is already tracked
886 - assert seq_id in self._cached_computed_seq_blocks
887 -
888 - # Get cached data (may be empty on the first time)
889 - prev_computed_block_ids, has_gap = self._cached_computed_seq_blocks[
890 - seq_id]
891 -
892 - if has_gap:
893 - # When gap is detected, we do not add more computed blocks at this
894 - # sequence iteration
895 - return prev_computed_block_ids
896 -
897 - # We do not consider the last block id for caching purposes.
898 - num_cur_blocks = len(block_ids) - 1
899 - assert num_cur_blocks >= 0
900 -
901 - if len(prev_computed_block_ids) >= num_cur_blocks:
902 - # Cache HIT
903 - assert len(prev_computed_block_ids) == num_cur_blocks
904 - return prev_computed_block_ids
905 -
906 - # If here, then we may possibly add more computed blocks. As a result,
907 - # traverse the additional blocks after prev_computed_block_ids to
908 - # detect more computed blocks and add them.
909 -
910 - # Incremental init for seq_id => Look only at the new blocks
911 - computed_block_ids = self._allocator.get_computed_block_ids( # noqa: E501
912 - prev_computed_block_ids,
913 - block_ids,
914 - skip_last_block_id=
915 - True, # We skip last block id to avoid caching of full seq
916 - )
881 + From the sequence hash, with prefix caching enabled, we could also calculate
882 + the number of cached tokens for the sequence by looking up the number of
883 + cached block hashes in the allocator.
884 + """
885
918 - # Detect if there is a "gap"
919 - has_gap = len(computed_block_ids) < num_cur_blocks
886 + def __init__(
887 + self,
888 + allocator: DeviceAwareBlockAllocator,
889 + block_size: int,
890 + enable_caching: bool,
891 + ):
892 + self._allocator = allocator
893 + self._block_size = block_size
894 + self._enable_caching = enable_caching
895 +
896 + # A map from seq_id to the list of block hashes for the
897 + # sequence. This is so that we don't have to recompute the block hashes
898 + # for the sequence when we need to check if the sequence is cached.
899 + # Note a block that's not full will not have its hash calculated and
900 + # recorded.
901 + self._seq_id_to_blocks_hashes: Dict[int, List[int]] = {}
902 +
903 + # A map from seq_id to the number of tokens that are cached for the
904 + # sequence.
905 + # We need this so that a sequence in continuous prefill doesn't
906 + # accidentally see its cached token count change. See comments in
907 + # `get_num_cached_tokens` for more details.
908 + self._seq_id_to_num_tokens_computed: Dict[int, int] = {}
909 +
910 + def _update_seq_hashes(self, seq: Sequence) -> None:
911 + """Incrementally update the sequence's block hashes and record them."""
912 + assert self._enable_caching
913 +
914 + block_hashes_recorded = self._seq_id_to_blocks_hashes.get(
915 + seq.seq_id, [])
916 + cur_num_blocks_recorded = len(block_hashes_recorded)
917 + token_ids = seq.get_token_ids()
918 + assert len(token_ids) >= cur_num_blocks_recorded * self._block_size, (
919 + f"The sequence has {len(token_ids)} tokens, but"
920 + f" already recorded {cur_num_blocks_recorded} blocks. "
921 + "This should not happen since we assume blocks are "
922 + "only appended other than recomputation. When the sequence is "
923 + "recomputed, we should have removed the info of the old blocks.")
924 + # Update the computed block hashes for the sequence. Since only full
925 + # blocks are considered as "computed", we take floor here.
926 + num_computed_blocks = len(token_ids) // self._block_size
927 +
928 + # We need to know the hash of the previous block to compute the hash of
929 + # the current block so that blocks could be uniquely identified across
930 + # sequences of prefixes.
931 + prev_block_hash = (None if cur_num_blocks_recorded == 0 else
932 + block_hashes_recorded[-1])
933 + # Only update the computed block hashes for the new blocks
934 + for i in range(cur_num_blocks_recorded, num_computed_blocks):
935 + assert len(token_ids) >= (i + 1) * self._block_size
936 + block_token_ids = token_ids[i * self._block_size:(i + 1) *
937 + self._block_size]
938 + # This has to be kept in sync with the allocator's hash
939 + # calculation.
940 + block_hash = PrefixCachingBlock.hash_block_tokens(
941 + is_first_block=prev_block_hash is None,
942 + prev_block_hash=prev_block_hash,
943 + cur_block_token_ids=block_token_ids,
944 + )
945 + block_hashes_recorded.append(block_hash)
946 + prev_block_hash = block_hash
947 +
948 + self._seq_id_to_blocks_hashes[seq.seq_id] = block_hashes_recorded
949 +
950 + def get_num_cached_tokens(self, seq: Sequence) -> int:
951 + if not self._enable_caching:
952 + return 0
953 +
954 + # We always try to update the sequence hashes on the fly.
955 + # This is to ensure that we don't miss any cached tokens for the
956 + # sequence during decode.
957 + # This routine should only update hash for any new blocks too.
958 + self._update_seq_hashes(seq)
959 +
960 + num_computed_tokens_prev = self._seq_id_to_num_tokens_computed.get(
961 + seq.seq_id, None)
962 +
963 + # TODO(rickyx): This hack could be removed once we mark blocks as
964 + # computed correctly with chunked prefills.
965 + if num_computed_tokens_prev is not None and seq.is_prefill():
966 + # For a sequence that is still in prefill, we don't
967 + # recompute the number of cached tokens.
968 + # This also handles correctly chunked prefill since currently
969 + # we mark blocks as computed even if the sequence is still partially
970 + # prefilled. So a continuously prefilled sequence should not
971 + # see its cached token count change while running.
972 + return num_computed_tokens_prev
973 +
974 + block_hashes = self._seq_id_to_blocks_hashes[seq.seq_id]
975 +
976 + # This is O(logN), where N is the number of blocks.
977 + num_cached_blocks = len(
978 + self._allocator.find_cached_blocks_prefix(block_hashes))
979 + num_cached_tokens = num_cached_blocks * self._block_size
980 + self._seq_id_to_num_tokens_computed[seq.seq_id] = num_cached_tokens
981 + return num_cached_tokens
982
921 - # Record
922 - self._cached_computed_seq_blocks[seq_id] = (computed_block_ids,
923 - has_gap)
983 + def remove_seq(self, seq_id: int) -> None:
984 + """Stop tracking the sequence."""
985 + if not self._enable_caching:
986 + return
987 + assert seq_id in self._seq_id_to_blocks_hashes
988 + del self._seq_id_to_blocks_hashes[seq_id]
989
925 - return computed_block_ids
990 + assert seq_id in self._seq_id_to_num_tokens_computed
991 + del self._seq_id_to_num_tokens_computed[seq_id]
992
993
994 class LastAccessBlocksTracker:
vllm/core/block_manager.py
+14 -9
@@ -101,7 +101,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
101 self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
102
103 self._computed_blocks_tracker = ComputedBlocksTracker(
104 - self.block_allocator)
104 + self.block_allocator, self.block_size, self.enable_caching)
105 self._last_access_blocks_tracker = LastAccessBlocksTracker(
106 self.block_allocator)
107
@@ -170,7 +170,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
170 self.block_tables[seq.seq_id] = block_table
171
172 # Track seq
173 - self._computed_blocks_tracker.add_seq(seq.seq_id)
173 self._last_access_blocks_tracker.add_seq(seq.seq_id)
174
175 # Assign the block table for each sequence.
@@ -178,7 +177,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
177 self.block_tables[seq.seq_id] = block_table.fork()
178
179 # Track seq
181 - self._computed_blocks_tracker.add_seq(seq.seq_id)
180 self._last_access_blocks_tracker.add_seq(seq.seq_id)
181
182 # Allocate cross-attention block table for encoder sequence
@@ -314,11 +312,13 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
312 """
313 computed_seq_block_ids = []
314 for seq in seqs:
317 - computed_seq_block_ids.append(
318 - self._computed_blocks_tracker.
319 - get_cached_computed_blocks_and_update(
320 - seq.seq_id,
321 - self.block_tables[seq.seq_id].physical_block_ids))
315 + all_blocks = self.block_tables[seq.seq_id].physical_block_ids
316 + num_cached_tokens = (
317 + self._computed_blocks_tracker.get_num_cached_tokens(seq))
318 + assert num_cached_tokens % self.block_size == 0
319 + num_cached_blocks = num_cached_tokens // self.block_size
320 + computed_block_ids = all_blocks[:num_cached_blocks]
321 + computed_seq_block_ids.append(computed_block_ids)
322
323 # NOTE(sang): This assumes seq_block_ids doesn't contain any None.
324 return self.block_allocator.get_common_computed_block_ids(
@@ -332,7 +332,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
332 self.block_tables[child_seq.seq_id] = src_block_table.fork()
333
334 # Track child seq
335 - self._computed_blocks_tracker.add_seq(child_seq.seq_id)
335 self._last_access_blocks_tracker.add_seq(child_seq.seq_id)
336
337 def can_swap_in(self, seq_group: SequenceGroup,
@@ -503,3 +502,9 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
502 return AllocStatus.OK
503 else:
504 return AllocStatus.LATER
505 +
506 + def get_num_cached_tokens(self, seq: Sequence) -> int:
507 + """Get the number of tokens in blocks that are already computed and
508 + cached in the block manager for the sequence.
509 + """
510 + return self._computed_blocks_tracker.get_num_cached_tokens(seq)
vllm/core/interfaces.py
+4
@@ -121,3 +121,7 @@ class BlockSpaceManager(ABC):
121 def get_prefix_cache_hit_rate(self, device: Device) -> float:
122 """Prefix cache hit rate. -1 means not supported or disabled."""
123 pass
124 +
125 + @abstractmethod
126 + def get_num_cached_tokens(self, seq: Sequence) -> int:
127 + pass
vllm/core/placeholder_block_space_manager.py
+3
@@ -89,3 +89,6 @@ class PlaceholderBlockSpaceManager(BlockSpaceManager):
89
90 def get_prefix_cache_hit_rate(self, device: Device) -> float:
91 return -1
92 +
93 + def get_num_cached_tokens(self, seq: Sequence) -> int:
94 + return 0
vllm/core/scheduler.py
+253 -85
@@ -56,11 +56,16 @@ class SchedulingBudget:
56 max_num_seqs: int
57 _request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
58 _request_ids_num_curr_seqs: Set[str] = field(default_factory=set)
59 + # Number of cached tokens in the batch.
60 + _num_cached_tokens: int = 0
61 + # Number of actual non-cached tokens in the batch.
62 _num_batched_tokens: int = 0
63 _num_curr_seqs: int = 0
64
65 def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
63 - assert num_new_tokens != 0
66 + # We allow num_new_tokens to be 0 when the entire sequence has
67 + # been cached.
68 + assert num_new_tokens >= 0
69 assert num_new_seqs != 0
70 return (self.num_batched_tokens + num_new_tokens <= self.token_budget
71 and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
@@ -68,12 +73,18 @@ class SchedulingBudget:
73 def remaining_token_budget(self):
74 return self.token_budget - self.num_batched_tokens
75
71 - def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
76 + def add_num_batched_tokens(self,
77 + req_id: str,
78 + num_batched_tokens: int,
79 + num_cached_tokens: int = 0):
80 if req_id in self._request_ids_num_batched_tokens:
81 return
82 + assert num_cached_tokens >= 0
83 + assert num_batched_tokens >= 0
84
85 self._request_ids_num_batched_tokens.add(req_id)
86 self._num_batched_tokens += num_batched_tokens
87 + self._num_cached_tokens += num_cached_tokens
88
89 def subtract_num_batched_tokens(self, req_id: str,
90 num_batched_tokens: int):
@@ -101,6 +112,10 @@ class SchedulingBudget:
112 def num_curr_seqs(self):
113 return self._num_curr_seqs
114
115 + @property
116 + def num_cached_tokens(self):
117 + return self._num_cached_tokens
118 +
119
120 @dataclass
121 class ScheduledSequenceGroup:
@@ -541,9 +556,19 @@ class Scheduler:
556 assert len(self._async_stopped) == 0
557 while running_queue:
558 seq_group = running_queue[0]
544 - num_running_tokens = self._get_num_new_tokens(
545 - seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
546 -
559 + # We discard the cached tokens info here because we don't need it
560 + # for running sequence:
561 + # 1. If a sequence is running with chunked prefill, the cached
562 + # tokens info was already used for the first prefill.
563 + # 2. If a sequence is running with non-chunked prefill, then
564 + # there it's a decoding sequence, and the cached tokens info is
565 + # irrelevant.
566 + num_uncached_new_tokens, _ = (
567 + self._get_num_new_uncached_and_cached_tokens(
568 + seq_group, SequenceStatus.RUNNING, enable_chunking,
569 + budget))
570 +
571 + num_running_tokens = num_uncached_new_tokens
572 if num_running_tokens == 0:
573 # No budget => Stop
574 break
@@ -715,13 +740,15 @@ class Scheduler:
740 # The total number of sequences in the RUNNING state should not
741 # exceed the maximum number of sequences.
742 num_new_seqs = seq_group.get_max_num_running_seqs()
718 - num_new_tokens = self._get_num_new_tokens(seq_group,
719 - SequenceStatus.SWAPPED,
720 - enable_chunking, budget)
721 -
722 - if (num_new_tokens == 0
723 - or not budget.can_schedule(num_new_tokens=num_new_tokens,
724 - num_new_seqs=num_new_seqs)):
743 + num_new_tokens_uncached, num_new_tokens_cached = (
744 + self._get_num_new_uncached_and_cached_tokens(
745 + seq_group, SequenceStatus.SWAPPED, enable_chunking,
746 + budget))
747 +
748 + if num_new_tokens_uncached == 0 or not budget.can_schedule(
749 + num_new_tokens=num_new_tokens_uncached,
750 + num_new_seqs=num_new_seqs,
751 + ):
752 break
753
754 if lora_int_id > 0 and curr_loras is not None:
@@ -732,12 +759,19 @@ class Scheduler:
759 is_prefill = seq_group.is_prefill()
760 if is_prefill:
761 prefill_seq_groups.append(
735 - ScheduledSequenceGroup(seq_group,
736 - token_chunk_size=num_new_tokens))
762 + ScheduledSequenceGroup(
763 + seq_group,
764 + token_chunk_size=num_new_tokens_uncached +
765 + num_new_tokens_cached,
766 + ))
767 else:
768 decode_seq_groups.append(
769 ScheduledSequenceGroup(seq_group, token_chunk_size=1))
740 - budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
770 + budget.add_num_batched_tokens(
771 + seq_group.request_id,
772 + num_batched_tokens=num_new_tokens_uncached,
773 + num_cached_tokens=num_new_tokens_cached,
774 + )
775 budget.add_num_seqs(seq_group.request_id, num_new_seqs)
776
777 swapped_queue.extendleft(leftover_swapped)
@@ -803,26 +837,30 @@ class Scheduler:
837 if waiting_queue:
838 seq_group = waiting_queue.popleft()
839 num_new_seqs = seq_group.get_max_num_running_seqs()
806 - num_new_tokens = self._get_num_new_tokens(seq_group,
807 - SequenceStatus.WAITING,
808 - False, budget)
840 + num_new_tokens_uncached, _ = (
841 + self._get_num_new_uncached_and_cached_tokens(
842 + seq_group, SequenceStatus.WAITING, False, budget))
843
844 #Only preempt if priority inversion exists
845 while running_queue and self._get_priority(
846 running_queue[-1]) > self._get_priority(seq_group):
847 #Only preempt if waiting sequence cannot be allocated
848 can_allocate = self.block_manager.can_allocate(seq_group)
815 - if (num_new_tokens and can_allocate == AllocStatus.OK
816 - and budget.can_schedule(num_new_tokens=num_new_tokens,
817 - num_new_seqs=num_new_seqs)):
849 + if (num_new_tokens_uncached > 0
850 + and can_allocate == AllocStatus.OK
851 + and budget.can_schedule(
852 + num_new_tokens=num_new_tokens_uncached,
853 + num_new_seqs=num_new_seqs,
854 + )):
855 break
856
857 #Adjust budget to remove the victim sequence group
858 vseq_group = running_queue.pop()
822 - num_running_tokens = self._get_num_new_tokens(
823 - vseq_group, SequenceStatus.RUNNING, False, budget)
824 - budget.subtract_num_batched_tokens(vseq_group.request_id,
825 - num_running_tokens)
859 + num_running_tokens_uncached, _ = (
860 + self._get_num_new_uncached_and_cached_tokens(
861 + vseq_group, SequenceStatus.RUNNING, False, budget))
862 + budget.subtract_num_batched_tokens(
863 + vseq_group.request_id, num_running_tokens_uncached)
864 num_running_seqs = vseq_group.get_max_num_running_seqs()
865 budget.subtract_num_seqs(vseq_group.request_id,
866 num_running_seqs)
@@ -882,9 +920,12 @@ class Scheduler:
920 assert len(waiting_seqs) == 1, (
921 "Waiting sequence group should have only one prompt "
922 "sequence.")
885 - num_new_tokens = self._get_num_new_tokens(seq_group,
886 - SequenceStatus.WAITING,
887 - enable_chunking, budget)
923 + num_new_tokens_uncached, num_new_tokens_cached = (
924 + self._get_num_new_uncached_and_cached_tokens(
925 + seq_group, SequenceStatus.WAITING, enable_chunking,
926 + budget))
927 + num_new_tokens = num_new_tokens_uncached + num_new_tokens_cached
928 +
929 if not enable_chunking:
930 num_prompt_tokens = waiting_seqs[0].get_len()
931 assert num_new_tokens == num_prompt_tokens
@@ -935,10 +976,18 @@ class Scheduler:
976 waiting_queue.popleft()
977 continue
978
979 + if (budget.num_batched_tokens >=
980 + self.scheduler_config.max_num_batched_tokens):
981 + # We've reached the budget limit - since there might be
982 + # continuous prefills in the running queue, we should break
983 + # to avoid scheduling any new prefills.
984 + break
985 +
986 num_new_seqs = seq_group.get_max_num_running_seqs()
939 - if (num_new_tokens == 0
940 - or not budget.can_schedule(num_new_tokens=num_new_tokens,
941 - num_new_seqs=num_new_seqs)):
987 + if num_new_tokens_uncached == 0 or not budget.can_schedule(
988 + num_new_tokens=num_new_tokens_uncached,
989 + num_new_seqs=num_new_seqs,
990 + ):
991 break
992
993 # Can schedule this request.
@@ -967,7 +1016,11 @@ class Scheduler:
1016 seq_groups.append(
1017 ScheduledSequenceGroup(seq_group=seq_group,
1018 token_chunk_size=num_new_tokens))
970 - budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
1019 + budget.add_num_batched_tokens(
1020 + seq_group.request_id,
1021 + num_batched_tokens=num_new_tokens_uncached,
1022 + num_cached_tokens=num_new_tokens_cached,
1023 + )
1024 budget.add_num_seqs(seq_group.request_id, num_new_seqs)
1025
1026 # Queue requests that couldn't be scheduled.
@@ -1075,7 +1128,8 @@ class Scheduler:
1128 return SchedulerOutputs(
1129 scheduled_seq_groups=scheduled_seq_groups,
1130 num_prefill_groups=num_prefill_groups,
1078 - num_batched_tokens=budget.num_batched_tokens,
1131 + num_batched_tokens=budget.num_batched_tokens +
1132 + budget.num_cached_tokens,
1133 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
1134 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
1135 blocks_to_copy=blocks_to_copy,
@@ -1119,7 +1173,6 @@ class Scheduler:
1173 running_scheduled.swapped_out) == 0:
1174 swapped_in = self._schedule_swapped(budget, curr_loras)
1175
1122 - # Schedule new prefills.
1176 prefills = self._schedule_prefills(budget,
1177 curr_loras,
1178 enable_chunking=True)
@@ -1157,7 +1210,8 @@ class Scheduler:
1210 num_prefill_groups=(len(prefills.seq_groups) +
1211 len(swapped_in.prefill_seq_groups) +
1212 len(running_scheduled.prefill_seq_groups)),
1160 - num_batched_tokens=budget.num_batched_tokens,
1213 + num_batched_tokens=budget.num_batched_tokens +
1214 + budget.num_cached_tokens,
1215 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
1216 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
1217 blocks_to_copy=running_scheduled.blocks_to_copy +
@@ -1584,64 +1638,178 @@ class Scheduler:
1638
1639 return self.scheduler_config.num_lookahead_slots
1640
1587 - def _get_num_new_tokens(self, seq_group: SequenceGroup,
1588 - status: SequenceStatus, enable_chunking: bool,
1589 - budget: SchedulingBudget) -> int:
1590 - """Get the next new tokens to compute for a given sequence group
1591 - that's in a given `status`.
1641 + def _get_num_new_uncached_and_cached_tokens(
1642 + self,
1643 + seq_group: SequenceGroup,
1644 + status: SequenceStatus,
1645 + enable_chunking: bool,
1646 + budget: SchedulingBudget,
1647 + ) -> Tuple[int, int]:
1648 + """
1649 + Returns the number of new uncached and cached tokens to schedule for a
1650 + given sequence group that's in a given `status`.
1651
1652 The API could chunk the number of tokens to compute based on `budget`
1653 if `enable_chunking` is True. If a sequence group has multiple
1654 sequences (e.g., running beam search), it means it is in decoding
1655 phase, so chunking doesn't happen.
1656
1598 - Returns 0 if the new token cannot be computed due to token budget.
1657 + Returns (0, 0) if the new token cannot be computed due to token budget.
1658 +
1659 + The cached tokens's blocks are already computed, and the attention
1660 + backend will reuse the cached blocks rather than recomputing them. So
1661 + the scheduler could schedule these cached tokens "for free".
1662 +
1663 + Args:
1664 + seq_group: The sequence group to get the number of new tokens to
1665 + schedule.
1666 + status: The status of the sequences to get the number of new tokens
1667 + to schedule.
1668 + enable_chunking: Whether to chunk the number of tokens to compute.
1669 + budget: The budget to chunk the number of tokens to compute.
1670 +
1671 +
1672 + Returns:
1673 + A tuple of two ints. The first int is the number of new uncached
1674 + tokens to schedule. The second int is the number of cached tokens.
1675 + If no more new tokens can be scheduled, returns (0, 0).
1676 """
1600 - num_new_tokens = 0
1677 + num_cached_new_tokens = 0
1678 + num_uncached_new_tokens = 0
1679 +
1680 seqs = seq_group.get_seqs(status=status)
1681 + # Compute the number of new uncached and cached tokens for
1682 + # each sequence.
1683 for seq in seqs:
1603 - num_new_tokens += seq.get_num_new_tokens()
1604 - assert num_new_tokens > 0
1605 - # Chunk if a running request cannot fit in the given budget.
1606 - # If number of seq > 1, it means it is doing beam search
1607 - # in a decode phase. Do not chunk.
1684 + if not seq.is_prefill():
1685 + # Decode sequences should always just have 1 uncached token
1686 + # TODO(rickyx): Actually is this still correct for multi-step?
1687 + num_uncached_new_tokens += 1
1688 + continue
1689 +
1690 + num_computed_tokens_seq = seq.get_num_computed_tokens()
1691 + all_num_new_tokens_seq = seq.get_len() - num_computed_tokens_seq
1692 + if not self.cache_config.enable_prefix_caching:
1693 + # If prefix caching is not enabled, all new tokens are uncached.
1694 + num_uncached_new_tokens += all_num_new_tokens_seq
1695 + continue
1696 +
1697 + # NOTE: the cache token might be currently in a block that's in an
1698 + # evictor meaning that it's not yet allocated. However, we don't
1699 + # exclude such tokens in the cache count because it will be
1700 + # guaranteed to be allocated later if the sequence can be allocated.
1701 + num_cached_tokens_seq = self.block_manager.get_num_cached_tokens(
1702 + seq)
1703 +
1704 + # Sanity check.
1705 + if num_cached_tokens_seq < num_computed_tokens_seq:
1706 + # This should only happen with chunked prefill, and
1707 + # the seq is still in prefill. The `num_cached_tokens_seq`
1708 + # is the value we calculated on scheduling the first prefill.
1709 + # For subsequent continuous prefill steps, we cached the
1710 + # number of cache tokens for the sequence so the cached token
1711 + # count could be less than the number of computed tokens.
1712 + # See comments on `ComputedBlocksTracker` for more details.
1713 + assert (
1714 + seq.is_prefill() and seq.status == SequenceStatus.RUNNING
1715 + and self.scheduler_config.chunked_prefill_enabled
1716 + ), ("Number of cached tokens should not be less than the "
1717 + "number of computed tokens for a sequence that's still "
1718 + f"in prefill. But there are {num_cached_tokens_seq} cached "
1719 + f"tokens and {num_computed_tokens_seq} computed tokens "
1720 + f"for sequence {seq.seq_id}.")
1721 +
1722 + num_cached_new_tokens_seq = max(
1723 + 0, num_cached_tokens_seq - num_computed_tokens_seq)
1724 + num_uncached_new_tokens_seq = (all_num_new_tokens_seq -
1725 + num_cached_new_tokens_seq)
1726 +
1727 + num_uncached_new_tokens += num_uncached_new_tokens_seq
1728 + num_cached_new_tokens += num_cached_new_tokens_seq
1729 +
1730 + if num_uncached_new_tokens == 0 and num_cached_new_tokens > 0:
1731 + # For a fully cached hit sequence, we actually need to recompute the
1732 + # last token. So we need at least 1 uncached token to schedule.
1733 + # See ModelRunner._compute_for_prefix_cache_hit for more details.
1734 + num_uncached_new_tokens = 1
1735 + num_cached_new_tokens -= 1
1736 +
1737 if enable_chunking and len(seqs) == 1:
1609 - remaining_token_budget = budget.remaining_token_budget()
1610 - if self.scheduler_config.is_multi_step:
1611 - # The current multi-step + chunked prefill capability does
1612 - # not actually support chunking prompts.
1613 - #
1614 - # Therefore, `num_new_tokens` is computed in the same fashion
1615 - # for both multi-step+chunked-prefill &
1616 - # multi-step+chunked-prefill+APC
1617 - #
1618 - # Prompts with more tokens than the current remaining budget
1619 - # are postponed to future scheduler steps
1620 - if num_new_tokens > self._get_prompt_limit(seq_group):
1621 - # If the seq_group is in prompt-stage, pass the
1622 - # num_new_tokens as-is so the caller can ignore
1623 - # the sequence.
1624 - pass
1625 - else:
1626 - num_new_tokens = 0 \
1627 - if num_new_tokens > remaining_token_budget \
1628 - else num_new_tokens
1629 - elif self.cache_config.enable_prefix_caching:
1630 - # When prefix caching is enabled, we always allocate
1631 - # the number of new tokens that is dividable by the block
1632 - # size to avoid partial block matching.
1633 - block_size = self.cache_config.block_size
1634 - remainder = budget.token_budget % block_size
1635 - if remainder != 0:
1636 - raise ValueError("When enabling chunked prefill and "
1637 - "prefix caching, max_num_batched_tokens "
1638 - "(chunk size) must be dividable by "
1639 - "block size, but got chunk_size "
1640 - f"({budget.token_budget}) % block_size "
1641 - f"({block_size}) = {remainder}")
1642 - if remaining_token_budget < num_new_tokens:
1643 - num_new_tokens = (remaining_token_budget //
1644 - block_size) * block_size
1645 - else:
1646 - num_new_tokens = min(num_new_tokens, remaining_token_budget)
1738 + # Chunk if a running request cannot fit in the given budget.
1739 + # If number of seq > 1, it means it is doing beam search
1740 + # in a decode phase. Do not chunk.
1741 + num_uncached_new_tokens = self._chunk_new_tokens_to_schedule(
1742 + self.scheduler_config,
1743 + self.cache_config,
1744 + budget,
1745 + self._get_prompt_limit(seq_group),
1746 + num_uncached_new_tokens,
1747 + )
1748 +
1749 + return num_uncached_new_tokens, num_cached_new_tokens
1750 +
1751 + @staticmethod
1752 + def _chunk_new_tokens_to_schedule(
1753 + scheduler_config: SchedulerConfig,
1754 + cache_config: CacheConfig,
1755 + budget: SchedulingBudget,
1756 + prompt_limit: int,
1757 + num_new_tokens: int,
1758 + ) -> int:
1759 + """
1760 + Chunks the number of new tokens to schedule based on the budget when
1761 + chunked prefill is enabled.
1762 +
1763 + Args:
1764 + scheduler_config: The scheduler config.
1765 + cache_config: The cache config.
1766 + budget: The budget to chunk the number of tokens to compute.
1767 + prompt_limit: The maximum number of tokens allowed in a prompt.
1768 + num_new_tokens: The number of new tokens to schedule.
1769 +
1770 + Returns:
1771 + The number of new tokens to schedule after chunking.
1772 + """
1773 + remaining_token_budget = budget.remaining_token_budget()
1774 + if scheduler_config.is_multi_step:
1775 + # The current multi-step + chunked prefill capability does
1776 + # not actually support chunking prompts.
1777 + #
1778 + # Therefore, `num_new_tokens` is computed in the same fashion
1779 + # for both multi-step+chunked-prefill &
1780 + # multi-step+chunked-prefill+APC
1781 + #
1782 + # Prompts with more tokens than the current remaining budget
1783 + # are postponed to future scheduler steps
1784 + if num_new_tokens > prompt_limit:
1785 + # If the seq_group is in prompt-stage, pass the
1786 + # num_new_tokens as-is so the caller can ignore
1787 + # the sequence.
1788 + return num_new_tokens
1789 +
1790 + return (0 if num_new_tokens > remaining_token_budget else
1791 + num_new_tokens)
1792 +
1793 + if cache_config.enable_prefix_caching:
1794 + # Adjust the remaining token budget to be divisible by the block
1795 + # size when prefix caching is enabled.
1796 +
1797 + # When prefix caching is enabled, we always allocate
1798 + # the number of new tokens that is dividable by the block
1799 + # size to avoid partial block matching.
1800 + block_size = cache_config.block_size
1801 + remainder = budget.token_budget % block_size
1802 + if remainder != 0:
1803 + raise ValueError("When enabling chunked prefill and "
1804 + "prefix caching, max_num_batched_tokens "
1805 + "(chunk size) must be dividable by "
1806 + "block size, but got chunk_size "
1807 + f"({budget.token_budget}) % block_size "
1808 + f"({block_size}) = {remainder}")
1809 + # Round down to block size.
1810 + remaining_token_budget = (remaining_token_budget // block_size *
1811 + block_size)
1812 +
1813 + num_new_tokens = min(num_new_tokens, remaining_token_budget)
1814 +
1815 return num_new_tokens
vllm/sequence.py
+3
@@ -579,6 +579,9 @@ class Sequence:
579 return 1
580 return self.data.get_num_uncomputed_tokens()
581
582 + def get_num_computed_tokens(self) -> int:
583 + return self.data.get_num_computed_tokens()
584 +
585 def is_prefill(self) -> bool:
586 return self.data.stage == SequenceStage.PREFILL
587