A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
Prefix Cache Aware Scheduling [1/n] (#10128)
Signed-off-by: rickyx <rickyx@anyscale.com>
Ricky Xu committed
Nov 22, 2024 at 21:15 UTC
4634a89d18569ef0ee2d7dd2d535377a1f460188
13 files changed
+967
-241
tests/core/block/test_prefix_caching_block.py
+175
-6
@@ -5,9 +5,14 @@ from unittest.mock import MagicMock
5
6
import pytest
7
8
+from tests.core.utils import create_dummy_sequence
9
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
10
from vllm.core.block.interfaces import Block, BlockAllocator
9
-from vllm.core.block.prefix_caching_block import (PrefixCachingBlock,
11
+from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
12
+ PrefixCachingBlock,
13
PrefixCachingBlockAllocator)
14
+from vllm.sequence import Logprob
15
+from vllm.utils import Device
16
17
18
class TestPrefixCachingBlock:
@@ -726,18 +731,71 @@ class TestPrefixCachingBlockAllocator:
731
token_ids=common_token_ids,
732
allocator=allocator,
733
)
729
- block_ids = [block.block_id for block in blocks]
734
+ block_hashes = [block.content_hash for block in blocks]
735
# The allocated blocks should be marked as touched
736
# but not computed.
732
- computed_block_ids = allocator.get_computed_block_ids(
733
- [], block_ids, skip_last_block_id=False)
737
+ computed_block_ids = allocator.find_cached_blocks_prefix(
738
+ block_hashes)
739
assert len(computed_block_ids) == 0
740
741
allocator.mark_blocks_as_computed([])
737
- computed_block_ids = allocator.get_computed_block_ids(
738
- [], block_ids, skip_last_block_id=False)
742
+ computed_block_ids = allocator.find_cached_blocks_prefix(
743
+ block_hashes=block_hashes)
744
assert len(computed_block_ids) == common_blocks
745
746
+ @staticmethod
747
+ def test_find_cached_blocks_prefix():
748
+ """
749
+ This test verifies the behavior of find_cached_blocks_prefix.
750
+ """
751
+ block_size = 4
752
+ num_blocks = 8
753
+ total_test_blocks = 12
754
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
755
+ block_size=block_size)
756
+
757
+ token_ids = list(range(total_test_blocks * block_size))
758
+ block_tokens_seq1 = token_ids[:num_blocks * block_size]
759
+ blocks_seq1 = TestPrefixCachingBlockAllocator.create_immutable_chain(
760
+ block_size=block_size,
761
+ token_ids=block_tokens_seq1,
762
+ allocator=allocator,
763
+ )
764
+ block_hashes_seq1 = [block.content_hash for block in blocks_seq1]
765
+ allocator.mark_blocks_as_computed([])
766
+
767
+ # All blocks should be cached.
768
+ cached_blocks_seq1 = allocator.find_cached_blocks_prefix(
769
+ block_hashes=block_hashes_seq1)
770
+ assert len(cached_blocks_seq1) == num_blocks
771
+
772
+ # Free the first sequence.
773
+ for block in blocks_seq1:
774
+ allocator.free(block)
775
+
776
+ # All blocks should be still be cached if not required to be allocated.
777
+ cached_blocks = allocator.find_cached_blocks_prefix(
778
+ block_hashes=block_hashes_seq1)
779
+ assert len(cached_blocks) == num_blocks
780
+
781
+ block_tokens_seq2 = token_ids[num_blocks * block_size:]
782
+ blocks_seq2 = TestPrefixCachingBlockAllocator.create_immutable_chain(
783
+ block_size=block_size,
784
+ token_ids=block_tokens_seq2,
785
+ allocator=allocator,
786
+ )
787
+ block_hashes_seq2 = [block.content_hash for block in blocks_seq2]
788
+ allocator.mark_blocks_as_computed([])
789
+ cached_blocks = allocator.find_cached_blocks_prefix(
790
+ block_hashes=block_hashes_seq2)
791
+ assert len(cached_blocks) == len(blocks_seq2)
792
+
793
+ # Half of the blocks from seq1 should still be cached.
794
+ num_evicted_blocks = len(blocks_seq2)
795
+ cached_blocks = allocator.find_cached_blocks_prefix(
796
+ block_hashes=block_hashes_seq1)
797
+ assert len(cached_blocks) == len(blocks_seq1) - num_evicted_blocks
798
+
799
@staticmethod
800
def create_immutable_chain(
801
block_size: int,
@@ -762,3 +820,114 @@ class TestPrefixCachingBlockAllocator:
820
blocks.append(prev_block)
821
822
return blocks
823
+
824
+
825
+class TestComputedBlocksTracker:
826
+
827
+ @staticmethod
828
+ def _get_mock_allocator():
829
+ return MagicMock(spec=PrefixCachingBlockAllocator)
830
+
831
+ @staticmethod
832
+ def test_get_num_cached_tokens():
833
+ """
834
+ Test it correctly computes the number of cached tokens for a given
835
+ sequence:
836
+
837
+ - The cache token count is derived from the number of cached blocks.
838
+ - The cache token count is updated when the allocator is updated.
839
+ - When a sequence is removed, the cache token count should be updated
840
+ accordingly.
841
+
842
+ # TODO(rickyx): This behaviour for prefill sequence is a hack until
843
+ we fix the computed blocks tracking.
844
+ - The cache token count for prefill sequence doesn't change while
845
+ the sequence is in continuous prefill (chunked prefill).
846
+ """
847
+ block_size = 4
848
+ mock_allocator = TestComputedBlocksTracker._get_mock_allocator()
849
+ tracker = ComputedBlocksTracker(
850
+ allocator=mock_allocator,
851
+ block_size=block_size,
852
+ enable_caching=True,
853
+ )
854
+
855
+ # Not yet allocated.
856
+ tokens = [0, 1, 2, 3, 4, 5]
857
+ seq1 = create_dummy_sequence(request_id=0,
858
+ token_ids=tokens,
859
+ block_size=block_size)
860
+ mock_allocator.find_cached_blocks_prefix.return_value = []
861
+ assert tracker.get_num_cached_tokens(seq1) == 0
862
+
863
+ mock_allocator.find_cached_blocks_prefix.return_value = [
864
+ None
865
+ ] # 1 block cached.
866
+ # Result is cached for prefill sequence.
867
+ assert tracker.get_num_cached_tokens(seq1) == 0
868
+
869
+ # Mark the sequence as non-prefill.
870
+ seq1.data.update_num_computed_tokens(len(tokens)) # 6 tokens computed.
871
+ assert not seq1.is_prefill()
872
+
873
+ # Recomputes for decoding sequence.
874
+ assert tracker.get_num_cached_tokens(seq1) == 4
875
+
876
+ # Append new tokens to the sequence.
877
+ num_new_tokens = 3
878
+ for i in range(num_new_tokens):
879
+ seq1.append_token_id(i, {i: Logprob(logprob=0.0)})
880
+
881
+ assert tracker.get_num_cached_tokens(seq1) == 4
882
+
883
+ # Update the allocator.
884
+ mock_allocator.find_cached_blocks_prefix.return_value = [
885
+ None
886
+ ] * 2 # 2 blocks cached.
887
+ assert tracker.get_num_cached_tokens(seq1) == 8
888
+
889
+ # Remove the sequence.
890
+ tracker.remove_seq(seq1.seq_id)
891
+
892
+ # Re-create the sequence with the same request id to simulate recompute.
893
+ seq1 = create_dummy_sequence(request_id=0,
894
+ token_ids=tokens,
895
+ block_size=block_size)
896
+ mock_allocator.find_cached_blocks_prefix.return_value = [
897
+ ] # no cached block
898
+ assert tracker.get_num_cached_tokens(seq1) == 0
899
+
900
+ @staticmethod
901
+ def test_correct_block_hash():
902
+ """
903
+ Test that the block hash is correctly computed for a sequence (should
904
+ match the underlying block allocator's block hash). So the number of
905
+ cached tokens is correctly retrieved.
906
+ """
907
+ block_size = 4
908
+ allocator = CpuGpuBlockAllocator.create(
909
+ allocator_type="prefix_caching",
910
+ num_gpu_blocks=16,
911
+ num_cpu_blocks=16,
912
+ block_size=block_size,
913
+ )
914
+ gpu_allocator = allocator._allocators[Device.GPU]
915
+
916
+ tracker = ComputedBlocksTracker(
917
+ allocator=allocator,
918
+ block_size=block_size,
919
+ enable_caching=True,
920
+ )
921
+
922
+ tokens = list(range(block_size * 4)) # 4 blocks.
923
+ seq = create_dummy_sequence(request_id=0,
924
+ token_ids=tokens,
925
+ block_size=block_size)
926
+ _ = TestPrefixCachingBlockAllocator.create_immutable_chain(
927
+ block_size=block_size,
928
+ token_ids=tokens,
929
+ allocator=gpu_allocator,
930
+ )
931
+ allocator.mark_blocks_as_computed([])
932
+
933
+ assert tracker.get_num_cached_tokens(seq) == len(tokens)
tests/core/test_scheduler.py
+175
-4
@@ -12,9 +12,9 @@ from vllm.core.scheduler import Scheduler, SchedulingBudget
12
from vllm.lora.request import LoRARequest
13
from vllm.sequence import SequenceGroup
14
15
-from .utils import (append_new_token, append_new_token_seq_group,
16
- create_dummy_prompt, get_sequence_groups,
17
- schedule_and_update_computed_tokens)
15
+from .utils import (append_new_token, append_new_token_seq,
16
+ append_new_token_seq_group, create_dummy_prompt,
17
+ get_sequence_groups, schedule_and_update_computed_tokens)
18
19
20
def test_scheduler_add_seq_group():
@@ -305,6 +305,8 @@ def initialize_scheduler(
305
block_size=4,
306
num_cpu_blocks=8,
307
num_gpu_blocks=8,
308
+ enable_prefix_caching=False,
309
+ enable_chunked_prefill=False,
310
):
311
block_size = block_size
312
scheduler_config = SchedulerConfig(
@@ -312,8 +314,15 @@ def initialize_scheduler(
314
max_num_batched_tokens=max_token_budget,
315
max_num_seqs=max_num_seqs,
316
max_model_len=max_model_len,
317
+ enable_chunked_prefill=enable_chunked_prefill,
318
+ )
319
+ cache_config = CacheConfig(
320
+ block_size,
321
+ 1.0,
322
+ 1,
323
+ "auto",
324
+ enable_prefix_caching=enable_prefix_caching,
325
)
316
- cache_config = CacheConfig(block_size, 1.0, 1, "auto")
326
cache_config.num_cpu_blocks = num_cpu_blocks
327
cache_config.num_gpu_blocks = num_gpu_blocks
328
scheduler = Scheduler(scheduler_config, cache_config, lora_config)
@@ -800,3 +809,165 @@ def test_scheduling_budget():
809
assert budget.num_curr_seqs == 0
810
budget.subtract_num_seqs(seq_group.request_id, 2)
811
assert budget.num_curr_seqs == 0
812
+
813
+
814
+@pytest.mark.parametrize("enable_prefix_caching", [True, False])
815
+def test_prefix_caching_aware_prefills(enable_prefix_caching):
816
+ """
817
+ Test the below scenario:
818
+
819
+ For 3 sequences, seqA, seqB, seqC, share the first block as prefix.
820
+
821
+ The test verifies the below scenarios:
822
+ 1. SeqA is first scheduled.
823
+ 2. SeqB and SeqC can be prefilled together in a single schedule round
824
+ even though there are not enough token budgets to prefill both without
825
+ considering prefix caching.
826
+ """
827
+
828
+ block_size = 4
829
+ max_num_batched_tokens = 12
830
+ max_seq_group = 3
831
+ scheduler = initialize_scheduler(
832
+ block_size=block_size,
833
+ num_cpu_blocks=16,
834
+ num_gpu_blocks=16,
835
+ max_token_budget=max_num_batched_tokens,
836
+ max_num_seqs=max_seq_group,
837
+ max_model_len=max_num_batched_tokens,
838
+ enable_prefix_caching=enable_prefix_caching,
839
+ )
840
+
841
+ seqA_tokens = list(range(8))
842
+ num_shared_tokens = 4
843
+ seqB_tokens = seqA_tokens[:num_shared_tokens] + list(range(
844
+ 12, 16)) # Shared prefix first 4.
845
+ seqC_tokens = seqA_tokens[:num_shared_tokens] + list(range(
846
+ 16, 20)) # Shared prefix first 4.
847
+
848
+ seqA, seqA_group = create_dummy_prompt("0",
849
+ prompt_tokens=seqA_tokens,
850
+ block_size=block_size)
851
+ seqB, seqB_group = create_dummy_prompt("1",
852
+ prompt_tokens=seqB_tokens,
853
+ block_size=block_size)
854
+ seqC, seqC_group = create_dummy_prompt("2",
855
+ prompt_tokens=seqC_tokens,
856
+ block_size=block_size)
857
+
858
+ # Schedule seqA prefill.
859
+ scheduler.add_seq_group(seqA_group)
860
+ metas, out, _ = scheduler.schedule()
861
+ assert (len(out.scheduled_seq_groups) == 1
862
+ and out.scheduled_seq_groups[0].seq_group == seqA_group)
863
+ assert out.scheduled_seq_groups[0].token_chunk_size == len(seqA_tokens)
864
+
865
+ # Schedule seqA decode.
866
+ append_new_token_seq_group(len(seqA_tokens), seqA_group, 999)
867
+ metas, out, _ = scheduler.schedule()
868
+
869
+ assert len(out.scheduled_seq_groups) == 1
870
+ assert out.scheduled_seq_groups[0].seq_group == seqA_group
871
+ assert out.scheduled_seq_groups[0].token_chunk_size == 1
872
+
873
+ # Schedule seqB and seqC prefills should work with prefix caching.
874
+ scheduler.add_seq_group(seqB_group)
875
+ scheduler.add_seq_group(seqC_group)
876
+ metas, out, _ = scheduler.schedule()
877
+
878
+ if enable_prefix_caching:
879
+ assert len(out.scheduled_seq_groups) == 2
880
+ assert set([
881
+ out.scheduled_seq_groups[0].seq_group,
882
+ out.scheduled_seq_groups[1].seq_group,
883
+ ]) == set([seqB_group, seqC_group])
884
+ assert len(metas) == 2
885
+ for meta in metas:
886
+ assert meta.token_chunk_size == 8
887
+ assert (len(meta.computed_block_nums) == num_shared_tokens //
888
+ block_size) # 1 Block for the 8 tokens.
889
+ else:
890
+ assert len(out.scheduled_seq_groups) == 1
891
+ assert len(metas) == 1
892
+ assert metas[0].token_chunk_size == 8
893
+ assert len(metas[0].computed_block_nums) == 0 # No blocks computed.
894
+
895
+
896
+def test_no_multiple_partial_prefills_with_chunked_prefill_and_prefix_caching(
897
+):
898
+ """
899
+ This test verifies that we don't schedule new prefills if there's already
900
+ a continuous prefill in progress even though the new prefills with shared
901
+ prefix can fit in the token budget:
902
+
903
+ - SeqA is being chunked prefill.
904
+ - SeqB with the same prompt shouldn't be scheduled for prefill even though
905
+ there's enough token budget to prefill the cached tokens.
906
+ - Neither should seqC be scheduled.
907
+
908
+ - When seqA is in decoding phase, seqB and seqC can be scheduled.
909
+ - Entire seqB should be prefilled since it's a full prefix cache hit.
910
+ - SeqC would be partially prefilled with the prefix shared, and the
911
+ remaining unique tokens would be prefilled (rounded down to be
912
+ block-size aligned).
913
+ """
914
+
915
+ block_size = 2
916
+ max_num_batched_tokens = 4
917
+ max_seq_group = 3
918
+ scheduler = initialize_scheduler(
919
+ block_size=block_size,
920
+ num_cpu_blocks=16,
921
+ num_gpu_blocks=16,
922
+ max_token_budget=max_num_batched_tokens,
923
+ max_num_seqs=max_seq_group,
924
+ max_model_len=100,
925
+ enable_prefix_caching=True,
926
+ enable_chunked_prefill=True,
927
+ )
928
+
929
+ seqA_tokens = list(range(8))
930
+ seqB_tokens = seqA_tokens
931
+ seqC_shared_prefix_len = 4
932
+ seqC_tokens = seqA_tokens[:seqC_shared_prefix_len] + list(range(12, 20))
933
+
934
+ seqA, seqA_group = create_dummy_prompt("0",
935
+ prompt_tokens=seqA_tokens,
936
+ block_size=block_size)
937
+ seqB, seqB_group = create_dummy_prompt("1",
938
+ prompt_tokens=seqB_tokens,
939
+ block_size=block_size)
940
+
941
+ # Chunked prefill seqA.
942
+ scheduler.add_seq_group(seqA_group)
943
+ metas, out = schedule_and_update_computed_tokens(scheduler)
944
+ assert len(out.scheduled_seq_groups) == 1
945
+ assert out.scheduled_seq_groups[0].seq_group == seqA_group
946
+ assert out.scheduled_seq_groups[0].token_chunk_size == 4
947
+
948
+ # seqB should not be scheduled with ongoing prefills.
949
+ scheduler.add_seq_group(seqB_group)
950
+ metas, out = schedule_and_update_computed_tokens(scheduler)
951
+ assert len(out.scheduled_seq_groups) == 1
952
+ assert out.scheduled_seq_groups[0].seq_group == seqA_group
953
+ assert out.scheduled_seq_groups[0].token_chunk_size == 4
954
+
955
+ # both seqB and seqC can now be scheduled with seqA is over.
956
+ # seqA is in decoding phase.
957
+ append_new_token_seq(seqA, 999)
958
+ seqC, seqC_group = create_dummy_prompt("2",
959
+ prompt_tokens=seqC_tokens,
960
+ block_size=block_size)
961
+ scheduler.add_seq_group(seqC_group)
962
+ metas, out = schedule_and_update_computed_tokens(scheduler)
963
+ assert len(out.scheduled_seq_groups) == 3
964
+
965
+ metas = {meta.request_id: meta for meta in metas}
966
+ assert metas[seqA_group.request_id].token_chunk_size == 1 # Decode
967
+ assert (metas[seqB_group.request_id].token_chunk_size == 8
968
+ ) # Fully cached prefill
969
+ assert (
970
+ metas[seqC_group.request_id].token_chunk_size == 6
971
+ ), "A partial prefix of C (4 tokens) should be prefilled, with the "
972
+ "remaining tokens fit into 3 token budget (4-1 from the seqA). It will "
973
+ "then be rounded down to 2 tokens on block size, thus 6 tokens in total."
tests/core/utils.py
+46
-5
@@ -1,17 +1,20 @@
1
import time
2
-from typing import List, Optional
2
+from collections import defaultdict
3
+from typing import Any, Dict, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Tuple
6
7
from vllm import SamplingParams
8
+from vllm.core.scheduler import Scheduler, SchedulerOutputs
9
from vllm.inputs import EncoderDecoderInputs, token_inputs
10
from vllm.lora.request import LoRARequest
9
-from vllm.sequence import Logprob, Sequence, SequenceGroup
11
+from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12
+ SequenceGroupMetadata)
13
14
15
def create_dummy_prompt(
16
request_id: str,
14
- prompt_length: int,
17
+ prompt_length: int = -1,
18
block_size: Optional[int] = None,
19
lora_request: Optional[LoRARequest] = None,
20
best_of: int = 1,
@@ -26,6 +29,7 @@ def create_dummy_prompt(
29
# Create dummy prompt sequence with tokens 0...block_size-1
30
# and prompt "0 ... block_size".
31
prompt_tokens = list(range(prompt_length))
32
+
33
prompt_str = " ".join([str(t) for t in prompt_tokens])
34
prompt = Sequence(int(request_id),
35
inputs=token_inputs(prompt_tokens, prompt=prompt_str),
@@ -42,6 +46,15 @@ def create_dummy_prompt(
46
return prompt, seq_group
47
48
49
+def create_dummy_sequence(request_id: int, token_ids: List[int],
50
+ block_size: int) -> Sequence:
51
+ return Sequence(
52
+ seq_id=request_id,
53
+ inputs=token_inputs(token_ids),
54
+ block_size=block_size,
55
+ )
56
+
57
+
58
def create_dummy_prompt_encoder_decoder(
59
request_id: str,
60
decoder_prompt_length: int,
@@ -194,12 +207,40 @@ def append_new_token(out, token_id: int):
207
208
def schedule_and_update_computed_tokens(scheduler):
209
metas, out, _ = scheduler.schedule()
197
- for s, meta in zip(out.scheduled_seq_groups, metas):
198
- s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
210
+ for s in out.scheduled_seq_groups:
211
+ s.seq_group.update_num_computed_tokens(s.token_chunk_size)
212
return metas, out
213
214
215
+def append_new_token_seq(seq: Sequence, token_id: int):
216
+ seq.append_token_id(token_id, {token_id: Logprob(token_id)})
217
+
218
+
219
def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
220
seq_group.update_num_computed_tokens(token_chunk_size)
221
for seq in seq_group.get_seqs():
222
seq.append_token_id(token_id, {token_id: Logprob(token_id)})
223
+
224
+
225
+class SchedulerProxy:
226
+ """
227
+ A proxy class to forward calls to the scheduler.
228
+ """
229
+
230
+ def __init__(self, scheduler: Scheduler):
231
+ self.scheduler_ = scheduler
232
+ self.call_history: Dict[str, List[Any]] = defaultdict(list)
233
+
234
+ def __getattr__(self, name: str) -> Any:
235
+
236
+ def wrapper(*args, **kwargs):
237
+ result = getattr(self.scheduler_, name)(*args, **kwargs)
238
+ self.call_history[name].append((args, kwargs, result))
239
+ return result
240
+
241
+ return wrapper
242
+
243
+ def last_schedule_ret(
244
+ self, ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, Any]:
245
+ _, _, ret = self.call_history["schedule"][-1]
246
+ return ret
tests/prefix_caching/test_prefix_caching.py
+100
-6
@@ -2,10 +2,15 @@
2
3
Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
+
6
import pytest
7
8
+from tests.conftest import VllmRunner
9
+from tests.core.utils import SchedulerProxy, create_dummy_prompt
10
from tests.kernels.utils import override_backend_env_variable
11
from vllm import SamplingParams, TokensPrompt
12
+from vllm.core.scheduler import Scheduler
13
+from vllm.engine.llm_engine import LLMEngine
14
15
from ..models.utils import check_outputs_equal
16
@@ -27,6 +32,7 @@ UNSTABLE_PROMPT_SEQUENCE = [
32
@pytest.mark.parametrize("dtype", ["half"])
33
@pytest.mark.parametrize("max_tokens", [5])
34
@pytest.mark.parametrize("cached_position", [0, 1])
35
+@pytest.mark.parametrize("enable_chunked_prefill", [True, False])
36
@pytest.mark.parametrize("block_size", [16])
37
def test_mixed_requests(
38
hf_runner,
@@ -37,6 +43,7 @@ def test_mixed_requests(
43
dtype: str,
44
max_tokens: int,
45
cached_position: int,
46
+ enable_chunked_prefill: bool,
47
block_size: int,
48
monkeypatch,
49
) -> None:
@@ -55,6 +62,7 @@ def test_mixed_requests(
62
model,
63
dtype=dtype,
64
enable_prefix_caching=True,
65
+ enable_chunked_prefill=enable_chunked_prefill,
66
block_size=block_size,
67
) as vllm_model:
68
# Run the first prompt so the cache is populated
@@ -72,13 +80,13 @@ def test_mixed_requests(
80
block_size) * block_size
81
else:
82
expected_num_cached_tokens = 0
75
- assert req_outputs[
76
- i].num_cached_tokens == expected_num_cached_tokens
83
+ assert (
84
+ req_outputs[i].num_cached_tokens == expected_num_cached_tokens)
85
78
- vllm_outputs = [
79
- (output.prompt_token_ids + list(output.outputs[0].token_ids),
80
- output.prompt + output.outputs[0].text) for output in req_outputs
81
- ]
86
+ vllm_outputs = [(
87
+ output.prompt_token_ids + list(output.outputs[0].token_ids),
88
+ output.prompt + output.outputs[0].text,
89
+ ) for output in req_outputs]
90
91
check_outputs_equal(
92
outputs_0_lst=hf_outputs,
@@ -105,3 +113,89 @@ def test_unstable_prompt_sequence(
113
for prompt in UNSTABLE_PROMPT_SEQUENCE:
114
vllm_model.generate(TokensPrompt(prompt_token_ids=prompt),
115
SamplingParams(max_tokens=1))
116
+
117
+
118
+@pytest.mark.parametrize("model", MODELS)
119
+def test_fully_cached_prefill_needs_uncached_token(model):
120
+ block_size = 16
121
+ max_num_batched_tokens = 16
122
+ num_output_tokens = 5
123
+ # Make a vllm engine
124
+ runner = VllmRunner(
125
+ model_name=model,
126
+ gpu_memory_utilization=0.7,
127
+ enable_chunked_prefill=True,
128
+ enforce_eager=True,
129
+ enable_prefix_caching=True,
130
+ block_size=block_size,
131
+ max_num_batched_tokens=max_num_batched_tokens,
132
+ max_num_seqs=max_num_batched_tokens,
133
+ )
134
+ engine: LLMEngine = runner.model.llm_engine
135
+
136
+ scheduler: Scheduler = SchedulerProxy(engine.scheduler[0]) # type: ignore
137
+ engine.scheduler[0] = scheduler
138
+
139
+ # SeqA
140
+ seqA_tokens = list(range(2 * block_size))
141
+ seqA, seq_groupA = create_dummy_prompt(
142
+ request_id="0",
143
+ prompt_tokens=seqA_tokens,
144
+ max_tokens=num_output_tokens,
145
+ block_size=block_size,
146
+ )
147
+
148
+ scheduler.add_seq_group(seq_groupA)
149
+
150
+ assert seqA.data.get_num_computed_tokens() == 0
151
+
152
+ # Prefill seqA
153
+ while not seqA.is_finished():
154
+ engine.step()
155
+
156
+ # seqB
157
+ seqB_tokens = [t + 1 for t in seqA_tokens] # shift by 1
158
+ seqB, seq_groupB = create_dummy_prompt(
159
+ request_id="1",
160
+ prompt_tokens=seqB_tokens,
161
+ max_tokens=num_output_tokens,
162
+ block_size=block_size,
163
+ )
164
+
165
+ # seqC is the same as seqA
166
+ seqC, seq_groupC = create_dummy_prompt(
167
+ request_id="2",
168
+ prompt_tokens=seqA_tokens,
169
+ max_tokens=num_output_tokens,
170
+ block_size=block_size,
171
+ )
172
+
173
+ scheduler.add_seq_group(seq_groupB)
174
+ scheduler.add_seq_group(seq_groupC)
175
+
176
+ # Even seqC is fully cached, it should not be prefilled since we
177
+ # require at least 1 uncached token.
178
+ engine.step()
179
+
180
+ sched_metas, sched_out, _ = scheduler.last_schedule_ret()
181
+ assert len(sched_out.scheduled_seq_groups) == 1
182
+ assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
183
+ seq_groupB.request_id)
184
+ assert (sched_out.scheduled_seq_groups[0].token_chunk_size ==
185
+ max_num_batched_tokens)
186
+
187
+ # When seqB is finished, seqC could be prefilled.
188
+ while not seqB.is_finished():
189
+ engine.step()
190
+ sched_metas, sched_out, _ = scheduler.last_schedule_ret()
191
+ assert len(sched_out.scheduled_seq_groups) == 1
192
+ assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
193
+ seq_groupB.request_id)
194
+
195
+ engine.step()
196
+ sched_metas, sched_out, _ = scheduler.last_schedule_ret()
197
+ assert len(sched_out.scheduled_seq_groups) == 1
198
+ assert (sched_out.scheduled_seq_groups[0].seq_group.request_id ==
199
+ seq_groupC.request_id)
200
+ assert sched_out.scheduled_seq_groups[0].token_chunk_size == len(
201
+ seqA_tokens)
vllm/core/block/cpu_gpu_block_allocator.py
+7
-8
@@ -306,14 +306,6 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
306
device = Device.GPU
307
return self._allocators[device].mark_blocks_as_computed(block_ids)
308
309
- def get_computed_block_ids(self, prev_computed_block_ids: List[int],
310
- block_ids: List[int],
311
- skip_last_block_id: bool) -> List[int]:
312
- # Prefix caching only supported on GPU.
313
- device = Device.GPU
314
- return self._allocators[device].get_computed_block_ids(
315
- prev_computed_block_ids, block_ids, skip_last_block_id)
316
-
309
def get_common_computed_block_ids(
310
self, computed_seq_block_ids: List[List[int]]) -> List[int]:
311
# Prefix caching only supported on GPU.
@@ -342,6 +334,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
334
self._swap_mapping.clear()
335
return list(mapping.items())
336
337
+ def find_cached_blocks_prefix(
338
+ self,
339
+ block_hashes: List[int],
340
+ device: Device = Device.GPU,
341
+ ) -> List[int]:
342
+ return self._allocators[device].find_cached_blocks_prefix(block_hashes)
343
+
344
345
class NullBlock(Block):
346
"""
vllm/core/block/interfaces.py
+21
-15
@@ -159,12 +159,6 @@ class BlockAllocator(ABC):
159
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
160
pass
161
162
- @abstractmethod
163
- def get_computed_block_ids(self, prev_computed_block_ids: List[int],
164
- block_ids: List[int],
165
- skip_last_block_id: bool) -> List[int]:
166
- pass
167
-
162
@abstractmethod
163
def get_common_computed_block_ids(
164
self, computed_seq_block_ids: List[List[int]]) -> List[int]:
@@ -192,6 +186,13 @@ class BlockAllocator(ABC):
186
class NoFreeBlocksError(ValueError):
187
pass
188
189
+ @abstractmethod
190
+ def find_cached_blocks_prefix(
191
+ self,
192
+ block_hashes: List[int],
193
+ ) -> List[int]:
194
+ pass
195
+
196
197
class DeviceAwareBlockAllocator(ABC):
198
@@ -207,9 +208,12 @@ class DeviceAwareBlockAllocator(ABC):
208
pass
209
210
@abstractmethod
210
- def allocate_immutable_blocks(self, prev_block: Optional[Block],
211
- block_token_ids: List[List[int]],
212
- device: Device) -> List[Block]:
211
+ def allocate_immutable_blocks(
212
+ self,
213
+ prev_block: Optional[Block],
214
+ block_token_ids: List[List[int]],
215
+ device: Device,
216
+ ) -> List[Block]:
217
pass
218
219
@abstractmethod
@@ -246,12 +250,6 @@ class DeviceAwareBlockAllocator(ABC):
250
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
251
pass
252
249
- @abstractmethod
250
- def get_computed_block_ids(self, prev_computed_block_ids: List[int],
251
- block_ids: List[int],
252
- skip_last_block_id: bool) -> List[int]:
253
- pass
254
-
253
@abstractmethod
254
def get_common_computed_block_ids(
255
self, computed_seq_block_ids: List[List[int]]) -> List[int]:
@@ -284,3 +282,11 @@ class DeviceAwareBlockAllocator(ABC):
282
def get_prefix_cache_hit_rate(self, device: Device) -> float:
283
"""Prefix cache hit rate. -1 means not supported or disabled."""
284
pass
285
+
286
+ @abstractmethod
287
+ def find_cached_blocks_prefix(
288
+ self,
289
+ block_hashes: List[int],
290
+ device: Device = Device.GPU,
291
+ ) -> List[int]:
292
+ pass
vllm/core/block/naive_block.py
+4
-7
@@ -262,13 +262,6 @@ class NaiveBlockAllocator(BlockAllocator):
262
"""
263
pass
264
265
- def get_computed_block_ids(self, prev_computed_block_ids: List[int],
266
- block_ids: List[int],
267
- skip_last_block_id: bool) -> List[int]:
268
- """No prefix caching here => return empty list
269
- """
270
- return []
271
-
265
def get_common_computed_block_ids(
266
self, computed_seq_block_ids: List[List[int]]) -> List[int]:
267
"""Determine blocks that can be skipped in prefill.
@@ -329,6 +322,10 @@ class NaiveBlockAllocator(BlockAllocator):
322
def get_prefix_cache_hit_rate(self) -> float:
323
return -1
324
325
+ def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
326
+ # Not applicable for naive block allocator.
327
+ return []
328
+
329
330
class NaiveBlock(Block):
331
"""An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+162
-96
@@ -1,13 +1,18 @@
1
"""Token blocks."""
2
+import sys
3
+from bisect import bisect_left
4
from os.path import commonprefix
3
-from typing import Dict, FrozenSet, Iterable, List, Optional, Set, Tuple
5
+from typing import (Callable, Dict, FrozenSet, Iterable, List, Optional, Set,
6
+ Tuple)
7
8
from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
9
get_all_blocks_recursively)
7
-from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10
+from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId, Device,
11
+ DeviceAwareBlockAllocator)
12
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
13
NaiveBlockAllocator)
14
from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
15
+from vllm.sequence import Sequence
16
17
PrefixHash = int
18
@@ -534,26 +539,6 @@ class PrefixCachingBlockAllocator(BlockAllocator):
539
else:
540
return block_id in self.evictor
541
537
- def get_computed_block_ids(self,
538
- prev_computed_block_ids: List[int],
539
- block_ids: List[int],
540
- skip_last_block_id: bool = True) -> List[int]:
541
- prev_prefix_size = len(prev_computed_block_ids)
542
- cur_size = len(block_ids)
543
- if skip_last_block_id:
544
- cur_size -= 1
545
-
546
- # Sanity checks
547
- assert cur_size >= 0
548
- assert prev_prefix_size <= cur_size
549
-
550
- ret = prev_computed_block_ids
551
- for i in range(prev_prefix_size, cur_size):
552
- block_id = block_ids[i]
553
- if self.block_is_computed(block_id):
554
- ret.append(block_id)
555
- return ret
556
-
542
def get_common_computed_block_ids(
543
self, computed_seq_block_ids: List[List[int]]) -> List[int]:
544
"""Return the block ids that are common for a given sequence group.
@@ -634,6 +619,47 @@ class PrefixCachingBlockAllocator(BlockAllocator):
619
620
block.block_id = block_id # Assign block_id
621
622
+ def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
623
+ """
624
+ Given a list of block hashes, return the prefix of the block hashes that
625
+ are all cached.
626
+
627
+ Since a block's block hash includes the hashes of all previous blocks,
628
+ and we only allocate/deallocate blocks in the entire sequence, so if a
629
+ block is cached, then all previous blocks are also cached. With this
630
+ property, we can use binary search to find the prefix of cached blocks.
631
+
632
+ Args:
633
+ block_hashes (List[int]): The list of block hashes.
634
+
635
+ Returns:
636
+ List[int]: The prefix of the `block_hashes` that are cached.
637
+ """
638
+
639
+ def _block_is_cached(block_hash: PrefixHash) -> bool:
640
+ if block_hash not in self._cached_blocks:
641
+ return False
642
+
643
+ cached_block_id = self._cached_blocks[block_hash]
644
+ # We only consider the blocks that are marked as computed.
645
+ return self.block_is_computed(cached_block_id)
646
+
647
+ def _bisect_left(a, x, key: Callable[[PrefixHash], bool]) -> int:
648
+
649
+ # python <= 3.10 don't have the key argument
650
+ if sys.version_info < (3, 10):
651
+ a = [key(e) for e in a]
652
+ return bisect_left(a, x)
653
+ else:
654
+ return bisect_left(a, x, key=key)
655
+
656
+ # Look for the first block that's not cached, and returns the prefix
657
+ # i.e. blocks that are cached.
658
+ idx = _bisect_left(block_hashes,
659
+ True,
660
+ key=lambda x: not _block_is_cached(x))
661
+ return block_hashes[:idx]
662
+
663
664
class PrefixCachingBlock(Block):
665
"""A block implementation that supports prefix caching.
@@ -843,86 +869,126 @@ class PrefixCachingBlock(Block):
869
870
871
class ComputedBlocksTracker:
846
- """Handles caching of per-sequence computed block ids.
847
- When a sequence appears for the first time, it traverses all of the
848
- blocks and detects the prefix of blocks that is computed. On the
849
- subsequent times, it only traverses the new blocks that were added
850
- and updates the already recorded prefix of blocks with the newly
851
- computed blocks.
852
-
853
- To avoid redundant traversals, the algorithm also detects when there
854
- is a "gap" in the computed prefix. For example, if we have blocks =
855
- [1,2,3,4,5], and we have detected [1,2,3] as the computed prefix, then
856
- we won't try to add more computed blocks to [1,2,3] in this sequence
857
- iteration, and will add more computed blocks only after the sequence is
858
- freed and reused again.
859
-
860
- Note that currently, for a given sequence, we also skip the last
861
- block id for caching purposes, to avoid caching of a full sequence
872
"""
873
+ Tracks the computed blocks for each sequence.
874
864
- def __init__(self, allocator):
865
- self._allocator = allocator
866
- self._cached_computed_seq_blocks: Dict[int, Tuple[List[int],
867
- bool]] = {}
875
+ Internally, it maintains a map from sequence id to the list of block hashes
876
+ for the sequence. We cache the hashes of the full blocks for each sequence,
877
+ and make sure the hash is calculated in the same way as the allocator.
878
+ When a sequence is being decoded, we also update the sequence's hash
879
+ accordingly and incrementally.
880
869
- def add_seq(self, seq_id: int) -> None:
870
- """Start tracking seq_id
871
- """
872
- assert seq_id not in self._cached_computed_seq_blocks
873
- self._cached_computed_seq_blocks[seq_id] = ([], False)
874
-
875
- def remove_seq(self, seq_id: int) -> None:
876
- """Stop tracking seq_id
877
- """
878
- assert seq_id in self._cached_computed_seq_blocks
879
- del self._cached_computed_seq_blocks[seq_id]
880
-
881
- def get_cached_computed_blocks_and_update(
882
- self, seq_id: int, block_ids: List[int]) -> List[int]:
883
- """ Look at the class documentation for details
884
- """
885
- # Ensure seq_id is already tracked
886
- assert seq_id in self._cached_computed_seq_blocks
887
-
888
- # Get cached data (may be empty on the first time)
889
- prev_computed_block_ids, has_gap = self._cached_computed_seq_blocks[
890
- seq_id]
891
-
892
- if has_gap:
893
- # When gap is detected, we do not add more computed blocks at this
894
- # sequence iteration
895
- return prev_computed_block_ids
896
-
897
- # We do not consider the last block id for caching purposes.
898
- num_cur_blocks = len(block_ids) - 1
899
- assert num_cur_blocks >= 0
900
-
901
- if len(prev_computed_block_ids) >= num_cur_blocks:
902
- # Cache HIT
903
- assert len(prev_computed_block_ids) == num_cur_blocks
904
- return prev_computed_block_ids
905
-
906
- # If here, then we may possibly add more computed blocks. As a result,
907
- # traverse the additional blocks after prev_computed_block_ids to
908
- # detect more computed blocks and add them.
909
-
910
- # Incremental init for seq_id => Look only at the new blocks
911
- computed_block_ids = self._allocator.get_computed_block_ids( # noqa: E501
912
- prev_computed_block_ids,
913
- block_ids,
914
- skip_last_block_id=
915
- True, # We skip last block id to avoid caching of full seq
916
- )
881
+ From the sequence hash, with prefix caching enabled, we could also calculate
882
+ the number of cached tokens for the sequence by looking up the number of
883
+ cached block hashes in the allocator.
884
+ """
885
918
- # Detect if there is a "gap"
919
- has_gap = len(computed_block_ids) < num_cur_blocks
886
+ def __init__(
887
+ self,
888
+ allocator: DeviceAwareBlockAllocator,
889
+ block_size: int,
890
+ enable_caching: bool,
891
+ ):
892
+ self._allocator = allocator
893
+ self._block_size = block_size
894
+ self._enable_caching = enable_caching
895
+
896
+ # A map from seq_id to the list of block hashes for the
897
+ # sequence. This is so that we don't have to recompute the block hashes
898
+ # for the sequence when we need to check if the sequence is cached.
899
+ # Note a block that's not full will not have its hash calculated and
900
+ # recorded.
901
+ self._seq_id_to_blocks_hashes: Dict[int, List[int]] = {}
902
+
903
+ # A map from seq_id to the number of tokens that are cached for the
904
+ # sequence.
905
+ # We need this so that a sequence in continuous prefill doesn't
906
+ # accidentally see its cached token count change. See comments in
907
+ # `get_num_cached_tokens` for more details.
908
+ self._seq_id_to_num_tokens_computed: Dict[int, int] = {}
909
+
910
+ def _update_seq_hashes(self, seq: Sequence) -> None:
911
+ """Incrementally update the sequence's block hashes and record them."""
912
+ assert self._enable_caching
913
+
914
+ block_hashes_recorded = self._seq_id_to_blocks_hashes.get(
915
+ seq.seq_id, [])
916
+ cur_num_blocks_recorded = len(block_hashes_recorded)
917
+ token_ids = seq.get_token_ids()
918
+ assert len(token_ids) >= cur_num_blocks_recorded * self._block_size, (
919
+ f"The sequence has {len(token_ids)} tokens, but"
920
+ f" already recorded {cur_num_blocks_recorded} blocks. "
921
+ "This should not happen since we assume blocks are "
922
+ "only appended other than recomputation. When the sequence is "
923
+ "recomputed, we should have removed the info of the old blocks.")
924
+ # Update the computed block hashes for the sequence. Since only full
925
+ # blocks are considered as "computed", we take floor here.
926
+ num_computed_blocks = len(token_ids) // self._block_size
927
+
928
+ # We need to know the hash of the previous block to compute the hash of
929
+ # the current block so that blocks could be uniquely identified across
930
+ # sequences of prefixes.
931
+ prev_block_hash = (None if cur_num_blocks_recorded == 0 else
932
+ block_hashes_recorded[-1])
933
+ # Only update the computed block hashes for the new blocks
934
+ for i in range(cur_num_blocks_recorded, num_computed_blocks):
935
+ assert len(token_ids) >= (i + 1) * self._block_size
936
+ block_token_ids = token_ids[i * self._block_size:(i + 1) *
937
+ self._block_size]
938
+ # This has to be kept in sync with the allocator's hash
939
+ # calculation.
940
+ block_hash = PrefixCachingBlock.hash_block_tokens(
941
+ is_first_block=prev_block_hash is None,
942
+ prev_block_hash=prev_block_hash,
943
+ cur_block_token_ids=block_token_ids,
944
+ )
945
+ block_hashes_recorded.append(block_hash)
946
+ prev_block_hash = block_hash
947
+
948
+ self._seq_id_to_blocks_hashes[seq.seq_id] = block_hashes_recorded
949
+
950
+ def get_num_cached_tokens(self, seq: Sequence) -> int:
951
+ if not self._enable_caching:
952
+ return 0
953
+
954
+ # We always try to update the sequence hashes on the fly.
955
+ # This is to ensure that we don't miss any cached tokens for the
956
+ # sequence during decode.
957
+ # This routine should only update hash for any new blocks too.
958
+ self._update_seq_hashes(seq)
959
+
960
+ num_computed_tokens_prev = self._seq_id_to_num_tokens_computed.get(
961
+ seq.seq_id, None)
962
+
963
+ # TODO(rickyx): This hack could be removed once we mark blocks as
964
+ # computed correctly with chunked prefills.
965
+ if num_computed_tokens_prev is not None and seq.is_prefill():
966
+ # For a sequence that is still in prefill, we don't
967
+ # recompute the number of cached tokens.
968
+ # This also handles correctly chunked prefill since currently
969
+ # we mark blocks as computed even if the sequence is still partially
970
+ # prefilled. So a continuously prefilled sequence should not
971
+ # see its cached token count change while running.
972
+ return num_computed_tokens_prev
973
+
974
+ block_hashes = self._seq_id_to_blocks_hashes[seq.seq_id]
975
+
976
+ # This is O(logN), where N is the number of blocks.
977
+ num_cached_blocks = len(
978
+ self._allocator.find_cached_blocks_prefix(block_hashes))
979
+ num_cached_tokens = num_cached_blocks * self._block_size
980
+ self._seq_id_to_num_tokens_computed[seq.seq_id] = num_cached_tokens
981
+ return num_cached_tokens
982
921
- # Record
922
- self._cached_computed_seq_blocks[seq_id] = (computed_block_ids,
923
- has_gap)
983
+ def remove_seq(self, seq_id: int) -> None:
984
+ """Stop tracking the sequence."""
985
+ if not self._enable_caching:
986
+ return
987
+ assert seq_id in self._seq_id_to_blocks_hashes
988
+ del self._seq_id_to_blocks_hashes[seq_id]
989
925
- return computed_block_ids
990
+ assert seq_id in self._seq_id_to_num_tokens_computed
991
+ del self._seq_id_to_num_tokens_computed[seq_id]
992
993
994
class LastAccessBlocksTracker:
vllm/core/block_manager.py
+14
-9
@@ -101,7 +101,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
101
self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
102
103
self._computed_blocks_tracker = ComputedBlocksTracker(
104
- self.block_allocator)
104
+ self.block_allocator, self.block_size, self.enable_caching)
105
self._last_access_blocks_tracker = LastAccessBlocksTracker(
106
self.block_allocator)
107
@@ -170,7 +170,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
170
self.block_tables[seq.seq_id] = block_table
171
172
# Track seq
173
- self._computed_blocks_tracker.add_seq(seq.seq_id)
173
self._last_access_blocks_tracker.add_seq(seq.seq_id)
174
175
# Assign the block table for each sequence.
@@ -178,7 +177,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
177
self.block_tables[seq.seq_id] = block_table.fork()
178
179
# Track seq
181
- self._computed_blocks_tracker.add_seq(seq.seq_id)
180
self._last_access_blocks_tracker.add_seq(seq.seq_id)
181
182
# Allocate cross-attention block table for encoder sequence
@@ -314,11 +312,13 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
312
"""
313
computed_seq_block_ids = []
314
for seq in seqs:
317
- computed_seq_block_ids.append(
318
- self._computed_blocks_tracker.
319
- get_cached_computed_blocks_and_update(
320
- seq.seq_id,
321
- self.block_tables[seq.seq_id].physical_block_ids))
315
+ all_blocks = self.block_tables[seq.seq_id].physical_block_ids
316
+ num_cached_tokens = (
317
+ self._computed_blocks_tracker.get_num_cached_tokens(seq))
318
+ assert num_cached_tokens % self.block_size == 0
319
+ num_cached_blocks = num_cached_tokens // self.block_size
320
+ computed_block_ids = all_blocks[:num_cached_blocks]
321
+ computed_seq_block_ids.append(computed_block_ids)
322
323
# NOTE(sang): This assumes seq_block_ids doesn't contain any None.
324
return self.block_allocator.get_common_computed_block_ids(
@@ -332,7 +332,6 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
332
self.block_tables[child_seq.seq_id] = src_block_table.fork()
333
334
# Track child seq
335
- self._computed_blocks_tracker.add_seq(child_seq.seq_id)
335
self._last_access_blocks_tracker.add_seq(child_seq.seq_id)
336
337
def can_swap_in(self, seq_group: SequenceGroup,
@@ -503,3 +502,9 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
502
return AllocStatus.OK
503
else:
504
return AllocStatus.LATER
505
+
506
+ def get_num_cached_tokens(self, seq: Sequence) -> int:
507
+ """Get the number of tokens in blocks that are already computed and
508
+ cached in the block manager for the sequence.
509
+ """
510
+ return self._computed_blocks_tracker.get_num_cached_tokens(seq)
vllm/core/interfaces.py
+4
@@ -121,3 +121,7 @@ class BlockSpaceManager(ABC):
121
def get_prefix_cache_hit_rate(self, device: Device) -> float:
122
"""Prefix cache hit rate. -1 means not supported or disabled."""
123
pass
124
+
125
+ @abstractmethod
126
+ def get_num_cached_tokens(self, seq: Sequence) -> int:
127
+ pass
vllm/core/placeholder_block_space_manager.py
+3
@@ -89,3 +89,6 @@ class PlaceholderBlockSpaceManager(BlockSpaceManager):
89
90
def get_prefix_cache_hit_rate(self, device: Device) -> float:
91
return -1
92
+
93
+ def get_num_cached_tokens(self, seq: Sequence) -> int:
94
+ return 0
vllm/core/scheduler.py
+253
-85
@@ -56,11 +56,16 @@ class SchedulingBudget:
56
max_num_seqs: int
57
_request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
58
_request_ids_num_curr_seqs: Set[str] = field(default_factory=set)
59
+ # Number of cached tokens in the batch.
60
+ _num_cached_tokens: int = 0
61
+ # Number of actual non-cached tokens in the batch.
62
_num_batched_tokens: int = 0
63
_num_curr_seqs: int = 0
64
65
def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
63
- assert num_new_tokens != 0
66
+ # We allow num_new_tokens to be 0 when the entire sequence has
67
+ # been cached.
68
+ assert num_new_tokens >= 0
69
assert num_new_seqs != 0
70
return (self.num_batched_tokens + num_new_tokens <= self.token_budget
71
and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
@@ -68,12 +73,18 @@ class SchedulingBudget:
73
def remaining_token_budget(self):
74
return self.token_budget - self.num_batched_tokens
75
71
- def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
76
+ def add_num_batched_tokens(self,
77
+ req_id: str,
78
+ num_batched_tokens: int,
79
+ num_cached_tokens: int = 0):
80
if req_id in self._request_ids_num_batched_tokens:
81
return
82
+ assert num_cached_tokens >= 0
83
+ assert num_batched_tokens >= 0
84
85
self._request_ids_num_batched_tokens.add(req_id)
86
self._num_batched_tokens += num_batched_tokens
87
+ self._num_cached_tokens += num_cached_tokens
88
89
def subtract_num_batched_tokens(self, req_id: str,
90
num_batched_tokens: int):
@@ -101,6 +112,10 @@ class SchedulingBudget:
112
def num_curr_seqs(self):
113
return self._num_curr_seqs
114
115
+ @property
116
+ def num_cached_tokens(self):
117
+ return self._num_cached_tokens
118
+
119
120
@dataclass
121
class ScheduledSequenceGroup:
@@ -541,9 +556,19 @@ class Scheduler:
556
assert len(self._async_stopped) == 0
557
while running_queue:
558
seq_group = running_queue[0]
544
- num_running_tokens = self._get_num_new_tokens(
545
- seq_group, SequenceStatus.RUNNING, enable_chunking, budget)
546
-
559
+ # We discard the cached tokens info here because we don't need it
560
+ # for running sequence:
561
+ # 1. If a sequence is running with chunked prefill, the cached
562
+ # tokens info was already used for the first prefill.
563
+ # 2. If a sequence is running with non-chunked prefill, then
564
+ # there it's a decoding sequence, and the cached tokens info is
565
+ # irrelevant.
566
+ num_uncached_new_tokens, _ = (
567
+ self._get_num_new_uncached_and_cached_tokens(
568
+ seq_group, SequenceStatus.RUNNING, enable_chunking,
569
+ budget))
570
+
571
+ num_running_tokens = num_uncached_new_tokens
572
if num_running_tokens == 0:
573
# No budget => Stop
574
break
@@ -715,13 +740,15 @@ class Scheduler:
740
# The total number of sequences in the RUNNING state should not
741
# exceed the maximum number of sequences.
742
num_new_seqs = seq_group.get_max_num_running_seqs()
718
- num_new_tokens = self._get_num_new_tokens(seq_group,
719
- SequenceStatus.SWAPPED,
720
- enable_chunking, budget)
721
-
722
- if (num_new_tokens == 0
723
- or not budget.can_schedule(num_new_tokens=num_new_tokens,
724
- num_new_seqs=num_new_seqs)):
743
+ num_new_tokens_uncached, num_new_tokens_cached = (
744
+ self._get_num_new_uncached_and_cached_tokens(
745
+ seq_group, SequenceStatus.SWAPPED, enable_chunking,
746
+ budget))
747
+
748
+ if num_new_tokens_uncached == 0 or not budget.can_schedule(
749
+ num_new_tokens=num_new_tokens_uncached,
750
+ num_new_seqs=num_new_seqs,
751
+ ):
752
break
753
754
if lora_int_id > 0 and curr_loras is not None:
@@ -732,12 +759,19 @@ class Scheduler:
759
is_prefill = seq_group.is_prefill()
760
if is_prefill:
761
prefill_seq_groups.append(
735
- ScheduledSequenceGroup(seq_group,
736
- token_chunk_size=num_new_tokens))
762
+ ScheduledSequenceGroup(
763
+ seq_group,
764
+ token_chunk_size=num_new_tokens_uncached +
765
+ num_new_tokens_cached,
766
+ ))
767
else:
768
decode_seq_groups.append(
769
ScheduledSequenceGroup(seq_group, token_chunk_size=1))
740
- budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
770
+ budget.add_num_batched_tokens(
771
+ seq_group.request_id,
772
+ num_batched_tokens=num_new_tokens_uncached,
773
+ num_cached_tokens=num_new_tokens_cached,
774
+ )
775
budget.add_num_seqs(seq_group.request_id, num_new_seqs)
776
777
swapped_queue.extendleft(leftover_swapped)
@@ -803,26 +837,30 @@ class Scheduler:
837
if waiting_queue:
838
seq_group = waiting_queue.popleft()
839
num_new_seqs = seq_group.get_max_num_running_seqs()
806
- num_new_tokens = self._get_num_new_tokens(seq_group,
807
- SequenceStatus.WAITING,
808
- False, budget)
840
+ num_new_tokens_uncached, _ = (
841
+ self._get_num_new_uncached_and_cached_tokens(
842
+ seq_group, SequenceStatus.WAITING, False, budget))
843
844
#Only preempt if priority inversion exists
845
while running_queue and self._get_priority(
846
running_queue[-1]) > self._get_priority(seq_group):
847
#Only preempt if waiting sequence cannot be allocated
848
can_allocate = self.block_manager.can_allocate(seq_group)
815
- if (num_new_tokens and can_allocate == AllocStatus.OK
816
- and budget.can_schedule(num_new_tokens=num_new_tokens,
817
- num_new_seqs=num_new_seqs)):
849
+ if (num_new_tokens_uncached > 0
850
+ and can_allocate == AllocStatus.OK
851
+ and budget.can_schedule(
852
+ num_new_tokens=num_new_tokens_uncached,
853
+ num_new_seqs=num_new_seqs,
854
+ )):
855
break
856
857
#Adjust budget to remove the victim sequence group
858
vseq_group = running_queue.pop()
822
- num_running_tokens = self._get_num_new_tokens(
823
- vseq_group, SequenceStatus.RUNNING, False, budget)
824
- budget.subtract_num_batched_tokens(vseq_group.request_id,
825
- num_running_tokens)
859
+ num_running_tokens_uncached, _ = (
860
+ self._get_num_new_uncached_and_cached_tokens(
861
+ vseq_group, SequenceStatus.RUNNING, False, budget))
862
+ budget.subtract_num_batched_tokens(
863
+ vseq_group.request_id, num_running_tokens_uncached)
864
num_running_seqs = vseq_group.get_max_num_running_seqs()
865
budget.subtract_num_seqs(vseq_group.request_id,
866
num_running_seqs)
@@ -882,9 +920,12 @@ class Scheduler:
920
assert len(waiting_seqs) == 1, (
921
"Waiting sequence group should have only one prompt "
922
"sequence.")
885
- num_new_tokens = self._get_num_new_tokens(seq_group,
886
- SequenceStatus.WAITING,
887
- enable_chunking, budget)
923
+ num_new_tokens_uncached, num_new_tokens_cached = (
924
+ self._get_num_new_uncached_and_cached_tokens(
925
+ seq_group, SequenceStatus.WAITING, enable_chunking,
926
+ budget))
927
+ num_new_tokens = num_new_tokens_uncached + num_new_tokens_cached
928
+
929
if not enable_chunking:
930
num_prompt_tokens = waiting_seqs[0].get_len()
931
assert num_new_tokens == num_prompt_tokens
@@ -935,10 +976,18 @@ class Scheduler:
976
waiting_queue.popleft()
977
continue
978
979
+ if (budget.num_batched_tokens >=
980
+ self.scheduler_config.max_num_batched_tokens):
981
+ # We've reached the budget limit - since there might be
982
+ # continuous prefills in the running queue, we should break
983
+ # to avoid scheduling any new prefills.
984
+ break
985
+
986
num_new_seqs = seq_group.get_max_num_running_seqs()
939
- if (num_new_tokens == 0
940
- or not budget.can_schedule(num_new_tokens=num_new_tokens,
941
- num_new_seqs=num_new_seqs)):
987
+ if num_new_tokens_uncached == 0 or not budget.can_schedule(
988
+ num_new_tokens=num_new_tokens_uncached,
989
+ num_new_seqs=num_new_seqs,
990
+ ):
991
break
992
993
# Can schedule this request.
@@ -967,7 +1016,11 @@ class Scheduler:
1016
seq_groups.append(
1017
ScheduledSequenceGroup(seq_group=seq_group,
1018
token_chunk_size=num_new_tokens))
970
- budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
1019
+ budget.add_num_batched_tokens(
1020
+ seq_group.request_id,
1021
+ num_batched_tokens=num_new_tokens_uncached,
1022
+ num_cached_tokens=num_new_tokens_cached,
1023
+ )
1024
budget.add_num_seqs(seq_group.request_id, num_new_seqs)
1025
1026
# Queue requests that couldn't be scheduled.
@@ -1075,7 +1128,8 @@ class Scheduler:
1128
return SchedulerOutputs(
1129
scheduled_seq_groups=scheduled_seq_groups,
1130
num_prefill_groups=num_prefill_groups,
1078
- num_batched_tokens=budget.num_batched_tokens,
1131
+ num_batched_tokens=budget.num_batched_tokens +
1132
+ budget.num_cached_tokens,
1133
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
1134
blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
1135
blocks_to_copy=blocks_to_copy,
@@ -1119,7 +1173,6 @@ class Scheduler:
1173
running_scheduled.swapped_out) == 0:
1174
swapped_in = self._schedule_swapped(budget, curr_loras)
1175
1122
- # Schedule new prefills.
1176
prefills = self._schedule_prefills(budget,
1177
curr_loras,
1178
enable_chunking=True)
@@ -1157,7 +1210,8 @@ class Scheduler:
1210
num_prefill_groups=(len(prefills.seq_groups) +
1211
len(swapped_in.prefill_seq_groups) +
1212
len(running_scheduled.prefill_seq_groups)),
1160
- num_batched_tokens=budget.num_batched_tokens,
1213
+ num_batched_tokens=budget.num_batched_tokens +
1214
+ budget.num_cached_tokens,
1215
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
1216
blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
1217
blocks_to_copy=running_scheduled.blocks_to_copy +
@@ -1584,64 +1638,178 @@ class Scheduler:
1638
1639
return self.scheduler_config.num_lookahead_slots
1640
1587
- def _get_num_new_tokens(self, seq_group: SequenceGroup,
1588
- status: SequenceStatus, enable_chunking: bool,
1589
- budget: SchedulingBudget) -> int:
1590
- """Get the next new tokens to compute for a given sequence group
1591
- that's in a given `status`.
1641
+ def _get_num_new_uncached_and_cached_tokens(
1642
+ self,
1643
+ seq_group: SequenceGroup,
1644
+ status: SequenceStatus,
1645
+ enable_chunking: bool,
1646
+ budget: SchedulingBudget,
1647
+ ) -> Tuple[int, int]:
1648
+ """
1649
+ Returns the number of new uncached and cached tokens to schedule for a
1650
+ given sequence group that's in a given `status`.
1651
1652
The API could chunk the number of tokens to compute based on `budget`
1653
if `enable_chunking` is True. If a sequence group has multiple
1654
sequences (e.g., running beam search), it means it is in decoding
1655
phase, so chunking doesn't happen.
1656
1598
- Returns 0 if the new token cannot be computed due to token budget.
1657
+ Returns (0, 0) if the new token cannot be computed due to token budget.
1658
+
1659
+ The cached tokens's blocks are already computed, and the attention
1660
+ backend will reuse the cached blocks rather than recomputing them. So
1661
+ the scheduler could schedule these cached tokens "for free".
1662
+
1663
+ Args:
1664
+ seq_group: The sequence group to get the number of new tokens to
1665
+ schedule.
1666
+ status: The status of the sequences to get the number of new tokens
1667
+ to schedule.
1668
+ enable_chunking: Whether to chunk the number of tokens to compute.
1669
+ budget: The budget to chunk the number of tokens to compute.
1670
+
1671
+
1672
+ Returns:
1673
+ A tuple of two ints. The first int is the number of new uncached
1674
+ tokens to schedule. The second int is the number of cached tokens.
1675
+ If no more new tokens can be scheduled, returns (0, 0).
1676
"""
1600
- num_new_tokens = 0
1677
+ num_cached_new_tokens = 0
1678
+ num_uncached_new_tokens = 0
1679
+
1680
seqs = seq_group.get_seqs(status=status)
1681
+ # Compute the number of new uncached and cached tokens for
1682
+ # each sequence.
1683
for seq in seqs:
1603
- num_new_tokens += seq.get_num_new_tokens()
1604
- assert num_new_tokens > 0
1605
- # Chunk if a running request cannot fit in the given budget.
1606
- # If number of seq > 1, it means it is doing beam search
1607
- # in a decode phase. Do not chunk.
1684
+ if not seq.is_prefill():
1685
+ # Decode sequences should always just have 1 uncached token
1686
+ # TODO(rickyx): Actually is this still correct for multi-step?
1687
+ num_uncached_new_tokens += 1
1688
+ continue
1689
+
1690
+ num_computed_tokens_seq = seq.get_num_computed_tokens()
1691
+ all_num_new_tokens_seq = seq.get_len() - num_computed_tokens_seq
1692
+ if not self.cache_config.enable_prefix_caching:
1693
+ # If prefix caching is not enabled, all new tokens are uncached.
1694
+ num_uncached_new_tokens += all_num_new_tokens_seq
1695
+ continue
1696
+
1697
+ # NOTE: the cache token might be currently in a block that's in an
1698
+ # evictor meaning that it's not yet allocated. However, we don't
1699
+ # exclude such tokens in the cache count because it will be
1700
+ # guaranteed to be allocated later if the sequence can be allocated.
1701
+ num_cached_tokens_seq = self.block_manager.get_num_cached_tokens(
1702
+ seq)
1703
+
1704
+ # Sanity check.
1705
+ if num_cached_tokens_seq < num_computed_tokens_seq:
1706
+ # This should only happen with chunked prefill, and
1707
+ # the seq is still in prefill. The `num_cached_tokens_seq`
1708
+ # is the value we calculated on scheduling the first prefill.
1709
+ # For subsequent continuous prefill steps, we cached the
1710
+ # number of cache tokens for the sequence so the cached token
1711
+ # count could be less than the number of computed tokens.
1712
+ # See comments on `ComputedBlocksTracker` for more details.
1713
+ assert (
1714
+ seq.is_prefill() and seq.status == SequenceStatus.RUNNING
1715
+ and self.scheduler_config.chunked_prefill_enabled
1716
+ ), ("Number of cached tokens should not be less than the "
1717
+ "number of computed tokens for a sequence that's still "
1718
+ f"in prefill. But there are {num_cached_tokens_seq} cached "
1719
+ f"tokens and {num_computed_tokens_seq} computed tokens "
1720
+ f"for sequence {seq.seq_id}.")
1721
+
1722
+ num_cached_new_tokens_seq = max(
1723
+ 0, num_cached_tokens_seq - num_computed_tokens_seq)
1724
+ num_uncached_new_tokens_seq = (all_num_new_tokens_seq -
1725
+ num_cached_new_tokens_seq)
1726
+
1727
+ num_uncached_new_tokens += num_uncached_new_tokens_seq
1728
+ num_cached_new_tokens += num_cached_new_tokens_seq
1729
+
1730
+ if num_uncached_new_tokens == 0 and num_cached_new_tokens > 0:
1731
+ # For a fully cached hit sequence, we actually need to recompute the
1732
+ # last token. So we need at least 1 uncached token to schedule.
1733
+ # See ModelRunner._compute_for_prefix_cache_hit for more details.
1734
+ num_uncached_new_tokens = 1
1735
+ num_cached_new_tokens -= 1
1736
+
1737
if enable_chunking and len(seqs) == 1:
1609
- remaining_token_budget = budget.remaining_token_budget()
1610
- if self.scheduler_config.is_multi_step:
1611
- # The current multi-step + chunked prefill capability does
1612
- # not actually support chunking prompts.
1613
- #
1614
- # Therefore, `num_new_tokens` is computed in the same fashion
1615
- # for both multi-step+chunked-prefill &
1616
- # multi-step+chunked-prefill+APC
1617
- #
1618
- # Prompts with more tokens than the current remaining budget
1619
- # are postponed to future scheduler steps
1620
- if num_new_tokens > self._get_prompt_limit(seq_group):
1621
- # If the seq_group is in prompt-stage, pass the
1622
- # num_new_tokens as-is so the caller can ignore
1623
- # the sequence.
1624
- pass
1625
- else:
1626
- num_new_tokens = 0 \
1627
- if num_new_tokens > remaining_token_budget \
1628
- else num_new_tokens
1629
- elif self.cache_config.enable_prefix_caching:
1630
- # When prefix caching is enabled, we always allocate
1631
- # the number of new tokens that is dividable by the block
1632
- # size to avoid partial block matching.
1633
- block_size = self.cache_config.block_size
1634
- remainder = budget.token_budget % block_size
1635
- if remainder != 0:
1636
- raise ValueError("When enabling chunked prefill and "
1637
- "prefix caching, max_num_batched_tokens "
1638
- "(chunk size) must be dividable by "
1639
- "block size, but got chunk_size "
1640
- f"({budget.token_budget}) % block_size "
1641
- f"({block_size}) = {remainder}")
1642
- if remaining_token_budget < num_new_tokens:
1643
- num_new_tokens = (remaining_token_budget //
1644
- block_size) * block_size
1645
- else:
1646
- num_new_tokens = min(num_new_tokens, remaining_token_budget)
1738
+ # Chunk if a running request cannot fit in the given budget.
1739
+ # If number of seq > 1, it means it is doing beam search
1740
+ # in a decode phase. Do not chunk.
1741
+ num_uncached_new_tokens = self._chunk_new_tokens_to_schedule(
1742
+ self.scheduler_config,
1743
+ self.cache_config,
1744
+ budget,
1745
+ self._get_prompt_limit(seq_group),
1746
+ num_uncached_new_tokens,
1747
+ )
1748
+
1749
+ return num_uncached_new_tokens, num_cached_new_tokens
1750
+
1751
+ @staticmethod
1752
+ def _chunk_new_tokens_to_schedule(
1753
+ scheduler_config: SchedulerConfig,
1754
+ cache_config: CacheConfig,
1755
+ budget: SchedulingBudget,
1756
+ prompt_limit: int,
1757
+ num_new_tokens: int,
1758
+ ) -> int:
1759
+ """
1760
+ Chunks the number of new tokens to schedule based on the budget when
1761
+ chunked prefill is enabled.
1762
+
1763
+ Args:
1764
+ scheduler_config: The scheduler config.
1765
+ cache_config: The cache config.
1766
+ budget: The budget to chunk the number of tokens to compute.
1767
+ prompt_limit: The maximum number of tokens allowed in a prompt.
1768
+ num_new_tokens: The number of new tokens to schedule.
1769
+
1770
+ Returns:
1771
+ The number of new tokens to schedule after chunking.
1772
+ """
1773
+ remaining_token_budget = budget.remaining_token_budget()
1774
+ if scheduler_config.is_multi_step:
1775
+ # The current multi-step + chunked prefill capability does
1776
+ # not actually support chunking prompts.
1777
+ #
1778
+ # Therefore, `num_new_tokens` is computed in the same fashion
1779
+ # for both multi-step+chunked-prefill &
1780
+ # multi-step+chunked-prefill+APC
1781
+ #
1782
+ # Prompts with more tokens than the current remaining budget
1783
+ # are postponed to future scheduler steps
1784
+ if num_new_tokens > prompt_limit:
1785
+ # If the seq_group is in prompt-stage, pass the
1786
+ # num_new_tokens as-is so the caller can ignore
1787
+ # the sequence.
1788
+ return num_new_tokens
1789
+
1790
+ return (0 if num_new_tokens > remaining_token_budget else
1791
+ num_new_tokens)
1792
+
1793
+ if cache_config.enable_prefix_caching:
1794
+ # Adjust the remaining token budget to be divisible by the block
1795
+ # size when prefix caching is enabled.
1796
+
1797
+ # When prefix caching is enabled, we always allocate
1798
+ # the number of new tokens that is dividable by the block
1799
+ # size to avoid partial block matching.
1800
+ block_size = cache_config.block_size
1801
+ remainder = budget.token_budget % block_size
1802
+ if remainder != 0:
1803
+ raise ValueError("When enabling chunked prefill and "
1804
+ "prefix caching, max_num_batched_tokens "
1805
+ "(chunk size) must be dividable by "
1806
+ "block size, but got chunk_size "
1807
+ f"({budget.token_budget}) % block_size "
1808
+ f"({block_size}) = {remainder}")
1809
+ # Round down to block size.
1810
+ remaining_token_budget = (remaining_token_budget // block_size *
1811
+ block_size)
1812
+
1813
+ num_new_tokens = min(num_new_tokens, remaining_token_budget)
1814
+
1815
return num_new_tokens
vllm/sequence.py
+3
@@ -579,6 +579,9 @@ class Sequence:
579
return 1
580
return self.data.get_num_uncomputed_tokens()
581
582
+ def get_num_computed_tokens(self) -> int:
583
+ return self.data.get_num_computed_tokens()
584
+
585
def is_prefill(self) -> bool:
586
return self.data.stage == SequenceStage.PREFILL
587