A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Feature] [Spec decode]: Combine chunked prefill with speculative decoding (#9291)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Nov 7, 2024 at 17:15 UTC
9d43afcc538645625ea5fc2bca01d3697dd0595c
17 files changed
+477
-147
tests/spec_decode/e2e/test_compatibility.py
-34
@@ -5,40 +5,6 @@ from vllm import SamplingParams
5
from .conftest import get_output_from_llm_generator
6
7
8
-@pytest.mark.parametrize("common_llm_kwargs", [{
9
- "model": "JackFram/llama-68m",
10
- "speculative_model": "JackFram/llama-68m",
11
- "num_speculative_tokens": 5,
12
-}])
13
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [
14
- {
15
- "enable_chunked_prefill": True,
16
- },
17
-])
18
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
19
-@pytest.mark.parametrize("seed", [1])
20
-def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
21
- """Verify that speculative decoding with chunked prefill fails.
22
- """
23
- output_len = 128
24
- temperature = 0.0
25
-
26
- prompts = [
27
- "Hello, my name is",
28
- ]
29
-
30
- sampling_params = SamplingParams(
31
- max_tokens=output_len,
32
- ignore_eos=True,
33
- temperature=temperature,
34
- )
35
-
36
- with pytest.raises(ValueError,
37
- match="Speculative decoding and chunked prefill"):
38
- get_output_from_llm_generator(test_llm_generator, prompts,
39
- sampling_params)
40
-
41
-
8
@pytest.mark.parametrize("common_llm_kwargs", [{
9
"model": "meta-llama/Llama-2-7b-chat-hf",
10
"speculative_model": "JackFram/llama-68m",
tests/spec_decode/e2e/test_multistep_correctness.py
+102
-3
@@ -62,6 +62,16 @@ from .conftest import (get_output_from_llm_generator,
62
{
63
"speculative_model": "JackFram/llama-68m",
64
"num_speculative_tokens": 5,
65
+ "enable_chunked_prefill": False,
66
+ },
67
+ {
68
+ # Chunked prefill enabled with small value
69
+ # to make sure we get mixed batches.
70
+ "speculative_model": "JackFram/llama-68m",
71
+ "num_speculative_tokens": 5,
72
+ "enable_chunked_prefill": True,
73
+ "max_num_batched_tokens": 4,
74
+ "max_num_seqs": 4
75
},
76
{
77
# Verify the detokenizer assertions in the test work when spec
@@ -141,6 +151,14 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
151
{
152
"speculative_model": "JackFram/llama-68m",
153
"num_speculative_tokens": 5,
154
+ "enable_chunked_prefill": False,
155
+ },
156
+ {
157
+ "speculative_model": "JackFram/llama-68m",
158
+ "num_speculative_tokens": 5,
159
+ "enable_chunked_prefill": True,
160
+ "max_num_batched_tokens": 4,
161
+ "max_num_seqs": 4,
162
},
163
])
164
@pytest.mark.parametrize(
@@ -204,6 +222,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
222
{
223
"speculative_model": "JackFram/llama-68m",
224
"num_speculative_tokens": 5,
225
+ "enable_chunked_prefill": False,
226
+ },
227
+ {
228
+ "speculative_model": "JackFram/llama-68m",
229
+ "num_speculative_tokens": 5,
230
+ "enable_chunked_prefill": True,
231
+ "max_num_batched_tokens": 4,
232
+ "max_num_seqs": 4
233
},
234
])
235
@pytest.mark.parametrize(
@@ -255,6 +281,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
281
{
282
"speculative_model": "JackFram/llama-68m",
283
"num_speculative_tokens": 5,
284
+ "enable_chunked_prefill": False,
285
+ },
286
+ {
287
+ "speculative_model": "JackFram/llama-68m",
288
+ "num_speculative_tokens": 5,
289
+ "enable_chunked_prefill": True,
290
+ "max_num_batched_tokens": 4,
291
+ "max_num_seqs": 4
292
},
293
])
294
@pytest.mark.parametrize("max_output_len", [
@@ -300,6 +334,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
334
{
335
"speculative_model": "JackFram/llama-68m",
336
"num_speculative_tokens": 5,
337
+ "enable_chunked_prefill": False,
338
+ },
339
+ {
340
+ "speculative_model": "JackFram/llama-68m",
341
+ "num_speculative_tokens": 5,
342
+ "enable_chunked_prefill": True,
343
+ "max_num_batched_tokens": 4,
344
+ "max_num_seqs": 4
345
},
346
])
347
@pytest.mark.parametrize("batch_size", [1])
@@ -347,6 +389,14 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
389
{
390
"speculative_model": "JackFram/llama-68m",
391
"num_speculative_tokens": 5,
392
+ "enable_chunked_prefill": False,
393
+ },
394
+ {
395
+ "speculative_model": "JackFram/llama-68m",
396
+ "num_speculative_tokens": 5,
397
+ "enable_chunked_prefill": True,
398
+ "max_num_batched_tokens": 4,
399
+ "max_num_seqs": 4
400
},
401
])
402
@pytest.mark.parametrize("batch_size", [32])
@@ -397,6 +447,14 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
447
{
448
"speculative_model": "JackFram/llama-68m",
449
"num_speculative_tokens": 5,
450
+ "enable_chunked_prefill": False,
451
+ },
452
+ {
453
+ "speculative_model": "JackFram/llama-68m",
454
+ "num_speculative_tokens": 5,
455
+ "enable_chunked_prefill": True,
456
+ "max_num_batched_tokens": 4,
457
+ "max_num_seqs": 4
458
},
459
])
460
@pytest.mark.parametrize(
@@ -454,6 +512,14 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
512
{
513
"speculative_model": "JackFram/llama-68m",
514
"num_speculative_tokens": 5,
515
+ "enable_chunked_prefill": False,
516
+ },
517
+ {
518
+ "speculative_model": "JackFram/llama-68m",
519
+ "num_speculative_tokens": 5,
520
+ "enable_chunked_prefill": True,
521
+ "max_num_batched_tokens": 4,
522
+ "max_num_seqs": 4
523
},
524
])
525
@pytest.mark.parametrize("batch_size", [2])
@@ -503,6 +569,15 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
569
# Artificially limit the draft model max model len; this forces vLLM
570
# to skip speculation once the sequences grow beyond 32-k tokens.
571
"speculative_max_model_len": 32,
572
+ "enable_chunked_prefill": False,
573
+ },
574
+ {
575
+ "speculative_model": "JackFram/llama-68m",
576
+ "num_speculative_tokens": 5,
577
+ "enable_chunked_prefill": True,
578
+ "max_num_batched_tokens": 4,
579
+ "max_num_seqs": 4,
580
+ "speculative_max_model_len": 32,
581
},
582
])
583
@pytest.mark.parametrize("batch_size", [8])
@@ -551,6 +626,15 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,
626
"speculative_model": "JackFram/llama-68m",
627
"num_speculative_tokens": 5,
628
"speculative_disable_by_batch_size": 2,
629
+ "enable_chunked_prefill": False,
630
+ },
631
+ {
632
+ "speculative_model": "JackFram/llama-68m",
633
+ "num_speculative_tokens": 5,
634
+ "speculative_disable_by_batch_size": 2,
635
+ "enable_chunked_prefill": True,
636
+ "max_num_batched_tokens": 4,
637
+ "max_num_seqs": 4,
638
},
639
])
640
@pytest.mark.parametrize("batch_size", [8])
@@ -590,10 +674,17 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,
674
{
675
"speculative_model": "JackFram/llama-68m",
676
"num_speculative_tokens": k,
677
+ "enable_chunked_prefill": False,
678
}
679
# Try a range of common k, as well as large speculation.
680
for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]
596
- ])
681
+ ] + [{
682
+ "speculative_model": "JackFram/llama-68m",
683
+ "num_speculative_tokens": k,
684
+ "enable_chunked_prefill": True,
685
+ "max_num_batched_tokens": 4,
686
+ "max_num_seqs": 4,
687
+ } for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]])
688
@pytest.mark.parametrize("batch_size", [2])
689
@pytest.mark.parametrize(
690
"output_len",
@@ -636,11 +727,19 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
727
{
728
"speculative_model": "JackFram/llama-68m",
729
"num_speculative_tokens": k,
639
- "spec_decoding_acceptance_method": "typical_acceptance_sampler"
730
+ "spec_decoding_acceptance_method": "typical_acceptance_sampler",
731
+ "enable_chunked_prefill": False
732
}
733
# Try a range of common k.
734
for k in [1, 2, 3]
643
- ])
735
+ ] + [{
736
+ "speculative_model": "JackFram/llama-68m",
737
+ "num_speculative_tokens": k,
738
+ "spec_decoding_acceptance_method": "typical_acceptance_sampler",
739
+ "enable_chunked_prefill": True,
740
+ "max_num_batched_tokens": 4,
741
+ "max_num_seqs": 4
742
+ } for k in [1, 2, 3]])
743
@pytest.mark.parametrize("batch_size", [1, 32])
744
@pytest.mark.parametrize(
745
"output_len",
tests/spec_decode/e2e/test_ngram_correctness.py
+35
-1
@@ -50,18 +50,33 @@ from .conftest import run_equality_correctness_test
50
"num_speculative_tokens": 5,
51
"ngram_prompt_lookup_max": 3,
52
},
53
+ {
54
+ "speculative_model": "[ngram]",
55
+ "num_speculative_tokens": 5,
56
+ "ngram_prompt_lookup_max": 3,
57
+ },
58
])
59
@pytest.mark.parametrize("output_len", [
60
256,
61
])
62
@pytest.mark.parametrize("batch_size", [1, 32])
63
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
64
@pytest.mark.parametrize("seed", [1])
65
def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
66
per_test_common_llm_kwargs,
67
baseline_llm_kwargs, test_llm_kwargs,
68
batch_size: int, output_len: int,
63
- seed: int):
69
+ prefill_chunk_size: int, seed: int):
70
"""Verify greedy equality on a tiny model with different batch size."""
71
+ if prefill_chunk_size > 0:
72
+ common_llm_kwargs.update(
73
+ **{
74
+ "enable_chunked_prefill": True,
75
+ "max_num_batched_tokens": prefill_chunk_size,
76
+ "max_num_seqs": prefill_chunk_size
77
+ })
78
+ else:
79
+ common_llm_kwargs["enable_chunked_prefill"] = False
80
run_equality_correctness_test(vllm_runner,
81
common_llm_kwargs,
82
per_test_common_llm_kwargs,
@@ -151,6 +166,16 @@ def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
166
"speculative_model": "[ngram]",
167
"num_speculative_tokens": 5,
168
"ngram_prompt_lookup_max": 3,
169
+ "enable_chunked_prefill": False,
170
+ },
171
+ {
172
+ "speculative_model": "[ngram]",
173
+ "num_speculative_tokens": 5,
174
+ "ngram_prompt_lookup_max": 3,
175
+ "enable_chunked_prefill": True,
176
+ "speculative_disable_mqa_scorer": True,
177
+ "max_num_batched_tokens": 4,
178
+ "max_num_seqs": 4
179
},
180
])
181
@pytest.mark.parametrize(
@@ -251,6 +276,15 @@ def test_ngram_different_k(vllm_runner, common_llm_kwargs,
276
"num_speculative_tokens": 5,
277
"ngram_prompt_lookup_max": 3,
278
"speculative_disable_by_batch_size": 4
279
+ }, {
280
+ "speculative_model": "[ngram]",
281
+ "num_speculative_tokens": 5,
282
+ "ngram_prompt_lookup_max": 3,
283
+ "speculative_disable_by_batch_size": 4,
284
+ "enable_chunked_prefill": True,
285
+ "speculative_disable_mqa_scorer": True,
286
+ "max_num_batched_tokens": 4,
287
+ "max_num_seqs": 4
288
}])
289
@pytest.mark.parametrize("batch_size", [1, 5])
290
@pytest.mark.parametrize(
tests/spec_decode/test_ngram_worker.py
+7
-2
@@ -118,7 +118,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
118
num_gpu_blocks,
119
block_size,
120
final_prompt_lens=final_prompt_lens)
121
-
121
+ for sg in seq_group_metadata_list:
122
+ sg.is_prompt = False
123
proposals = proposer.get_spec_proposals(
124
execute_model_req=ExecuteModelRequest(
125
seq_group_metadata_list=seq_group_metadata_list,
@@ -147,7 +148,7 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
148
def test_ngram_algo_correctness_for_batches_match_all():
149
"""Verify our ngram algo find the right candidate in the prompt
150
150
- For the scenario find candidate in all batchs
151
+ For the scenario find candidate in all batches
152
"""
153
154
block_size = 32
@@ -192,6 +193,10 @@ def test_ngram_algo_correctness_for_batches_match_all():
193
block_size,
194
final_prompt_lens=final_prompt_lens)
195
196
+ # Normally drafter is run on decode requests only; here we check the output
197
+ # of the ngram worker as it is the sole proposer that has no forward.
198
+ for sg in seq_group_metadata_list:
199
+ sg.is_prompt = False
200
proposals = proposer.get_spec_proposals(
201
execute_model_req=ExecuteModelRequest(
202
seq_group_metadata_list=seq_group_metadata_list,
tests/spec_decode/test_scorer.py
+27
-4
@@ -46,12 +46,14 @@ def assert_score_equal(score1: SpeculativeScores,
46
@pytest.mark.parametrize('max_propose_len', [1, 3, 5])
47
@pytest.mark.parametrize('mixed_propose_len', [True])
48
@pytest.mark.parametrize('device', ['cuda'])
49
+@pytest.mark.parametrize('prefill_chunking', [False, True])
50
def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
50
- mixed_propose_len: bool, device: str) -> None:
51
+ mixed_propose_len: bool, device: str,
52
+ prefill_chunking: bool) -> None:
53
"""
54
Compare the batch expansion scorer and mqa scorer return the same score.
55
We test for both queries with the same propose length and different
54
- propose length.
56
+ propose length, as well as mixed prefill-decode batches.
57
"""
58
seed = 0
59
block_size = 32
@@ -67,16 +69,37 @@ def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
69
if not mixed_propose_len:
70
propose_lens = [max_propose_len] * batch_size
71
else:
70
- non_zero_cnt = random.randint(0, batch_size)
72
+ # There must be at least 1 decode request, otherwise
73
+ # we have nothing to score (`_run_no_spec`).
74
+ non_zero_cnt = random.randint(1, batch_size)
75
propose_lens = [max_propose_len
76
] * non_zero_cnt + [0] * (batch_size - non_zero_cnt)
77
random.shuffle(propose_lens)
78
75
- proposals = create_proposal(propose_lens, vocab_size, device)
79
seq_group_metadatalist, _, _ = create_batch(batch_size,
80
max_propose_len,
81
block_size=block_size,
82
num_gpu_blocks=num_gpu_blocks)
83
+
84
+ if mixed_propose_len and prefill_chunking and (n_prefills :=
85
+ batch_size - non_zero_cnt):
86
+ prefill, _, _ = create_batch(n_prefills,
87
+ None,
88
+ prefill_chunk_size=4,
89
+ block_size=block_size,
90
+ num_gpu_blocks=num_gpu_blocks,
91
+ seq_ids=list(
92
+ range(batch_size,
93
+ batch_size + n_prefills)))
94
+ # re-order to guarantee prefill|decode order
95
+ target_group_metadatalist = [
96
+ seq_group_metadatalist[i] for i, p in enumerate(propose_lens)
97
+ if p > 0
98
+ ]
99
+ seq_group_metadatalist = prefill + target_group_metadatalist
100
+ propose_lens = [0] * n_prefills + [p for p in propose_lens if p > 0]
101
+
102
+ proposals = create_proposal(propose_lens, vocab_size, device)
103
requests = ExecuteModelRequest(seq_group_metadatalist,
104
num_lookahead_slots=max_propose_len)
105
tests/spec_decode/test_spec_decode_worker.py
+82
@@ -10,6 +10,7 @@ import torch
10
from vllm.model_executor.layers.sampler import SamplerOutput
11
from vllm.model_executor.utils import set_random_seed
12
from vllm.sequence import ExecuteModelRequest, SequenceOutput
13
+from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
14
from vllm.spec_decode.interfaces import SpeculativeProposals
15
from vllm.spec_decode.metrics import (AsyncMetricsCollector,
16
SpecDecodeWorkerMetrics)
@@ -819,3 +820,84 @@ def test_handle_finished_requests():
820
# and 'request-3' are removed from seq_with_bonus_token_in_last_step.
821
assert worker._seq_with_bonus_token_in_last_step == \
822
{4,5,10}
823
+
824
+
825
+@pytest.mark.parametrize('k', [3])
826
+@pytest.mark.parametrize('batch_size', [2, 32])
827
+@pytest.mark.parametrize("batch_composition",
828
+ ["prefill_only", "decode_only", "mixed"])
829
+@torch.inference_mode()
830
+def test_chunked_prefill_flow(k: int, batch_size: int, batch_composition: str):
831
+ """
832
+ Verify SpecDecodeWorker calls match the expected flow.
833
+ """
834
+ vocab_size = 32_000
835
+ draft_worker = mock_worker(cls=MultiStepWorker)
836
+ target_worker = mock_worker()
837
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
838
+ worker = SpecDecodeWorker(draft_worker,
839
+ target_worker,
840
+ mock_spec_decode_sampler("rejection_sampler"),
841
+ disable_logprobs=False,
842
+ metrics_collector=metrics_collector)
843
+ exception_secret = 'artificial stop'
844
+ worker.scorer = mock_worker(BatchExpansionTop1Scorer)
845
+ worker.scorer.score_proposals.side_effect = ValueError(exception_secret)
846
+
847
+ # Create batch with combination of terminal/non-terminal prefill chunks
848
+ # and decodes (different seq_ids).
849
+ decodes, _, _ = create_batch(batch_size, k)
850
+ # Pre-chunking here, get 'batch_size' chunks.
851
+ prefill, _, _ = create_batch(batch_size,
852
+ k,
853
+ prefill_chunk_size=4,
854
+ seq_ids=list(range(batch_size,
855
+ batch_size * 2)))
856
+
857
+ if batch_composition == "prefill_only":
858
+ n_prefills = batch_size
859
+ elif batch_composition == "decode_only":
860
+ n_prefills = 0
861
+ else:
862
+ n_prefills = random.randint(1, batch_size - 1)
863
+ n_decodes = batch_size - n_prefills
864
+
865
+ prefill = random.sample(prefill, n_prefills)
866
+ decodes = random.sample(decodes, n_decodes)
867
+ target_group_metadata_list = prefill + decodes
868
+ execute_model_req = ExecuteModelRequest(
869
+ seq_group_metadata_list=target_group_metadata_list,
870
+ num_lookahead_slots=k)
871
+
872
+ target_token_ids = torch.randint(low=0,
873
+ high=vocab_size,
874
+ size=(1, batch_size * (k + 1)),
875
+ dtype=torch.int64,
876
+ device='cuda')
877
+ target_token_probs = torch.rand(1,
878
+ batch_size * (k + 1),
879
+ vocab_size,
880
+ dtype=torch.float32,
881
+ device='cuda')
882
+ target_token_logprobs = torch.rand(1,
883
+ batch_size * (k + 1),
884
+ vocab_size,
885
+ dtype=torch.float32,
886
+ device='cuda')
887
+ target_output = create_sampler_output_list(target_token_ids,
888
+ target_token_probs,
889
+ target_token_logprobs)
890
+
891
+ target_worker.execute_model.return_value = [target_output[0]]
892
+
893
+ if not len(decodes):
894
+ worker.execute_model(execute_model_req=execute_model_req)
895
+ # no spec run (prefill only)
896
+ draft_worker.execute_model.assert_called_once_with(execute_model_req)
897
+ target_worker.execute_model.assert_called_once_with(execute_model_req)
898
+ else:
899
+ # Decode-only run OR mixed batch, scorer call fails (it's mocked)
900
+ with pytest.raises(ValueError, match=exception_secret):
901
+ worker.execute_model(execute_model_req=execute_model_req)
902
+ # but first draft still counted
903
+ assert draft_worker.get_spec_proposals.call_count == 1
tests/spec_decode/utils.py
+60
-11
@@ -146,6 +146,41 @@ def create_seq_group_metadata_from_prompts(
146
return seq_grou_metadata_list
147
148
149
+def create_chunked_seq_group_metadata_from_prompt(
150
+ prompt: List[int],
151
+ num_gpu_blocks: int,
152
+ chunk_size: int,
153
+ block_size: int,
154
+ seq_id: Optional[int] = None) -> List[SequenceGroupMetadata]:
155
+
156
+ if seq_id is None:
157
+ seq_id = 0
158
+
159
+ free_gpu_blocks = list(range(num_gpu_blocks))
160
+
161
+ block_allocations = [
162
+ free_gpu_blocks.pop()
163
+ for _ in range(round_up_to_next_block(len(prompt), block_size))
164
+ ]
165
+
166
+ seq_group_metadata_list = []
167
+ for i, idx in enumerate(range(0, len(prompt), chunk_size)):
168
+ chunk_ids = prompt[idx:idx + chunk_size]
169
+ data = SequenceData.from_seqs(prompt)
170
+ data.update_num_computed_tokens(idx)
171
+ seq_data = {i: data}
172
+ seq_group_metadata_list.append(
173
+ SequenceGroupMetadata(
174
+ request_id=str(seq_id),
175
+ is_prompt=True,
176
+ do_sample=idx + chunk_size >= len(prompt), # terminal chunk
177
+ seq_data=seq_data,
178
+ sampling_params=SamplingParams(temperature=0.0),
179
+ block_tables={i: block_allocations},
180
+ token_chunk_size=len(chunk_ids)))
181
+ return seq_group_metadata_list
182
+
183
+
184
def assert_logprobs_dict_allclose(
185
actual_logprobs: List[Dict[int, Logprob]],
186
expected_logprobs: List[Dict[int, Logprob]]) -> None:
@@ -198,7 +233,8 @@ def create_batch(batch_size,
233
prev_output_token_len: int = 10,
234
seq_ids: Optional[List[int]] = None,
235
num_gpu_blocks: Optional[int] = None,
201
- block_size: Optional[int] = None):
236
+ block_size: Optional[int] = None,
237
+ prefill_chunk_size: Optional[int] = None):
238
if block_size is None:
239
block_size = 8
240
@@ -213,15 +249,28 @@ def create_batch(batch_size,
249
prompt_lens = prompt_len
250
251
prompts = [[next(iterator) for _ in range(p_len)] for p_len in prompt_lens]
216
- prev_output_tokens = [[
217
- next(iterator) for _ in range(prev_output_token_len)
218
- ] for _ in range(batch_size)]
219
- final_prompt_lens = [
220
- len(prompt) + len(prev_output_token) + k + 1
221
- for prompt, prev_output_token in zip(prompts, prev_output_tokens)
222
- ]
252
224
- seq_group_metadata_list = create_seq_group_metadata_from_prompts(
225
- prompts, num_gpu_blocks, block_size, final_prompt_lens,
226
- prev_output_tokens, seq_ids)
253
+ if prefill_chunk_size:
254
+ # Create a batch of chunked prompts.
255
+ if not seq_ids:
256
+ seq_ids = list(range(len(prompts)))
257
+ seq_group_metadata_list = []
258
+ for p, sid in zip(prompts, seq_ids):
259
+ seq_group_metadata_list += \
260
+ create_chunked_seq_group_metadata_from_prompt(
261
+ p, num_gpu_blocks, prefill_chunk_size, block_size, sid)
262
+ seq_group_metadata_list = seq_group_metadata_list[:batch_size]
263
+ prev_output_tokens = []
264
+ else:
265
+ prev_output_tokens = [[
266
+ next(iterator) for _ in range(prev_output_token_len)
267
+ ] for _ in range(batch_size)]
268
+ final_prompt_lens = [
269
+ len(prompt) + len(prev_output_token) + k + 1
270
+ for prompt, prev_output_token in zip(prompts, prev_output_tokens)
271
+ ]
272
+
273
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
274
+ prompts, num_gpu_blocks, block_size, final_prompt_lens,
275
+ prev_output_tokens, seq_ids)
276
return seq_group_metadata_list, prompts, prev_output_tokens
vllm/attention/backends/flash_attn.py
+7
-3
@@ -276,7 +276,11 @@ class FlashAttentionMetadata(AttentionMetadata):
276
max_query_len=self.max_query_len,
277
max_prefill_seq_len=0,
278
max_decode_seq_len=self.max_decode_seq_len,
279
- query_start_loc=self.query_start_loc[self.num_prefills:]
279
+ # Batch may be composed of prefill|decodes, adjust query start
280
+ # indices to refer to the start of decodes. E.g.
281
+ # in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
282
+ query_start_loc=(self.query_start_loc[self.num_prefills:] -
283
+ self.query_start_loc[self.num_prefills])
284
if self.query_start_loc is not None else None,
285
seq_start_loc=self.seq_start_loc[self.num_prefills:]
286
if self.seq_start_loc is not None else None,
@@ -903,7 +907,9 @@ def unified_flash_attention(
907
# Decoding run.
908
# Use flash_attn_varlen_func kernel for speculative decoding
909
# because different queries might have different lengths.
910
+
911
assert decode_meta.max_decode_query_len is not None
912
+ # use only for actual varlen decoding
913
if decode_meta.max_decode_query_len > 1:
914
assert attn_type == AttentionType.DECODER, (
915
"Only decoder-only models support max_decode_query_len > 1")
@@ -949,8 +955,6 @@ def unified_flash_attention(
955
assert prefill_output is not None
956
return prefill_output.view(num_prefill_query_tokens, hidden_size)
957
952
- # Chunked prefill does not work with speculative decoding.
953
- # Therefore, the query length for decode should be 1 in chunked prefill.
958
assert decode_meta is not None
959
decode_output = decode_output.squeeze(1)
960
output = torch.cat([prefill_output, decode_output], dim=0)
vllm/attention/backends/rocm_flash_attn.py
+6
@@ -192,6 +192,12 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
192
block_tables=self.block_tables[self.num_prefills:],
193
use_cuda_graph=self.use_cuda_graph,
194
)
195
+ # Batch may be composed of prefill|decodes, adjust query start indices
196
+ # to refer to the start of decodes when the two are split apart.
197
+ # E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
198
+ if self._cached_decode_metadata.query_start_loc is not None:
199
+ qs = self._cached_decode_metadata.query_start_loc
200
+ self._cached_decode_metadata.query_start_loc = qs - qs[0]
201
return self._cached_decode_metadata
202
203
def advance_step(self,
vllm/attention/backends/xformers.py
+7
@@ -272,6 +272,13 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
272
max_encoder_seq_len=self.max_encoder_seq_len,
273
cross_slot_mapping=self.cross_slot_mapping,
274
cross_block_tables=self.cross_block_tables)
275
+
276
+ # Batch may be composed of prefill|decodes, adjust query start indices
277
+ # to refer to the start of decodes when the two are split apart.
278
+ # E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
279
+ if self._cached_decode_metadata.query_start_loc is not None:
280
+ qs = self._cached_decode_metadata.query_start_loc
281
+ self._cached_decode_metadata.query_start_loc = qs - qs[0]
282
return self._cached_decode_metadata
283
284
vllm/config.py
+6
-8
@@ -192,7 +192,6 @@ class ModelConfig:
192
self.max_logprobs = max_logprobs
193
self.disable_sliding_window = disable_sliding_window
194
self.skip_tokenizer_init = skip_tokenizer_init
195
-
195
self.hf_config = get_config(self.model, trust_remote_code, revision,
196
code_revision, rope_scaling, rope_theta,
197
config_format)
@@ -1317,13 +1316,6 @@ class SpeculativeConfig:
1316
"speculative decoding is > 1, but got "
1317
f"{speculative_disable_by_batch_size=}")
1318
1320
- # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1321
- # If the feature combo become valid
1322
- if enable_chunked_prefill:
1323
- raise ValueError(
1324
- "Speculative decoding and chunked prefill are "
1325
- f"currently mutually exclusive ({enable_chunked_prefill=}).")
1326
-
1319
# TODO: The user should be able to specify revision/max model len
1320
# for the draft model. It is not currently supported.
1321
draft_revision = None
@@ -1390,6 +1382,12 @@ class SpeculativeConfig:
1382
f"num_speculative_tokens={n_predict}, but "
1383
f"{num_speculative_tokens=} was provided.")
1384
1385
+ if enable_chunked_prefill and draft_hf_config.model_type in (
1386
+ "medusa", "mlp_speculator", "eagle"):
1387
+ raise ValueError(
1388
+ "Chunked prefill and hidden-state based draft models are "
1389
+ "not compatible.")
1390
+
1391
draft_model_config.max_model_len = (
1392
SpeculativeConfig._maybe_override_draft_max_model_len(
1393
speculative_max_model_len,
vllm/core/scheduler.py
+1
@@ -1147,6 +1147,7 @@ class Scheduler:
1147
1148
# Update swapped requests.
1149
self.swapped.extend(running_scheduled.swapped_out)
1150
+ # Put prefills first due to Attention backend ordering assumption.
1151
return SchedulerOutputs(
1152
scheduled_seq_groups=(prefills.seq_groups +
1153
running_scheduled.prefill_seq_groups +
vllm/engine/output_processor/multi_step.py
+5
-3
@@ -134,10 +134,12 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
134
sample for sample in samples
135
if sample.output_token != VLLM_INVALID_TOKEN_ID
136
]
137
- assert valid_samples
137
139
- self._process_seq_outputs(seq, valid_samples,
140
- sequence_group.sampling_params)
138
+ # When both spec-decode and pre-fill chunking are enabled, we
139
+ # don't have guaranteed samples here (e.g. all -1s).
140
+ if valid_samples:
141
+ self._process_seq_outputs(seq, valid_samples,
142
+ sequence_group.sampling_params)
143
144
def _process_decode_and_stop(self, seq: Sequence,
145
sampling_params: SamplingParams) -> None:
vllm/spec_decode/batch_expansion.py
+31
-30
@@ -90,7 +90,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
90
else:
91
# Batch has a mix of spec decode enabled and disabled seq groups
92
contracted = self._contract_batch(
93
- contracted_bs=len(execute_model_req.seq_group_metadata_list),
93
+ execute_model_req.seq_group_metadata_list,
94
target_sampler_output=target_sampler_output,
95
proposals=proposals,
96
num_scoring_tokens=num_scoring_tokens,
@@ -126,7 +126,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
126
split_batch_by_proposal_len(
127
seq_group_metadata_list, proposal_lens_list)
128
129
- target_seq_group_metadata_list = self._create_scoring_model_input(
129
+ spec_expanded_seqs = self._create_scoring_model_input(
130
seq_group_metadata_list=spec_seqs,
131
proposal_token_ids=proposal_token_ids_list,
132
# NOTE: We determine the seq ids in the expanded batch using the
@@ -135,16 +135,19 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
135
seq_ids=get_all_seq_ids(seq_group_metadata_list)),
136
)
137
138
- num_scoring_tokens = len(target_seq_group_metadata_list)
139
- target_seq_group_metadata_list.extend(non_spec_seqs)
138
+ num_scoring_tokens = len(spec_expanded_seqs)
139
+ # Batch speculative and non-speculative (e.g. chunked prefill) requests
140
+ # but make sure order is prefill|decode due to backend requirement.
141
+ target_seq_group_metadata_list = non_spec_seqs + spec_expanded_seqs
142
143
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
144
num_scoring_tokens)
145
146
def _contract_batch(
145
- self, contracted_bs: int, target_sampler_output: SamplerOutput,
146
- proposals: SpeculativeProposals, num_scoring_tokens: int,
147
- non_spec_indices: List[int], spec_indices: List[int], k: int
147
+ self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
148
+ target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
149
+ num_scoring_tokens: int, non_spec_indices: List[int],
150
+ spec_indices: List[int], k: int
151
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
152
Optional[torch.Tensor]]:
153
"""Contract the expanded batch back into its original size.
@@ -154,6 +157,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
157
contracted_bs is the original batch size, and the batch size that the
158
target_sampler_output will be contracted to.
159
"""
160
+ contracted_bs = len(contracted_seq_group_metadata_list)
161
(target_token_ids, target_probs, target_logprobs, target_hidden_states,
162
non_spec_target_token_ids, non_spec_target_probs,
163
non_spec_target_logprobs,
@@ -166,8 +170,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
170
171
# The number of tokens in the expanded batch used for speculation is
172
# equal to the total expanded batch size minus the number of samples for
169
- # non-speculative sequences.
170
- non_spec_expanded_bs = len(non_spec_target_token_ids)
173
+ # non-speculative sequences, prefill chunks with no out tokens included
174
+ non_spec_expanded_bs = len(non_spec_indices)
175
spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
176
177
target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
@@ -191,7 +195,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
195
else:
196
all_hidden_states = None
197
194
- if non_spec_indices:
198
+ # Rule out prefills that produce no tokens.
199
+ non_spec_indices = [
200
+ idx for idx in non_spec_indices
201
+ if contracted_seq_group_metadata_list[idx].do_sample
202
+ ]
203
+ if len(non_spec_indices):
204
all_tokens[non_spec_indices, :1] = \
205
non_spec_target_token_ids.unsqueeze(1)
206
all_probs[non_spec_indices, :1, :] = \
@@ -290,9 +299,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
299
This function creates K+1 target SequenceGroupMetadata to take
300
advantage of the bonus token.
301
"""
293
- assert not input_seq_group_metadata.is_prompt, (
294
- "Speculating on "
295
- "prompts not yet supported")
302
assert len(input_seq_group_metadata.seq_data) == 1, (
303
"Beam search "
304
"not supported in speculative decoding")
@@ -390,27 +396,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
396
# and non spec sequences) and should be removed in the future. It can be
397
# done by supporting per-sequence proposal lens.
398
#
393
- # First samples are from speculative scoring, latter samples are non-
394
- # speculative samples.
395
- split_sizes = (num_scoring_tokens,
396
- sampler_output.sampled_token_ids.numel() -
397
- num_scoring_tokens)
398
- (spec_probs, non_spec_probs
399
- ) = sampler_output.sampled_token_probs.split(split_sizes)
400
- (spec_sampled_tokens, non_spec_sampled_tokens
399
+ # First samples are non-speculative, latter samples are from speculative
400
+ # scoring (prefill|decode order).
401
+ split_sizes = (sampler_output.sampled_token_ids.numel() -
402
+ num_scoring_tokens, num_scoring_tokens)
403
+ (non_spec_probs,
404
+ spec_probs) = sampler_output.sampled_token_probs.split(split_sizes)
405
+ (non_spec_sampled_tokens, spec_sampled_tokens
406
) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
402
- (
403
- spec_logprobs,
404
- non_spec_logprobs,
405
- ) = sampler_output.logprobs.split(split_sizes)
407
+ (non_spec_logprobs,
408
+ spec_logprobs) = sampler_output.logprobs.split(split_sizes)
409
410
if sampler_output.hidden_states is not None:
408
- (
409
- spec_hidden_states,
410
- non_spec_hidden_states,
411
- ) = sampler_output.hidden_states.split(split_sizes)
411
+ (non_spec_hidden_states, spec_hidden_states
412
+ ) = sampler_output.hidden_states.split(split_sizes)
413
else:
413
- spec_hidden_states, non_spec_hidden_states = None, None
414
+ non_spec_hidden_states, spec_hidden_states = None, None
415
416
return (spec_sampled_tokens, spec_probs, spec_logprobs,
417
spec_hidden_states, non_spec_sampled_tokens, non_spec_probs,
vllm/spec_decode/mqa_scorer.py
+19
-12
@@ -21,6 +21,11 @@ class MQAScorer(SpeculativeScorer):
21
all_proposal_lengths = proposals.proposal_lens.tolist()
22
for i, seq_group_metadata in enumerate(
23
execute_model_req.seq_group_metadata_list):
24
+ if all_proposal_lengths[i] == 0:
25
+ # Keep prompt seqs untouched (keep computed_tokens for chunks).
26
+ target_seq_group_metadata_list.append(seq_group_metadata)
27
+ continue
28
+
29
seq_data_dict = seq_group_metadata.seq_data
30
assert len(seq_data_dict) == 1
31
seq_id = next(iter(seq_data_dict.keys()))
@@ -40,8 +45,7 @@ class MQAScorer(SpeculativeScorer):
45
new_seq_data.update_num_computed_tokens(
46
len(prompt_token_ids) + len(output_token_ids) - 1)
47
43
- # Ensure that the new sequence has at least one token
44
- # because we only use mqa scorer in the decoding stage.
48
+ # Ensure that the new decode sequence has at least one token.
49
assert len(output_token_ids) >= 1
50
new_seq_data_dict = {target_seq_id: new_seq_data}
51
@@ -54,7 +58,6 @@ class MQAScorer(SpeculativeScorer):
58
target_seq_id: seq_group_metadata.block_tables[seq_id],
59
},
60
lora_request=None,
57
- token_chunk_size=1,
61
)
62
target_seq_group_metadata_list.append(new_seq_group_metadata)
63
@@ -77,6 +80,7 @@ class MQAScorer(SpeculativeScorer):
80
all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
81
all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size)
82
else:
83
+ # We either have decodes with different lens or prefill+decodes.
84
all_tokens = target_token_ids.new_full(size=(bs, k + 1),
85
fill_value=-1)
86
all_probs = target_probs.new_zeros(*all_tokens.shape,
@@ -85,15 +89,18 @@ class MQAScorer(SpeculativeScorer):
89
fill_value=-float("inf"))
90
target_token_ids = target_token_ids.flatten()
91
start_loc = 0
88
- for i, proposed_len in enumerate(all_proposal_lengths):
89
- output_len = proposed_len + 1
90
- end_loc = start_loc + output_len
91
- all_tokens[
92
- i, :output_len] = target_token_ids[start_loc:end_loc]
93
- all_probs[i, :output_len] = target_probs[start_loc:end_loc]
94
- all_logprobs[
95
- i, :output_len] = target_logprobs[start_loc:end_loc]
96
- start_loc = end_loc
92
+ for i, (proposed_len, seq_meta) in enumerate(
93
+ zip(all_proposal_lengths, target_seq_group_metadata_list)):
94
+ # Skip chunks with no output tokens.
95
+ if seq_meta.do_sample:
96
+ output_len = proposed_len + 1
97
+ end_loc = start_loc + output_len
98
+ all_tokens[
99
+ i, :output_len] = target_token_ids[start_loc:end_loc]
100
+ all_probs[i, :output_len] = target_probs[start_loc:end_loc]
101
+ all_logprobs[
102
+ i, :output_len] = target_logprobs[start_loc:end_loc]
103
+ start_loc = end_loc
104
105
hidden_states = None
106
if target_sampler_output.hidden_states is not None:
vllm/spec_decode/spec_decode_worker.py
+78
-32
@@ -418,7 +418,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
418
# none of the requests in the batch have spec decoding enabled.
419
# In any of these cases, the proposer and scorer workers
420
# are called normally.
421
- no_spec = num_lookahead_slots == 0 or disable_all_speculation or all(
421
+ # We expect `num_speculative_tokens` to be None for prefills.
422
+ no_spec = all(
423
+ sgm.is_prompt for sgm in execute_model_req.seq_group_metadata_list
424
+ ) or num_lookahead_slots == 0 or disable_all_speculation or all(
425
sgm.num_speculative_tokens == 0
426
for sgm in execute_model_req.seq_group_metadata_list)
427
@@ -484,7 +487,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
487
488
def _serialize_sampler_output_no_logprobs(
489
self, execute_model_req: ExecuteModelRequest,
487
- sampler_output: SamplerOutput) -> SamplerOutput:
490
+ sampler_output: SamplerOutput) -> List[SamplerOutput]:
491
"""
492
Creates and returns a `SamplerOutput` with only the token IDs being
493
serialized to CPU and populated in `CompletionSequenceGroupOutput`.
@@ -514,41 +517,56 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
517
if any(seq_output_prompt_logprobs) else \
518
sampler_output.sampled_token_ids).tolist()
519
517
- seq_data_entries = (
520
+ seq_data_entries = [
521
(seq_id, seq_data) for sg in \
522
execute_model_req.seq_group_metadata_list \
523
for seq_id, seq_data in sg.seq_data.items()
521
- )
524
+ if sg.do_sample # ignore empty token sequences
525
+ ]
526
completion_seq_group_output_list: List[
527
CompletionSequenceGroupOutput] = []
524
- for index, ((seq_id, seq_data), needs_prompt_logprobs) in \
525
- enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)):
526
- if needs_prompt_logprobs:
527
- prompt_token_ids = seq_data.get_prompt_token_ids()
528
- prompt_logprobs = [
529
- create_logprobs_output(
530
- token_id=p_token_id,
528
+ output_index = 0
529
+ # Make sure the non-terminal prefill chunks are still aligned with
530
+ # their own empty output.
531
+ for seq_group_meta in execute_model_req.seq_group_metadata_list:
532
+ # Since we can get chunks here, we dont always have a sampled token
533
+ # (only on last chunk) but we still have to provide an output.
534
+ if not seq_group_meta.do_sample:
535
+ completion_seq_group_output_list.append(
536
+ CompletionSequenceGroupOutput(samples=[],
537
+ prompt_logprobs=None))
538
+ else:
539
+ # Sequence with output.
540
+ seq_id, seq_data = seq_data_entries[output_index]
541
+ needs_prompt_logprobs = seq_output_prompt_logprobs[
542
+ output_index]
543
+ if needs_prompt_logprobs:
544
+ prompt_token_ids = seq_data.get_prompt_token_ids()
545
+ prompt_logprobs = [
546
+ create_logprobs_output(
547
+ token_id=p_token_id,
548
+ token_id_logprob_rank=-1,
549
+ token_id_logprob=0.0,
550
+ topk_token_ids=[],
551
+ topk_logprobs=[],
552
+ )
553
+ # no prompt logprobs for the first token
554
+ for p_token_id in prompt_token_ids[1:]
555
+ ]
556
+ else:
557
+ prompt_logprobs = None
558
+ completion_seq_group_output_list.append(
559
+ create_sequence_group_output(
560
+ token_id=sampled_token_ids_list[output_index][0],
561
token_id_logprob_rank=-1,
562
token_id_logprob=0.0,
563
+ seq_id=seq_id,
564
topk_token_ids=[],
565
topk_logprobs=[],
535
- )
536
- # no prompt logprobs for the first token
537
- for p_token_id in prompt_token_ids[1:]
538
- ]
539
- else:
540
- prompt_logprobs = None
541
-
542
- completion_seq_group_output_list.append(
543
- create_sequence_group_output(
544
- token_id=sampled_token_ids_list[index][0],
545
- token_id_logprob_rank=-1,
546
- token_id_logprob=0.0,
547
- seq_id=seq_id,
548
- topk_token_ids=[],
549
- topk_logprobs=[],
550
- prompt_logprobs=prompt_logprobs))
551
- return SamplerOutput(outputs=completion_seq_group_output_list)
566
+ prompt_logprobs=prompt_logprobs))
567
+ output_index += 1
568
+
569
+ return [SamplerOutput(outputs=completion_seq_group_output_list)]
570
571
@nvtx_range("spec_decode_worker._run_no_spec")
572
def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
@@ -568,6 +586,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
586
hidden_states = sampler_output.hidden_states
587
if hidden_states is not None:
588
# remove hidden_states for prompt tokens
589
+ # TODO Enable `return_hidden_states`: prefill chunks hidden states
590
+ # are pruned by the logits processor. Also, they should be arranged
591
+ # back into full-prefill latent. Address it to enable MLPSpeculator.
592
if any(seq.is_prompt
593
for seq in execute_model_req.seq_group_metadata_list):
594
hidden_states = hidden_states[
@@ -593,14 +614,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
614
sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
615
execute_model_req=execute_model_req, sampler_output=sampler_output)
616
if self._disable_logprobs else
596
- sampler_output)
617
+ [sampler_output])
618
619
# Clear device tensors from sampler output. This reduces communication
620
# overhead when the engine runs in a different process than the workers.
621
sampler_output.sampled_token_probs = None
622
sampler_output.sampled_token_ids = None
623
sampler_output.logprobs = None
603
- return [sampler_output_to_return]
624
+ return sampler_output_to_return
625
626
def _run_non_driver_rank(self) -> bool:
627
"""Run proposer and verifier model in non-driver workers. This is used
@@ -644,9 +665,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
665
This invokes the proposer worker to get k speculative tokens for each
666
sequence, then scores each speculative token using the scoring worker.
667
668
+ When `enable_chunked_prefill` is set, scorer will batch decodes and
669
+ prefills, while proposer will sync its KV-cache by running an extra
670
+ forward on prefills.
671
+
672
Returns a list of SamplerOutput, each containing a single token per
673
sequence.
674
"""
675
+ # With prefill chunking, expect requests to have prompts first
676
+ # so that backend gets prefill|decode.
677
assert num_lookahead_slots == execute_model_req.num_lookahead_slots
678
679
# Pass last hidden states from target model to proposer
@@ -671,6 +698,25 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
698
proposals,
699
)
700
701
+ _, (non_spec_seqs, non_spec_indices) = split_batch_by_proposal_len(
702
+ execute_model_req.seq_group_metadata_list, proposals.proposal_lens)
703
+ # With prefill chunking enabled, `non_spec_seqs` contains prefills too:
704
+ # discard decodes that have already been processed by proposer.
705
+ non_spec_indices = [
706
+ idx for idx in non_spec_indices
707
+ if execute_model_req.seq_group_metadata_list[idx].is_prompt
708
+ ]
709
+ if len(non_spec_indices):
710
+ all_hidden_states = proposal_scores.hidden_states
711
+ # TODO fix `return_hidden_states`, same as in `_run_no_spec`
712
+ if all_hidden_states is not None:
713
+ prefill_hidden_states = all_hidden_states[non_spec_indices]
714
+ execute_model_req.previous_hidden_states = \
715
+ prepare_prefill_hidden_states(prefill_hidden_states)
716
+ # Sync proposer KV cache for prefills.
717
+ prefill_req = execute_model_req.clone(non_spec_seqs)
718
+ self.proposer_worker.execute_model(prefill_req)
719
+
720
with Timer() as verification_timer:
721
accepted_token_ids, target_logprobs = self._verify_tokens(
722
execute_model_req.seq_group_metadata_list, proposal_scores,
@@ -769,7 +815,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
815
self.previous_hidden_states = HiddenStates(
816
hidden_states, seq_group_metadata_list,
817
second_last_token_hidden_states)
772
-
818
return accepted_token_ids, logprobs
819
820
def _create_output_sampler_list(
@@ -819,6 +864,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
864
accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
865
866
# Construct the output on a per-step, per-sequence basis.
867
+ # Non-terminal prefill chunks will end up here as rows with just -1s
868
+ # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
869
sampler_output_list: List[SamplerOutput] = []
870
for step_index in range(num_steps):
871
if all(token_id == -1
@@ -861,7 +908,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
908
# This is periodic because the rejection sampler emits metrics
909
# periodically.
910
self._maybe_log_stage_times(*stage_times)
864
-
911
return sampler_output_list
912
913
def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,
vllm/spec_decode/top1_proposer.py
+4
-4
@@ -109,7 +109,6 @@ class Top1Proposer(SpeculativeProposer):
109
proposal_probs=proposal_probs,
110
proposal_lens=proposal_lens,
111
no_proposals=maybe_sampler_output is None)
112
-
112
return proposals
113
114
def _split_by_proposal_len(
@@ -127,9 +126,10 @@ class Top1Proposer(SpeculativeProposer):
126
nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
127
nonzero_proposal_len_indices: List[int] = []
128
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
130
- # The speculative decoding for this request has been disabled
131
- # (e.g. due to high traffic).
132
- if seq_group_metadata.num_speculative_tokens == 0:
129
+ # The speculative decoding for this request has either been disabled
130
+ # (e.g. due to high traffic) or this is a prompt request.
131
+ if (seq_group_metadata.is_prompt
132
+ or seq_group_metadata.num_speculative_tokens == 0):
133
proposal_lens.append(0)
134
continue
135