@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d43afcc53
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Feature] [Spec decode]: Combine chunked prefill with speculative decoding (#9291)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Nov 7, 2024 at 17:15 UTC 9d43afcc538645625ea5fc2bca01d3697dd0595c
17 files changed +477 -147
tests/spec_decode/e2e/test_compatibility.py
-34
@@ -5,40 +5,6 @@ from vllm import SamplingParams
5 from .conftest import get_output_from_llm_generator
6
7
8 -@pytest.mark.parametrize("common_llm_kwargs", [{
9 - "model": "JackFram/llama-68m",
10 - "speculative_model": "JackFram/llama-68m",
11 - "num_speculative_tokens": 5,
12 -}])
13 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [
14 - {
15 - "enable_chunked_prefill": True,
16 - },
17 -])
18 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
19 -@pytest.mark.parametrize("seed", [1])
20 -def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
21 - """Verify that speculative decoding with chunked prefill fails.
22 - """
23 - output_len = 128
24 - temperature = 0.0
25 -
26 - prompts = [
27 - "Hello, my name is",
28 - ]
29 -
30 - sampling_params = SamplingParams(
31 - max_tokens=output_len,
32 - ignore_eos=True,
33 - temperature=temperature,
34 - )
35 -
36 - with pytest.raises(ValueError,
37 - match="Speculative decoding and chunked prefill"):
38 - get_output_from_llm_generator(test_llm_generator, prompts,
39 - sampling_params)
40 -
41 -
8 @pytest.mark.parametrize("common_llm_kwargs", [{
9 "model": "meta-llama/Llama-2-7b-chat-hf",
10 "speculative_model": "JackFram/llama-68m",
tests/spec_decode/e2e/test_multistep_correctness.py
+102 -3
@@ -62,6 +62,16 @@ from .conftest import (get_output_from_llm_generator,
62 {
63 "speculative_model": "JackFram/llama-68m",
64 "num_speculative_tokens": 5,
65 + "enable_chunked_prefill": False,
66 + },
67 + {
68 + # Chunked prefill enabled with small value
69 + # to make sure we get mixed batches.
70 + "speculative_model": "JackFram/llama-68m",
71 + "num_speculative_tokens": 5,
72 + "enable_chunked_prefill": True,
73 + "max_num_batched_tokens": 4,
74 + "max_num_seqs": 4
75 },
76 {
77 # Verify the detokenizer assertions in the test work when spec
@@ -141,6 +151,14 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
151 {
152 "speculative_model": "JackFram/llama-68m",
153 "num_speculative_tokens": 5,
154 + "enable_chunked_prefill": False,
155 + },
156 + {
157 + "speculative_model": "JackFram/llama-68m",
158 + "num_speculative_tokens": 5,
159 + "enable_chunked_prefill": True,
160 + "max_num_batched_tokens": 4,
161 + "max_num_seqs": 4,
162 },
163 ])
164 @pytest.mark.parametrize(
@@ -204,6 +222,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
222 {
223 "speculative_model": "JackFram/llama-68m",
224 "num_speculative_tokens": 5,
225 + "enable_chunked_prefill": False,
226 + },
227 + {
228 + "speculative_model": "JackFram/llama-68m",
229 + "num_speculative_tokens": 5,
230 + "enable_chunked_prefill": True,
231 + "max_num_batched_tokens": 4,
232 + "max_num_seqs": 4
233 },
234 ])
235 @pytest.mark.parametrize(
@@ -255,6 +281,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
281 {
282 "speculative_model": "JackFram/llama-68m",
283 "num_speculative_tokens": 5,
284 + "enable_chunked_prefill": False,
285 + },
286 + {
287 + "speculative_model": "JackFram/llama-68m",
288 + "num_speculative_tokens": 5,
289 + "enable_chunked_prefill": True,
290 + "max_num_batched_tokens": 4,
291 + "max_num_seqs": 4
292 },
293 ])
294 @pytest.mark.parametrize("max_output_len", [
@@ -300,6 +334,14 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
334 {
335 "speculative_model": "JackFram/llama-68m",
336 "num_speculative_tokens": 5,
337 + "enable_chunked_prefill": False,
338 + },
339 + {
340 + "speculative_model": "JackFram/llama-68m",
341 + "num_speculative_tokens": 5,
342 + "enable_chunked_prefill": True,
343 + "max_num_batched_tokens": 4,
344 + "max_num_seqs": 4
345 },
346 ])
347 @pytest.mark.parametrize("batch_size", [1])
@@ -347,6 +389,14 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
389 {
390 "speculative_model": "JackFram/llama-68m",
391 "num_speculative_tokens": 5,
392 + "enable_chunked_prefill": False,
393 + },
394 + {
395 + "speculative_model": "JackFram/llama-68m",
396 + "num_speculative_tokens": 5,
397 + "enable_chunked_prefill": True,
398 + "max_num_batched_tokens": 4,
399 + "max_num_seqs": 4
400 },
401 ])
402 @pytest.mark.parametrize("batch_size", [32])
@@ -397,6 +447,14 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
447 {
448 "speculative_model": "JackFram/llama-68m",
449 "num_speculative_tokens": 5,
450 + "enable_chunked_prefill": False,
451 + },
452 + {
453 + "speculative_model": "JackFram/llama-68m",
454 + "num_speculative_tokens": 5,
455 + "enable_chunked_prefill": True,
456 + "max_num_batched_tokens": 4,
457 + "max_num_seqs": 4
458 },
459 ])
460 @pytest.mark.parametrize(
@@ -454,6 +512,14 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
512 {
513 "speculative_model": "JackFram/llama-68m",
514 "num_speculative_tokens": 5,
515 + "enable_chunked_prefill": False,
516 + },
517 + {
518 + "speculative_model": "JackFram/llama-68m",
519 + "num_speculative_tokens": 5,
520 + "enable_chunked_prefill": True,
521 + "max_num_batched_tokens": 4,
522 + "max_num_seqs": 4
523 },
524 ])
525 @pytest.mark.parametrize("batch_size", [2])
@@ -503,6 +569,15 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
569 # Artificially limit the draft model max model len; this forces vLLM
570 # to skip speculation once the sequences grow beyond 32-k tokens.
571 "speculative_max_model_len": 32,
572 + "enable_chunked_prefill": False,
573 + },
574 + {
575 + "speculative_model": "JackFram/llama-68m",
576 + "num_speculative_tokens": 5,
577 + "enable_chunked_prefill": True,
578 + "max_num_batched_tokens": 4,
579 + "max_num_seqs": 4,
580 + "speculative_max_model_len": 32,
581 },
582 ])
583 @pytest.mark.parametrize("batch_size", [8])
@@ -551,6 +626,15 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,
626 "speculative_model": "JackFram/llama-68m",
627 "num_speculative_tokens": 5,
628 "speculative_disable_by_batch_size": 2,
629 + "enable_chunked_prefill": False,
630 + },
631 + {
632 + "speculative_model": "JackFram/llama-68m",
633 + "num_speculative_tokens": 5,
634 + "speculative_disable_by_batch_size": 2,
635 + "enable_chunked_prefill": True,
636 + "max_num_batched_tokens": 4,
637 + "max_num_seqs": 4,
638 },
639 ])
640 @pytest.mark.parametrize("batch_size", [8])
@@ -590,10 +674,17 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,
674 {
675 "speculative_model": "JackFram/llama-68m",
676 "num_speculative_tokens": k,
677 + "enable_chunked_prefill": False,
678 }
679 # Try a range of common k, as well as large speculation.
680 for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]
596 - ])
681 + ] + [{
682 + "speculative_model": "JackFram/llama-68m",
683 + "num_speculative_tokens": k,
684 + "enable_chunked_prefill": True,
685 + "max_num_batched_tokens": 4,
686 + "max_num_seqs": 4,
687 + } for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]])
688 @pytest.mark.parametrize("batch_size", [2])
689 @pytest.mark.parametrize(
690 "output_len",
@@ -636,11 +727,19 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
727 {
728 "speculative_model": "JackFram/llama-68m",
729 "num_speculative_tokens": k,
639 - "spec_decoding_acceptance_method": "typical_acceptance_sampler"
730 + "spec_decoding_acceptance_method": "typical_acceptance_sampler",
731 + "enable_chunked_prefill": False
732 }
733 # Try a range of common k.
734 for k in [1, 2, 3]
643 - ])
735 + ] + [{
736 + "speculative_model": "JackFram/llama-68m",
737 + "num_speculative_tokens": k,
738 + "spec_decoding_acceptance_method": "typical_acceptance_sampler",
739 + "enable_chunked_prefill": True,
740 + "max_num_batched_tokens": 4,
741 + "max_num_seqs": 4
742 + } for k in [1, 2, 3]])
743 @pytest.mark.parametrize("batch_size", [1, 32])
744 @pytest.mark.parametrize(
745 "output_len",
tests/spec_decode/e2e/test_ngram_correctness.py
+35 -1
@@ -50,18 +50,33 @@ from .conftest import run_equality_correctness_test
50 "num_speculative_tokens": 5,
51 "ngram_prompt_lookup_max": 3,
52 },
53 + {
54 + "speculative_model": "[ngram]",
55 + "num_speculative_tokens": 5,
56 + "ngram_prompt_lookup_max": 3,
57 + },
58 ])
59 @pytest.mark.parametrize("output_len", [
60 256,
61 ])
62 @pytest.mark.parametrize("batch_size", [1, 32])
63 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
64 @pytest.mark.parametrize("seed", [1])
65 def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
66 per_test_common_llm_kwargs,
67 baseline_llm_kwargs, test_llm_kwargs,
68 batch_size: int, output_len: int,
63 - seed: int):
69 + prefill_chunk_size: int, seed: int):
70 """Verify greedy equality on a tiny model with different batch size."""
71 + if prefill_chunk_size > 0:
72 + common_llm_kwargs.update(
73 + **{
74 + "enable_chunked_prefill": True,
75 + "max_num_batched_tokens": prefill_chunk_size,
76 + "max_num_seqs": prefill_chunk_size
77 + })
78 + else:
79 + common_llm_kwargs["enable_chunked_prefill"] = False
80 run_equality_correctness_test(vllm_runner,
81 common_llm_kwargs,
82 per_test_common_llm_kwargs,
@@ -151,6 +166,16 @@ def test_ngram_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
166 "speculative_model": "[ngram]",
167 "num_speculative_tokens": 5,
168 "ngram_prompt_lookup_max": 3,
169 + "enable_chunked_prefill": False,
170 + },
171 + {
172 + "speculative_model": "[ngram]",
173 + "num_speculative_tokens": 5,
174 + "ngram_prompt_lookup_max": 3,
175 + "enable_chunked_prefill": True,
176 + "speculative_disable_mqa_scorer": True,
177 + "max_num_batched_tokens": 4,
178 + "max_num_seqs": 4
179 },
180 ])
181 @pytest.mark.parametrize(
@@ -251,6 +276,15 @@ def test_ngram_different_k(vllm_runner, common_llm_kwargs,
276 "num_speculative_tokens": 5,
277 "ngram_prompt_lookup_max": 3,
278 "speculative_disable_by_batch_size": 4
279 + }, {
280 + "speculative_model": "[ngram]",
281 + "num_speculative_tokens": 5,
282 + "ngram_prompt_lookup_max": 3,
283 + "speculative_disable_by_batch_size": 4,
284 + "enable_chunked_prefill": True,
285 + "speculative_disable_mqa_scorer": True,
286 + "max_num_batched_tokens": 4,
287 + "max_num_seqs": 4
288 }])
289 @pytest.mark.parametrize("batch_size", [1, 5])
290 @pytest.mark.parametrize(
tests/spec_decode/test_ngram_worker.py
+7 -2
@@ -118,7 +118,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
118 num_gpu_blocks,
119 block_size,
120 final_prompt_lens=final_prompt_lens)
121 -
121 + for sg in seq_group_metadata_list:
122 + sg.is_prompt = False
123 proposals = proposer.get_spec_proposals(
124 execute_model_req=ExecuteModelRequest(
125 seq_group_metadata_list=seq_group_metadata_list,
@@ -147,7 +148,7 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
148 def test_ngram_algo_correctness_for_batches_match_all():
149 """Verify our ngram algo find the right candidate in the prompt
150
150 - For the scenario find candidate in all batchs
151 + For the scenario find candidate in all batches
152 """
153
154 block_size = 32
@@ -192,6 +193,10 @@ def test_ngram_algo_correctness_for_batches_match_all():
193 block_size,
194 final_prompt_lens=final_prompt_lens)
195
196 + # Normally drafter is run on decode requests only; here we check the output
197 + # of the ngram worker as it is the sole proposer that has no forward.
198 + for sg in seq_group_metadata_list:
199 + sg.is_prompt = False
200 proposals = proposer.get_spec_proposals(
201 execute_model_req=ExecuteModelRequest(
202 seq_group_metadata_list=seq_group_metadata_list,
tests/spec_decode/test_scorer.py
+27 -4
@@ -46,12 +46,14 @@ def assert_score_equal(score1: SpeculativeScores,
46 @pytest.mark.parametrize('max_propose_len', [1, 3, 5])
47 @pytest.mark.parametrize('mixed_propose_len', [True])
48 @pytest.mark.parametrize('device', ['cuda'])
49 +@pytest.mark.parametrize('prefill_chunking', [False, True])
50 def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
50 - mixed_propose_len: bool, device: str) -> None:
51 + mixed_propose_len: bool, device: str,
52 + prefill_chunking: bool) -> None:
53 """
54 Compare the batch expansion scorer and mqa scorer return the same score.
55 We test for both queries with the same propose length and different
54 - propose length.
56 + propose length, as well as mixed prefill-decode batches.
57 """
58 seed = 0
59 block_size = 32
@@ -67,16 +69,37 @@ def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
69 if not mixed_propose_len:
70 propose_lens = [max_propose_len] * batch_size
71 else:
70 - non_zero_cnt = random.randint(0, batch_size)
72 + # There must be at least 1 decode request, otherwise
73 + # we have nothing to score (`_run_no_spec`).
74 + non_zero_cnt = random.randint(1, batch_size)
75 propose_lens = [max_propose_len
76 ] * non_zero_cnt + [0] * (batch_size - non_zero_cnt)
77 random.shuffle(propose_lens)
78
75 - proposals = create_proposal(propose_lens, vocab_size, device)
79 seq_group_metadatalist, _, _ = create_batch(batch_size,
80 max_propose_len,
81 block_size=block_size,
82 num_gpu_blocks=num_gpu_blocks)
83 +
84 + if mixed_propose_len and prefill_chunking and (n_prefills :=
85 + batch_size - non_zero_cnt):
86 + prefill, _, _ = create_batch(n_prefills,
87 + None,
88 + prefill_chunk_size=4,
89 + block_size=block_size,
90 + num_gpu_blocks=num_gpu_blocks,
91 + seq_ids=list(
92 + range(batch_size,
93 + batch_size + n_prefills)))
94 + # re-order to guarantee prefill|decode order
95 + target_group_metadatalist = [
96 + seq_group_metadatalist[i] for i, p in enumerate(propose_lens)
97 + if p > 0
98 + ]
99 + seq_group_metadatalist = prefill + target_group_metadatalist
100 + propose_lens = [0] * n_prefills + [p for p in propose_lens if p > 0]
101 +
102 + proposals = create_proposal(propose_lens, vocab_size, device)
103 requests = ExecuteModelRequest(seq_group_metadatalist,
104 num_lookahead_slots=max_propose_len)
105
tests/spec_decode/test_spec_decode_worker.py
+82
@@ -10,6 +10,7 @@ import torch
10 from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.model_executor.utils import set_random_seed
12 from vllm.sequence import ExecuteModelRequest, SequenceOutput
13 +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
14 from vllm.spec_decode.interfaces import SpeculativeProposals
15 from vllm.spec_decode.metrics import (AsyncMetricsCollector,
16 SpecDecodeWorkerMetrics)
@@ -819,3 +820,84 @@ def test_handle_finished_requests():
820 # and 'request-3' are removed from seq_with_bonus_token_in_last_step.
821 assert worker._seq_with_bonus_token_in_last_step == \
822 {4,5,10}
823 +
824 +
825 +@pytest.mark.parametrize('k', [3])
826 +@pytest.mark.parametrize('batch_size', [2, 32])
827 +@pytest.mark.parametrize("batch_composition",
828 + ["prefill_only", "decode_only", "mixed"])
829 +@torch.inference_mode()
830 +def test_chunked_prefill_flow(k: int, batch_size: int, batch_composition: str):
831 + """
832 + Verify SpecDecodeWorker calls match the expected flow.
833 + """
834 + vocab_size = 32_000
835 + draft_worker = mock_worker(cls=MultiStepWorker)
836 + target_worker = mock_worker()
837 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
838 + worker = SpecDecodeWorker(draft_worker,
839 + target_worker,
840 + mock_spec_decode_sampler("rejection_sampler"),
841 + disable_logprobs=False,
842 + metrics_collector=metrics_collector)
843 + exception_secret = 'artificial stop'
844 + worker.scorer = mock_worker(BatchExpansionTop1Scorer)
845 + worker.scorer.score_proposals.side_effect = ValueError(exception_secret)
846 +
847 + # Create batch with combination of terminal/non-terminal prefill chunks
848 + # and decodes (different seq_ids).
849 + decodes, _, _ = create_batch(batch_size, k)
850 + # Pre-chunking here, get 'batch_size' chunks.
851 + prefill, _, _ = create_batch(batch_size,
852 + k,
853 + prefill_chunk_size=4,
854 + seq_ids=list(range(batch_size,
855 + batch_size * 2)))
856 +
857 + if batch_composition == "prefill_only":
858 + n_prefills = batch_size
859 + elif batch_composition == "decode_only":
860 + n_prefills = 0
861 + else:
862 + n_prefills = random.randint(1, batch_size - 1)
863 + n_decodes = batch_size - n_prefills
864 +
865 + prefill = random.sample(prefill, n_prefills)
866 + decodes = random.sample(decodes, n_decodes)
867 + target_group_metadata_list = prefill + decodes
868 + execute_model_req = ExecuteModelRequest(
869 + seq_group_metadata_list=target_group_metadata_list,
870 + num_lookahead_slots=k)
871 +
872 + target_token_ids = torch.randint(low=0,
873 + high=vocab_size,
874 + size=(1, batch_size * (k + 1)),
875 + dtype=torch.int64,
876 + device='cuda')
877 + target_token_probs = torch.rand(1,
878 + batch_size * (k + 1),
879 + vocab_size,
880 + dtype=torch.float32,
881 + device='cuda')
882 + target_token_logprobs = torch.rand(1,
883 + batch_size * (k + 1),
884 + vocab_size,
885 + dtype=torch.float32,
886 + device='cuda')
887 + target_output = create_sampler_output_list(target_token_ids,
888 + target_token_probs,
889 + target_token_logprobs)
890 +
891 + target_worker.execute_model.return_value = [target_output[0]]
892 +
893 + if not len(decodes):
894 + worker.execute_model(execute_model_req=execute_model_req)
895 + # no spec run (prefill only)
896 + draft_worker.execute_model.assert_called_once_with(execute_model_req)
897 + target_worker.execute_model.assert_called_once_with(execute_model_req)
898 + else:
899 + # Decode-only run OR mixed batch, scorer call fails (it's mocked)
900 + with pytest.raises(ValueError, match=exception_secret):
901 + worker.execute_model(execute_model_req=execute_model_req)
902 + # but first draft still counted
903 + assert draft_worker.get_spec_proposals.call_count == 1
tests/spec_decode/utils.py
+60 -11
@@ -146,6 +146,41 @@ def create_seq_group_metadata_from_prompts(
146 return seq_grou_metadata_list
147
148
149 +def create_chunked_seq_group_metadata_from_prompt(
150 + prompt: List[int],
151 + num_gpu_blocks: int,
152 + chunk_size: int,
153 + block_size: int,
154 + seq_id: Optional[int] = None) -> List[SequenceGroupMetadata]:
155 +
156 + if seq_id is None:
157 + seq_id = 0
158 +
159 + free_gpu_blocks = list(range(num_gpu_blocks))
160 +
161 + block_allocations = [
162 + free_gpu_blocks.pop()
163 + for _ in range(round_up_to_next_block(len(prompt), block_size))
164 + ]
165 +
166 + seq_group_metadata_list = []
167 + for i, idx in enumerate(range(0, len(prompt), chunk_size)):
168 + chunk_ids = prompt[idx:idx + chunk_size]
169 + data = SequenceData.from_seqs(prompt)
170 + data.update_num_computed_tokens(idx)
171 + seq_data = {i: data}
172 + seq_group_metadata_list.append(
173 + SequenceGroupMetadata(
174 + request_id=str(seq_id),
175 + is_prompt=True,
176 + do_sample=idx + chunk_size >= len(prompt), # terminal chunk
177 + seq_data=seq_data,
178 + sampling_params=SamplingParams(temperature=0.0),
179 + block_tables={i: block_allocations},
180 + token_chunk_size=len(chunk_ids)))
181 + return seq_group_metadata_list
182 +
183 +
184 def assert_logprobs_dict_allclose(
185 actual_logprobs: List[Dict[int, Logprob]],
186 expected_logprobs: List[Dict[int, Logprob]]) -> None:
@@ -198,7 +233,8 @@ def create_batch(batch_size,
233 prev_output_token_len: int = 10,
234 seq_ids: Optional[List[int]] = None,
235 num_gpu_blocks: Optional[int] = None,
201 - block_size: Optional[int] = None):
236 + block_size: Optional[int] = None,
237 + prefill_chunk_size: Optional[int] = None):
238 if block_size is None:
239 block_size = 8
240
@@ -213,15 +249,28 @@ def create_batch(batch_size,
249 prompt_lens = prompt_len
250
251 prompts = [[next(iterator) for _ in range(p_len)] for p_len in prompt_lens]
216 - prev_output_tokens = [[
217 - next(iterator) for _ in range(prev_output_token_len)
218 - ] for _ in range(batch_size)]
219 - final_prompt_lens = [
220 - len(prompt) + len(prev_output_token) + k + 1
221 - for prompt, prev_output_token in zip(prompts, prev_output_tokens)
222 - ]
252
224 - seq_group_metadata_list = create_seq_group_metadata_from_prompts(
225 - prompts, num_gpu_blocks, block_size, final_prompt_lens,
226 - prev_output_tokens, seq_ids)
253 + if prefill_chunk_size:
254 + # Create a batch of chunked prompts.
255 + if not seq_ids:
256 + seq_ids = list(range(len(prompts)))
257 + seq_group_metadata_list = []
258 + for p, sid in zip(prompts, seq_ids):
259 + seq_group_metadata_list += \
260 + create_chunked_seq_group_metadata_from_prompt(
261 + p, num_gpu_blocks, prefill_chunk_size, block_size, sid)
262 + seq_group_metadata_list = seq_group_metadata_list[:batch_size]
263 + prev_output_tokens = []
264 + else:
265 + prev_output_tokens = [[
266 + next(iterator) for _ in range(prev_output_token_len)
267 + ] for _ in range(batch_size)]
268 + final_prompt_lens = [
269 + len(prompt) + len(prev_output_token) + k + 1
270 + for prompt, prev_output_token in zip(prompts, prev_output_tokens)
271 + ]
272 +
273 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
274 + prompts, num_gpu_blocks, block_size, final_prompt_lens,
275 + prev_output_tokens, seq_ids)
276 return seq_group_metadata_list, prompts, prev_output_tokens
vllm/attention/backends/flash_attn.py
+7 -3
@@ -276,7 +276,11 @@ class FlashAttentionMetadata(AttentionMetadata):
276 max_query_len=self.max_query_len,
277 max_prefill_seq_len=0,
278 max_decode_seq_len=self.max_decode_seq_len,
279 - query_start_loc=self.query_start_loc[self.num_prefills:]
279 + # Batch may be composed of prefill|decodes, adjust query start
280 + # indices to refer to the start of decodes. E.g.
281 + # in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
282 + query_start_loc=(self.query_start_loc[self.num_prefills:] -
283 + self.query_start_loc[self.num_prefills])
284 if self.query_start_loc is not None else None,
285 seq_start_loc=self.seq_start_loc[self.num_prefills:]
286 if self.seq_start_loc is not None else None,
@@ -903,7 +907,9 @@ def unified_flash_attention(
907 # Decoding run.
908 # Use flash_attn_varlen_func kernel for speculative decoding
909 # because different queries might have different lengths.
910 +
911 assert decode_meta.max_decode_query_len is not None
912 + # use only for actual varlen decoding
913 if decode_meta.max_decode_query_len > 1:
914 assert attn_type == AttentionType.DECODER, (
915 "Only decoder-only models support max_decode_query_len > 1")
@@ -949,8 +955,6 @@ def unified_flash_attention(
955 assert prefill_output is not None
956 return prefill_output.view(num_prefill_query_tokens, hidden_size)
957
952 - # Chunked prefill does not work with speculative decoding.
953 - # Therefore, the query length for decode should be 1 in chunked prefill.
958 assert decode_meta is not None
959 decode_output = decode_output.squeeze(1)
960 output = torch.cat([prefill_output, decode_output], dim=0)
vllm/attention/backends/rocm_flash_attn.py
+6
@@ -192,6 +192,12 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
192 block_tables=self.block_tables[self.num_prefills:],
193 use_cuda_graph=self.use_cuda_graph,
194 )
195 + # Batch may be composed of prefill|decodes, adjust query start indices
196 + # to refer to the start of decodes when the two are split apart.
197 + # E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
198 + if self._cached_decode_metadata.query_start_loc is not None:
199 + qs = self._cached_decode_metadata.query_start_loc
200 + self._cached_decode_metadata.query_start_loc = qs - qs[0]
201 return self._cached_decode_metadata
202
203 def advance_step(self,
vllm/attention/backends/xformers.py
+7
@@ -272,6 +272,13 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
272 max_encoder_seq_len=self.max_encoder_seq_len,
273 cross_slot_mapping=self.cross_slot_mapping,
274 cross_block_tables=self.cross_block_tables)
275 +
276 + # Batch may be composed of prefill|decodes, adjust query start indices
277 + # to refer to the start of decodes when the two are split apart.
278 + # E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
279 + if self._cached_decode_metadata.query_start_loc is not None:
280 + qs = self._cached_decode_metadata.query_start_loc
281 + self._cached_decode_metadata.query_start_loc = qs - qs[0]
282 return self._cached_decode_metadata
283
284
vllm/config.py
+6 -8
@@ -192,7 +192,6 @@ class ModelConfig:
192 self.max_logprobs = max_logprobs
193 self.disable_sliding_window = disable_sliding_window
194 self.skip_tokenizer_init = skip_tokenizer_init
195 -
195 self.hf_config = get_config(self.model, trust_remote_code, revision,
196 code_revision, rope_scaling, rope_theta,
197 config_format)
@@ -1317,13 +1316,6 @@ class SpeculativeConfig:
1316 "speculative decoding is > 1, but got "
1317 f"{speculative_disable_by_batch_size=}")
1318
1320 - # Reminder: Please update docs/source/serving/compatibility_matrix.rst
1321 - # If the feature combo become valid
1322 - if enable_chunked_prefill:
1323 - raise ValueError(
1324 - "Speculative decoding and chunked prefill are "
1325 - f"currently mutually exclusive ({enable_chunked_prefill=}).")
1326 -
1319 # TODO: The user should be able to specify revision/max model len
1320 # for the draft model. It is not currently supported.
1321 draft_revision = None
@@ -1390,6 +1382,12 @@ class SpeculativeConfig:
1382 f"num_speculative_tokens={n_predict}, but "
1383 f"{num_speculative_tokens=} was provided.")
1384
1385 + if enable_chunked_prefill and draft_hf_config.model_type in (
1386 + "medusa", "mlp_speculator", "eagle"):
1387 + raise ValueError(
1388 + "Chunked prefill and hidden-state based draft models are "
1389 + "not compatible.")
1390 +
1391 draft_model_config.max_model_len = (
1392 SpeculativeConfig._maybe_override_draft_max_model_len(
1393 speculative_max_model_len,
vllm/core/scheduler.py
+1
@@ -1147,6 +1147,7 @@ class Scheduler:
1147
1148 # Update swapped requests.
1149 self.swapped.extend(running_scheduled.swapped_out)
1150 + # Put prefills first due to Attention backend ordering assumption.
1151 return SchedulerOutputs(
1152 scheduled_seq_groups=(prefills.seq_groups +
1153 running_scheduled.prefill_seq_groups +
vllm/engine/output_processor/multi_step.py
+5 -3
@@ -134,10 +134,12 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
134 sample for sample in samples
135 if sample.output_token != VLLM_INVALID_TOKEN_ID
136 ]
137 - assert valid_samples
137
139 - self._process_seq_outputs(seq, valid_samples,
140 - sequence_group.sampling_params)
138 + # When both spec-decode and pre-fill chunking are enabled, we
139 + # don't have guaranteed samples here (e.g. all -1s).
140 + if valid_samples:
141 + self._process_seq_outputs(seq, valid_samples,
142 + sequence_group.sampling_params)
143
144 def _process_decode_and_stop(self, seq: Sequence,
145 sampling_params: SamplingParams) -> None:
vllm/spec_decode/batch_expansion.py
+31 -30
@@ -90,7 +90,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
90 else:
91 # Batch has a mix of spec decode enabled and disabled seq groups
92 contracted = self._contract_batch(
93 - contracted_bs=len(execute_model_req.seq_group_metadata_list),
93 + execute_model_req.seq_group_metadata_list,
94 target_sampler_output=target_sampler_output,
95 proposals=proposals,
96 num_scoring_tokens=num_scoring_tokens,
@@ -126,7 +126,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
126 split_batch_by_proposal_len(
127 seq_group_metadata_list, proposal_lens_list)
128
129 - target_seq_group_metadata_list = self._create_scoring_model_input(
129 + spec_expanded_seqs = self._create_scoring_model_input(
130 seq_group_metadata_list=spec_seqs,
131 proposal_token_ids=proposal_token_ids_list,
132 # NOTE: We determine the seq ids in the expanded batch using the
@@ -135,16 +135,19 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
135 seq_ids=get_all_seq_ids(seq_group_metadata_list)),
136 )
137
138 - num_scoring_tokens = len(target_seq_group_metadata_list)
139 - target_seq_group_metadata_list.extend(non_spec_seqs)
138 + num_scoring_tokens = len(spec_expanded_seqs)
139 + # Batch speculative and non-speculative (e.g. chunked prefill) requests
140 + # but make sure order is prefill|decode due to backend requirement.
141 + target_seq_group_metadata_list = non_spec_seqs + spec_expanded_seqs
142
143 return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
144 num_scoring_tokens)
145
146 def _contract_batch(
145 - self, contracted_bs: int, target_sampler_output: SamplerOutput,
146 - proposals: SpeculativeProposals, num_scoring_tokens: int,
147 - non_spec_indices: List[int], spec_indices: List[int], k: int
147 + self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
148 + target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
149 + num_scoring_tokens: int, non_spec_indices: List[int],
150 + spec_indices: List[int], k: int
151 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
152 Optional[torch.Tensor]]:
153 """Contract the expanded batch back into its original size.
@@ -154,6 +157,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
157 contracted_bs is the original batch size, and the batch size that the
158 target_sampler_output will be contracted to.
159 """
160 + contracted_bs = len(contracted_seq_group_metadata_list)
161 (target_token_ids, target_probs, target_logprobs, target_hidden_states,
162 non_spec_target_token_ids, non_spec_target_probs,
163 non_spec_target_logprobs,
@@ -166,8 +170,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
170
171 # The number of tokens in the expanded batch used for speculation is
172 # equal to the total expanded batch size minus the number of samples for
169 - # non-speculative sequences.
170 - non_spec_expanded_bs = len(non_spec_target_token_ids)
173 + # non-speculative sequences, prefill chunks with no out tokens included
174 + non_spec_expanded_bs = len(non_spec_indices)
175 spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
176
177 target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
@@ -191,7 +195,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
195 else:
196 all_hidden_states = None
197
194 - if non_spec_indices:
198 + # Rule out prefills that produce no tokens.
199 + non_spec_indices = [
200 + idx for idx in non_spec_indices
201 + if contracted_seq_group_metadata_list[idx].do_sample
202 + ]
203 + if len(non_spec_indices):
204 all_tokens[non_spec_indices, :1] = \
205 non_spec_target_token_ids.unsqueeze(1)
206 all_probs[non_spec_indices, :1, :] = \
@@ -290,9 +299,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
299 This function creates K+1 target SequenceGroupMetadata to take
300 advantage of the bonus token.
301 """
293 - assert not input_seq_group_metadata.is_prompt, (
294 - "Speculating on "
295 - "prompts not yet supported")
302 assert len(input_seq_group_metadata.seq_data) == 1, (
303 "Beam search "
304 "not supported in speculative decoding")
@@ -390,27 +396,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
396 # and non spec sequences) and should be removed in the future. It can be
397 # done by supporting per-sequence proposal lens.
398 #
393 - # First samples are from speculative scoring, latter samples are non-
394 - # speculative samples.
395 - split_sizes = (num_scoring_tokens,
396 - sampler_output.sampled_token_ids.numel() -
397 - num_scoring_tokens)
398 - (spec_probs, non_spec_probs
399 - ) = sampler_output.sampled_token_probs.split(split_sizes)
400 - (spec_sampled_tokens, non_spec_sampled_tokens
399 + # First samples are non-speculative, latter samples are from speculative
400 + # scoring (prefill|decode order).
401 + split_sizes = (sampler_output.sampled_token_ids.numel() -
402 + num_scoring_tokens, num_scoring_tokens)
403 + (non_spec_probs,
404 + spec_probs) = sampler_output.sampled_token_probs.split(split_sizes)
405 + (non_spec_sampled_tokens, spec_sampled_tokens
406 ) = sampler_output.sampled_token_ids.flatten().split(split_sizes)
402 - (
403 - spec_logprobs,
404 - non_spec_logprobs,
405 - ) = sampler_output.logprobs.split(split_sizes)
407 + (non_spec_logprobs,
408 + spec_logprobs) = sampler_output.logprobs.split(split_sizes)
409
410 if sampler_output.hidden_states is not None:
408 - (
409 - spec_hidden_states,
410 - non_spec_hidden_states,
411 - ) = sampler_output.hidden_states.split(split_sizes)
411 + (non_spec_hidden_states, spec_hidden_states
412 + ) = sampler_output.hidden_states.split(split_sizes)
413 else:
413 - spec_hidden_states, non_spec_hidden_states = None, None
414 + non_spec_hidden_states, spec_hidden_states = None, None
415
416 return (spec_sampled_tokens, spec_probs, spec_logprobs,
417 spec_hidden_states, non_spec_sampled_tokens, non_spec_probs,
vllm/spec_decode/mqa_scorer.py
+19 -12
@@ -21,6 +21,11 @@ class MQAScorer(SpeculativeScorer):
21 all_proposal_lengths = proposals.proposal_lens.tolist()
22 for i, seq_group_metadata in enumerate(
23 execute_model_req.seq_group_metadata_list):
24 + if all_proposal_lengths[i] == 0:
25 + # Keep prompt seqs untouched (keep computed_tokens for chunks).
26 + target_seq_group_metadata_list.append(seq_group_metadata)
27 + continue
28 +
29 seq_data_dict = seq_group_metadata.seq_data
30 assert len(seq_data_dict) == 1
31 seq_id = next(iter(seq_data_dict.keys()))
@@ -40,8 +45,7 @@ class MQAScorer(SpeculativeScorer):
45 new_seq_data.update_num_computed_tokens(
46 len(prompt_token_ids) + len(output_token_ids) - 1)
47
43 - # Ensure that the new sequence has at least one token
44 - # because we only use mqa scorer in the decoding stage.
48 + # Ensure that the new decode sequence has at least one token.
49 assert len(output_token_ids) >= 1
50 new_seq_data_dict = {target_seq_id: new_seq_data}
51
@@ -54,7 +58,6 @@ class MQAScorer(SpeculativeScorer):
58 target_seq_id: seq_group_metadata.block_tables[seq_id],
59 },
60 lora_request=None,
57 - token_chunk_size=1,
61 )
62 target_seq_group_metadata_list.append(new_seq_group_metadata)
63
@@ -77,6 +80,7 @@ class MQAScorer(SpeculativeScorer):
80 all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
81 all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size)
82 else:
83 + # We either have decodes with different lens or prefill+decodes.
84 all_tokens = target_token_ids.new_full(size=(bs, k + 1),
85 fill_value=-1)
86 all_probs = target_probs.new_zeros(*all_tokens.shape,
@@ -85,15 +89,18 @@ class MQAScorer(SpeculativeScorer):
89 fill_value=-float("inf"))
90 target_token_ids = target_token_ids.flatten()
91 start_loc = 0
88 - for i, proposed_len in enumerate(all_proposal_lengths):
89 - output_len = proposed_len + 1
90 - end_loc = start_loc + output_len
91 - all_tokens[
92 - i, :output_len] = target_token_ids[start_loc:end_loc]
93 - all_probs[i, :output_len] = target_probs[start_loc:end_loc]
94 - all_logprobs[
95 - i, :output_len] = target_logprobs[start_loc:end_loc]
96 - start_loc = end_loc
92 + for i, (proposed_len, seq_meta) in enumerate(
93 + zip(all_proposal_lengths, target_seq_group_metadata_list)):
94 + # Skip chunks with no output tokens.
95 + if seq_meta.do_sample:
96 + output_len = proposed_len + 1
97 + end_loc = start_loc + output_len
98 + all_tokens[
99 + i, :output_len] = target_token_ids[start_loc:end_loc]
100 + all_probs[i, :output_len] = target_probs[start_loc:end_loc]
101 + all_logprobs[
102 + i, :output_len] = target_logprobs[start_loc:end_loc]
103 + start_loc = end_loc
104
105 hidden_states = None
106 if target_sampler_output.hidden_states is not None:
vllm/spec_decode/spec_decode_worker.py
+78 -32
@@ -418,7 +418,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
418 # none of the requests in the batch have spec decoding enabled.
419 # In any of these cases, the proposer and scorer workers
420 # are called normally.
421 - no_spec = num_lookahead_slots == 0 or disable_all_speculation or all(
421 + # We expect `num_speculative_tokens` to be None for prefills.
422 + no_spec = all(
423 + sgm.is_prompt for sgm in execute_model_req.seq_group_metadata_list
424 + ) or num_lookahead_slots == 0 or disable_all_speculation or all(
425 sgm.num_speculative_tokens == 0
426 for sgm in execute_model_req.seq_group_metadata_list)
427
@@ -484,7 +487,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
487
488 def _serialize_sampler_output_no_logprobs(
489 self, execute_model_req: ExecuteModelRequest,
487 - sampler_output: SamplerOutput) -> SamplerOutput:
490 + sampler_output: SamplerOutput) -> List[SamplerOutput]:
491 """
492 Creates and returns a `SamplerOutput` with only the token IDs being
493 serialized to CPU and populated in `CompletionSequenceGroupOutput`.
@@ -514,41 +517,56 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
517 if any(seq_output_prompt_logprobs) else \
518 sampler_output.sampled_token_ids).tolist()
519
517 - seq_data_entries = (
520 + seq_data_entries = [
521 (seq_id, seq_data) for sg in \
522 execute_model_req.seq_group_metadata_list \
523 for seq_id, seq_data in sg.seq_data.items()
521 - )
524 + if sg.do_sample # ignore empty token sequences
525 + ]
526 completion_seq_group_output_list: List[
527 CompletionSequenceGroupOutput] = []
524 - for index, ((seq_id, seq_data), needs_prompt_logprobs) in \
525 - enumerate(zip(seq_data_entries, seq_output_prompt_logprobs)):
526 - if needs_prompt_logprobs:
527 - prompt_token_ids = seq_data.get_prompt_token_ids()
528 - prompt_logprobs = [
529 - create_logprobs_output(
530 - token_id=p_token_id,
528 + output_index = 0
529 + # Make sure the non-terminal prefill chunks are still aligned with
530 + # their own empty output.
531 + for seq_group_meta in execute_model_req.seq_group_metadata_list:
532 + # Since we can get chunks here, we dont always have a sampled token
533 + # (only on last chunk) but we still have to provide an output.
534 + if not seq_group_meta.do_sample:
535 + completion_seq_group_output_list.append(
536 + CompletionSequenceGroupOutput(samples=[],
537 + prompt_logprobs=None))
538 + else:
539 + # Sequence with output.
540 + seq_id, seq_data = seq_data_entries[output_index]
541 + needs_prompt_logprobs = seq_output_prompt_logprobs[
542 + output_index]
543 + if needs_prompt_logprobs:
544 + prompt_token_ids = seq_data.get_prompt_token_ids()
545 + prompt_logprobs = [
546 + create_logprobs_output(
547 + token_id=p_token_id,
548 + token_id_logprob_rank=-1,
549 + token_id_logprob=0.0,
550 + topk_token_ids=[],
551 + topk_logprobs=[],
552 + )
553 + # no prompt logprobs for the first token
554 + for p_token_id in prompt_token_ids[1:]
555 + ]
556 + else:
557 + prompt_logprobs = None
558 + completion_seq_group_output_list.append(
559 + create_sequence_group_output(
560 + token_id=sampled_token_ids_list[output_index][0],
561 token_id_logprob_rank=-1,
562 token_id_logprob=0.0,
563 + seq_id=seq_id,
564 topk_token_ids=[],
565 topk_logprobs=[],
535 - )
536 - # no prompt logprobs for the first token
537 - for p_token_id in prompt_token_ids[1:]
538 - ]
539 - else:
540 - prompt_logprobs = None
541 -
542 - completion_seq_group_output_list.append(
543 - create_sequence_group_output(
544 - token_id=sampled_token_ids_list[index][0],
545 - token_id_logprob_rank=-1,
546 - token_id_logprob=0.0,
547 - seq_id=seq_id,
548 - topk_token_ids=[],
549 - topk_logprobs=[],
550 - prompt_logprobs=prompt_logprobs))
551 - return SamplerOutput(outputs=completion_seq_group_output_list)
566 + prompt_logprobs=prompt_logprobs))
567 + output_index += 1
568 +
569 + return [SamplerOutput(outputs=completion_seq_group_output_list)]
570
571 @nvtx_range("spec_decode_worker._run_no_spec")
572 def _run_no_spec(self, execute_model_req: ExecuteModelRequest,
@@ -568,6 +586,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
586 hidden_states = sampler_output.hidden_states
587 if hidden_states is not None:
588 # remove hidden_states for prompt tokens
589 + # TODO Enable `return_hidden_states`: prefill chunks hidden states
590 + # are pruned by the logits processor. Also, they should be arranged
591 + # back into full-prefill latent. Address it to enable MLPSpeculator.
592 if any(seq.is_prompt
593 for seq in execute_model_req.seq_group_metadata_list):
594 hidden_states = hidden_states[
@@ -593,14 +614,14 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
614 sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
615 execute_model_req=execute_model_req, sampler_output=sampler_output)
616 if self._disable_logprobs else
596 - sampler_output)
617 + [sampler_output])
618
619 # Clear device tensors from sampler output. This reduces communication
620 # overhead when the engine runs in a different process than the workers.
621 sampler_output.sampled_token_probs = None
622 sampler_output.sampled_token_ids = None
623 sampler_output.logprobs = None
603 - return [sampler_output_to_return]
624 + return sampler_output_to_return
625
626 def _run_non_driver_rank(self) -> bool:
627 """Run proposer and verifier model in non-driver workers. This is used
@@ -644,9 +665,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
665 This invokes the proposer worker to get k speculative tokens for each
666 sequence, then scores each speculative token using the scoring worker.
667
668 + When `enable_chunked_prefill` is set, scorer will batch decodes and
669 + prefills, while proposer will sync its KV-cache by running an extra
670 + forward on prefills.
671 +
672 Returns a list of SamplerOutput, each containing a single token per
673 sequence.
674 """
675 + # With prefill chunking, expect requests to have prompts first
676 + # so that backend gets prefill|decode.
677 assert num_lookahead_slots == execute_model_req.num_lookahead_slots
678
679 # Pass last hidden states from target model to proposer
@@ -671,6 +698,25 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
698 proposals,
699 )
700
701 + _, (non_spec_seqs, non_spec_indices) = split_batch_by_proposal_len(
702 + execute_model_req.seq_group_metadata_list, proposals.proposal_lens)
703 + # With prefill chunking enabled, `non_spec_seqs` contains prefills too:
704 + # discard decodes that have already been processed by proposer.
705 + non_spec_indices = [
706 + idx for idx in non_spec_indices
707 + if execute_model_req.seq_group_metadata_list[idx].is_prompt
708 + ]
709 + if len(non_spec_indices):
710 + all_hidden_states = proposal_scores.hidden_states
711 + # TODO fix `return_hidden_states`, same as in `_run_no_spec`
712 + if all_hidden_states is not None:
713 + prefill_hidden_states = all_hidden_states[non_spec_indices]
714 + execute_model_req.previous_hidden_states = \
715 + prepare_prefill_hidden_states(prefill_hidden_states)
716 + # Sync proposer KV cache for prefills.
717 + prefill_req = execute_model_req.clone(non_spec_seqs)
718 + self.proposer_worker.execute_model(prefill_req)
719 +
720 with Timer() as verification_timer:
721 accepted_token_ids, target_logprobs = self._verify_tokens(
722 execute_model_req.seq_group_metadata_list, proposal_scores,
@@ -769,7 +815,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
815 self.previous_hidden_states = HiddenStates(
816 hidden_states, seq_group_metadata_list,
817 second_last_token_hidden_states)
772 -
818 return accepted_token_ids, logprobs
819
820 def _create_output_sampler_list(
@@ -819,6 +864,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
864 accepted_token_ids_by_step = accepted_token_ids_by_step.tolist()
865
866 # Construct the output on a per-step, per-sequence basis.
867 + # Non-terminal prefill chunks will end up here as rows with just -1s
868 + # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
869 sampler_output_list: List[SamplerOutput] = []
870 for step_index in range(num_steps):
871 if all(token_id == -1
@@ -861,7 +908,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
908 # This is periodic because the rejection sampler emits metrics
909 # periodically.
910 self._maybe_log_stage_times(*stage_times)
864 -
911 return sampler_output_list
912
913 def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,
vllm/spec_decode/top1_proposer.py
+4 -4
@@ -109,7 +109,6 @@ class Top1Proposer(SpeculativeProposer):
109 proposal_probs=proposal_probs,
110 proposal_lens=proposal_lens,
111 no_proposals=maybe_sampler_output is None)
112 -
112 return proposals
113
114 def _split_by_proposal_len(
@@ -127,9 +126,10 @@ class Top1Proposer(SpeculativeProposer):
126 nonzero_proposal_len_seqs: List[SequenceGroupMetadata] = []
127 nonzero_proposal_len_indices: List[int] = []
128 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
130 - # The speculative decoding for this request has been disabled
131 - # (e.g. due to high traffic).
132 - if seq_group_metadata.num_speculative_tokens == 0:
129 + # The speculative decoding for this request has either been disabled
130 + # (e.g. due to high traffic) or this is a prompt request.
131 + if (seq_group_metadata.is_prompt
132 + or seq_group_metadata.num_speculative_tokens == 0):
133 proposal_lens.append(0)
134 continue
135