@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6116ca8cd7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Feature] [Spec decode]: Enable MLPSpeculator/Medusa and `prompt_logprobs` with ChunkedPrefill (#10132)

Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: wallashss <wallashss@ibm.com> Co-authored-by: wallashss <wallashss@ibm.com>

Nicolò Lucchesi committed Jan 27, 2025 at 22:38 UTC 6116ca8cd79b642c64f4ae6f050a6bc12b96d037
16 files changed +469 -166
tests/spec_decode/e2e/conftest.py
+16 -3
@@ -2,6 +2,7 @@ from itertools import cycle
2 from typing import List, Optional, Sequence, Tuple, Union
3
4 import pytest
5 +import torch
6
7 from vllm import LLM, SamplingParams
8 from vllm.distributed import cleanup_dist_env_and_memory
@@ -154,6 +155,8 @@ def _check_logprobs_when_output_disabled(
155 spec_pos_logprob) = next(iter(spec_pos_logprobs.items()))
156 assert spec_pos_logprob.rank == -1
157 assert spec_pos_logprob.logprob == 0.0
158 + if isinstance(spec_pos_logprob_token_id, torch.Tensor):
159 + spec_pos_logprob_token_id = spec_pos_logprob_token_id.item()
160 assert spec_pos_logprob_token_id in baseline_pos_logprobs
161
162
@@ -244,7 +247,8 @@ def run_equality_correctness_test_tp(model,
247 batch_size: int,
248 max_output_len: int,
249 seed: int = 0,
247 - temperature: float = 0.0):
250 + temperature: float = 0.0,
251 + logprobs: Optional[int] = None):
252 """Helper method that compares the outputs of both the baseline LLM and
253 the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
254 the same when temperature is zero.
@@ -257,7 +261,6 @@ def run_equality_correctness_test_tp(model,
261 results = []
262
263 prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
260 -
264 for args, env in ((arg1, env1), (arg2, env2)):
265 with RemoteOpenAIServer(model,
266 args,
@@ -269,12 +272,14 @@ def run_equality_correctness_test_tp(model,
272 prompt=prompts,
273 max_tokens=max_output_len,
274 seed=seed,
272 - temperature=temperature)
275 + temperature=temperature,
276 + logprobs=logprobs)
277
278 results.append({
279 "test":
280 "seeded_sampling",
281 "text": [choice.text for choice in completion.choices],
282 + "logprobs": [choice.logprobs for choice in completion.choices],
283 "finish_reason":
284 [choice.finish_reason for choice in completion.choices],
285 "usage":
@@ -284,7 +289,15 @@ def run_equality_correctness_test_tp(model,
289 n = len(results) // 2
290 arg1_results = results[:n]
291 arg2_results = results[n:]
292 + # Separate logprobs to avoid asserting exact equality.
293 + arg1_logprobs = [r.pop("logprobs") for r in arg1_results]
294 + arg2_logprobs = [r.pop("logprobs") for r in arg2_results]
295 +
296 for arg1_result, arg2_result in zip(arg1_results, arg2_results):
297 assert arg1_result == arg2_result, (
298 f"Results for {model=} are not the same with {arg1=} and {arg2=}. "
299 f"{arg1_result=} != {arg2_result=}")
300 + if logprobs:
301 + for logs1, logs2 in zip(arg1_logprobs, arg2_logprobs):
302 + for l1, l2 in zip(logs1, logs2):
303 + assert l1.tokens == l2.tokens
tests/spec_decode/e2e/test_integration_dist_tp2.py
+9 -1
@@ -2,6 +2,8 @@
2 tensor parallelism.
3 """
4
5 +from typing import Optional
6 +
7 import pytest
8 import torch
9
@@ -154,15 +156,20 @@ def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
156 "--speculative-draft-tensor-parallel-size",
157 "1",
158 ])])
159 +@pytest.mark.parametrize("logprobs", [None, 2])
160 @pytest.mark.parametrize("batch_size", [2])
161 @pytest.mark.parametrize("seed", [1])
162 def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
163 per_test_common_llm_kwargs,
164 baseline_llm_kwargs, test_llm_kwargs,
165 + logprobs: Optional[int],
166 batch_size: int, seed: int):
167 """Verify spec decode works well with same and different TP size for
168 the draft model with chunked prefill.
169 """
170 + if logprobs:
171 + test_llm_kwargs.extend(
172 + ["--disable_logprobs_during_spec_decoding", "False"])
173 run_equality_correctness_test_tp(model,
174 common_llm_kwargs,
175 per_test_common_llm_kwargs,
@@ -171,4 +178,5 @@ def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
178 batch_size,
179 max_output_len=32,
180 seed=seed,
174 - temperature=0.0)
181 + temperature=0.0,
182 + logprobs=logprobs)
tests/spec_decode/e2e/test_logprobs.py
+10 -6
@@ -4,26 +4,27 @@ import pytest
4
5 from vllm import SamplingParams
6
7 +from ..utils import maybe_enable_chunked_prefill
8 from .conftest import run_equality_correctness_test
9
10
11 @pytest.mark.parametrize(
12 "common_llm_kwargs",
13 [{
13 - "model_name": "JackFram/llama-68m",
14 + "model_name": "JackFram/llama-160m",
15
16 # Skip cuda graph recording for fast test.
16 - "enforce_eager": True,
17 + "enforce_eager": True
18 }])
19 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
20 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
21 @pytest.mark.parametrize("test_llm_kwargs",
22 [{
22 - "speculative_model": "JackFram/llama-160m",
23 + "speculative_model": "JackFram/llama-68m",
24 "num_speculative_tokens": 3,
25 "disable_logprobs_during_spec_decoding": False,
26 }, {
26 - "speculative_model": "JackFram/llama-160m",
27 + "speculative_model": "JackFram/llama-68m",
28 "num_speculative_tokens": 3,
29 "disable_logprobs_during_spec_decoding": True,
30 }])
@@ -36,12 +37,15 @@ from .conftest import run_equality_correctness_test
37 ])
38 @pytest.mark.parametrize("seed", [1])
39 @pytest.mark.parametrize("logprobs", [1, 6])
40 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4, 12])
41 def test_logprobs_equality(vllm_runner, common_llm_kwargs,
42 per_test_common_llm_kwargs, baseline_llm_kwargs,
43 test_llm_kwargs, batch_size: int, output_len: int,
42 - seed: int, logprobs: int):
43 - """Verify output logprobs are equal with and without speculative decoding.
44 + seed: int, logprobs: int, prefill_chunk_size: int):
45 + """Verify output logprobs are equal with and without speculative decoding,
46 + as well as with and without chunked prefill.
47 """
48 + maybe_enable_chunked_prefill(prefill_chunk_size, common_llm_kwargs)
49 run_equality_correctness_test(vllm_runner,
50 common_llm_kwargs,
51 per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_medusa_correctness.py
+24 -7
@@ -21,6 +21,7 @@ correctess for the target model outputs.
21
22 import pytest
23
24 +from ..utils import maybe_enable_chunked_prefill
25 from .conftest import run_equality_correctness_test
26
27 # main model
@@ -67,12 +68,14 @@ PRECISION = "float32"
68 ])
69 @pytest.mark.parametrize("batch_size", [1, 32])
70 @pytest.mark.parametrize("seed", [1])
71 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
72 def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
73 per_test_common_llm_kwargs,
74 baseline_llm_kwargs, test_llm_kwargs,
75 batch_size: int, output_len: int,
74 - seed: int):
76 + seed: int, prefill_chunk_size: int):
77 """Verify greedy equality with different batch size."""
78 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
79 run_equality_correctness_test(vllm_runner,
80 common_llm_kwargs,
81 per_test_common_llm_kwargs,
@@ -119,12 +122,15 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
122 @pytest.mark.parametrize("batch_size", [8])
123 @pytest.mark.parametrize("seed", [1])
124 @pytest.mark.parametrize("logprobs", [1, 6])
125 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
126 def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
127 per_test_common_llm_kwargs,
128 baseline_llm_kwargs, test_llm_kwargs,
129 batch_size: int, output_len: int,
126 - seed: int, logprobs: int):
130 + seed: int, logprobs: int,
131 + prefill_chunk_size: int):
132 """Verify greedy equality with different batch size."""
133 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
134 run_equality_correctness_test(vllm_runner,
135 common_llm_kwargs,
136 per_test_common_llm_kwargs,
@@ -167,12 +173,14 @@ def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
173 ])
174 @pytest.mark.parametrize("batch_size", [1, 32])
175 @pytest.mark.parametrize("seed", [1])
176 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
177 def test_medusa_e2e_greedy_correctness_cuda_graph(
178 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
179 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
173 - seed: int):
180 + seed: int, prefill_chunk_size: int):
181 """Verify greedy equality with cuda graph enabled and different
182 batch sizes."""
183 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
184 run_equality_correctness_test(vllm_runner,
185 common_llm_kwargs,
186 per_test_common_llm_kwargs,
@@ -217,13 +225,15 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(
225 ])
226 @pytest.mark.parametrize("batch_size", [4])
227 @pytest.mark.parametrize("seed", [1])
228 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
229 def test_medusa_e2e_greedy_correctness_with_preemption(
230 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
231 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
223 - seed: int):
232 + seed: int, prefill_chunk_size: int):
233 """Verify greedy equality, even when some sequences are preempted mid-
234 generation.
235 """
236 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
237 run_equality_correctness_test(vllm_runner,
238 common_llm_kwargs,
239 per_test_common_llm_kwargs,
@@ -267,13 +277,15 @@ def test_medusa_e2e_greedy_correctness_with_preemption(
277 32,
278 ])
279 @pytest.mark.parametrize("seed", [1])
280 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
281 def test_medusa_different_k(vllm_runner, common_llm_kwargs,
282 per_test_common_llm_kwargs, baseline_llm_kwargs,
283 test_llm_kwargs, batch_size: int, output_len: int,
273 - seed: int):
284 + seed: int, prefill_chunk_size: int):
285 """Verify that medusa speculative decoding produces exact equality
286 to without spec decode with different values of num_speculative_tokens.
287 """
288 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
289 run_equality_correctness_test(vllm_runner,
290 common_llm_kwargs,
291 per_test_common_llm_kwargs,
@@ -313,14 +325,17 @@ def test_medusa_different_k(vllm_runner, common_llm_kwargs,
325 32,
326 ])
327 @pytest.mark.parametrize("seed", [1])
328 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
329 def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
330 per_test_common_llm_kwargs, baseline_llm_kwargs,
331 test_llm_kwargs, batch_size: int,
319 - output_len: int, seed: int):
332 + output_len: int, seed: int,
333 + prefill_chunk_size: int):
334 """Verify that medusa speculative decoding produces exact equality
335 to without spec decode when speculation is disabled for large
336 batch sizes.
337 """
338 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
339 run_equality_correctness_test(vllm_runner,
340 common_llm_kwargs,
341 per_test_common_llm_kwargs,
@@ -361,12 +376,14 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
376 32,
377 ])
378 @pytest.mark.parametrize("seed", [1])
379 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
380 def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
381 baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
366 - output_len: int, seed: int):
382 + output_len: int, seed: int, prefill_chunk_size: int):
383 """Verify that speculative decoding generates the same output
384 with batch expansion scorer and mqa scorer.
385 """
386 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
387 run_equality_correctness_test(vllm_runner,
388 common_llm_kwargs,
389 per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_mlp_correctness.py
+42 -11
@@ -25,6 +25,7 @@ import pytest
25
26 from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
27
28 +from ..utils import maybe_enable_chunked_prefill
29 from .conftest import run_equality_correctness_test
30
31 # main model
@@ -66,14 +67,16 @@ PRECISION = "float32"
67 @pytest.mark.parametrize("output_len", [
68 128,
69 ])
69 -@pytest.mark.parametrize("batch_size", [1, 32])
70 +@pytest.mark.parametrize("batch_size", [4, 32])
71 @pytest.mark.parametrize("seed", [1])
72 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
73 def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
74 per_test_common_llm_kwargs,
75 baseline_llm_kwargs, test_llm_kwargs,
76 batch_size: int, output_len: int,
75 - seed: int):
77 + seed: int, prefill_chunk_size: int):
78 """Verify greedy equality with different batch size."""
79 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
80 run_equality_correctness_test(vllm_runner,
81 common_llm_kwargs,
82 per_test_common_llm_kwargs,
@@ -116,12 +119,19 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
119 @pytest.mark.parametrize("batch_size", [8])
120 @pytest.mark.parametrize("seed", [1])
121 @pytest.mark.parametrize("logprobs", [1, 6])
122 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
123 def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
124 per_test_common_llm_kwargs,
125 baseline_llm_kwargs, test_llm_kwargs,
126 batch_size: int, output_len: int, seed: int,
123 - logprobs: int):
127 + logprobs: int, prefill_chunk_size: int):
128 """Verify greedy equality with different batch size."""
129 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
130 + # NOTE Test is sensitive enough st if we don't enable chunked prefill
131 + # scheduling on baseline too, we get slightly different logprobs, ending
132 + # up sampling different tokens at the tail (ie top tokens don't change).
133 + # TL;DR: sd+cp == org+cp but sd+cp != org..is this expected?
134 + maybe_enable_chunked_prefill(prefill_chunk_size, baseline_llm_kwargs)
135 run_equality_correctness_test(vllm_runner,
136 common_llm_kwargs,
137 per_test_common_llm_kwargs,
@@ -162,12 +172,15 @@ def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
172 @pytest.mark.parametrize("output_len", [2048])
173 @pytest.mark.parametrize("batch_size", [1, 32])
174 @pytest.mark.parametrize("seed", [1])
175 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
176 def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
177 per_test_common_llm_kwargs,
178 baseline_llm_kwargs, test_llm_kwargs,
168 - batch_size: int, output_len: int, seed: int):
179 + batch_size: int, output_len: int,
180 + prefill_chunk_size: int, seed: int):
181 """Verify acceptance rate with different batch size and large output
182 length."""
183 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
184 run_equality_correctness_test(vllm_runner,
185 common_llm_kwargs,
186 per_test_common_llm_kwargs,
@@ -204,13 +217,17 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
217 @pytest.mark.parametrize("output_len", [64])
218 @pytest.mark.parametrize("batch_size", [1, 32])
219 @pytest.mark.parametrize("temperature", [1.0])
220 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
221 @pytest.mark.parametrize("seed", [1])
222 def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
223 per_test_common_llm_kwargs,
224 baseline_llm_kwargs, test_llm_kwargs,
225 batch_size: int, output_len: int,
212 - temperature: float, seed: int):
226 + temperature: float,
227 + prefill_chunk_size: int, seed: int):
228 """Verify seeded runs produce the same output."""
229 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
230 + maybe_enable_chunked_prefill(prefill_chunk_size, baseline_llm_kwargs)
231 run_equality_correctness_test(vllm_runner,
232 common_llm_kwargs,
233 per_test_common_llm_kwargs,
@@ -266,14 +283,16 @@ def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
283 128,
284 ])
285 @pytest.mark.parametrize("batch_size", [4])
286 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
287 @pytest.mark.parametrize("seed", [1])
288 def test_mlp_e2e_greedy_correctness_with_preemption(
289 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
290 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
273 - seed: int):
291 + prefill_chunk_size: int, seed: int):
292 """Verify greedy equality, even when some sequences are preempted mid-
293 generation.
294 """
295 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
296 run_equality_correctness_test(vllm_runner,
297 common_llm_kwargs,
298 per_test_common_llm_kwargs,
@@ -317,12 +336,14 @@ def test_mlp_e2e_greedy_correctness_with_preemption(
336 ])
337 @pytest.mark.parametrize("batch_size", [4])
338 @pytest.mark.parametrize("seed", [1])
339 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
340 def test_mlp_e2e_greedy_correctness_with_padding(
341 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
342 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
323 - seed: int):
343 + prefill_chunk_size: int, seed: int):
344 """Verify greedy equality when the vocab dimension is padded
345 """
346 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
347
348 # Default pad_to is 64, test model has vocab_size of 32000
349 def patched_pad_vocab_size(vocab_size, pad_to=None):
@@ -373,14 +394,16 @@ def test_mlp_e2e_greedy_correctness_with_padding(
394 # Use smaller output len for fast test.
395 32,
396 ])
397 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
398 @pytest.mark.parametrize("seed", [1])
399 def test_mlp_different_k(vllm_runner, common_llm_kwargs,
400 per_test_common_llm_kwargs, baseline_llm_kwargs,
379 - test_llm_kwargs, batch_size: int, seed: int,
380 - output_len: int):
401 + test_llm_kwargs, batch_size: int,
402 + prefill_chunk_size: int, seed: int, output_len: int):
403 """Verify that mlp speculative decoding produces exact equality
404 to without spec decode with different values of num_speculative_tokens.
405 """
406 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
407 run_equality_correctness_test(vllm_runner,
408 common_llm_kwargs,
409 per_test_common_llm_kwargs,
@@ -418,15 +441,21 @@ def test_mlp_different_k(vllm_runner, common_llm_kwargs,
441 # Use smaller output len for fast test.
442 32,
443 ])
444 +# Speculative decoding is disabled when sequences reach decoding and the batch
445 +# consists of single-token requests. Hence we set `max_num_seqs`
446 +# >= `speculative_disable_by_batch_size` to test feature interaction.
447 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
448 @pytest.mark.parametrize("seed", [1])
449 def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
450 per_test_common_llm_kwargs, baseline_llm_kwargs,
424 - test_llm_kwargs, batch_size: int, seed: int,
451 + test_llm_kwargs, batch_size: int,
452 + prefill_chunk_size: int, seed: int,
453 output_len: int):
454 """Verify that mlp speculative decoding produces exact equality
455 to without spec decode when speculation is disabled for large
456 batch sizes.
457 """
458 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
459 run_equality_correctness_test(vllm_runner,
460 common_llm_kwargs,
461 per_test_common_llm_kwargs,
@@ -460,13 +489,15 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
489 # Use smaller output len for fast test.
490 32,
491 ])
492 +@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
493 @pytest.mark.parametrize("seed", [1])
494 def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
495 baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
466 - output_len: int, seed: int):
496 + output_len: int, prefill_chunk_size: int, seed: int):
497 """Verify that speculative decoding generates the same output
498 with batch expansion scorer and mqa scorer.
499 """
500 + maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
501 run_equality_correctness_test(vllm_runner,
502 common_llm_kwargs,
503 per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_multistep_correctness.py
+17 -14
@@ -147,20 +147,20 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
147 },
148 ])
149 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
150 -@pytest.mark.parametrize("test_llm_kwargs", [
151 - {
152 - "speculative_model": "JackFram/llama-68m",
153 - "num_speculative_tokens": 5,
154 - "enable_chunked_prefill": False,
155 - },
156 - {
157 - "speculative_model": "JackFram/llama-68m",
158 - "num_speculative_tokens": 5,
159 - "enable_chunked_prefill": True,
160 - "max_num_batched_tokens": 4,
161 - "max_num_seqs": 4,
162 - },
163 -])
150 +@pytest.mark.parametrize("test_llm_kwargs",
151 + [{
152 + "speculative_model": "JackFram/llama-68m",
153 + "num_speculative_tokens": 5,
154 + "enable_chunked_prefill": False,
155 + "disable_logprobs_during_spec_decoding": False
156 + }, {
157 + "speculative_model": "JackFram/llama-68m",
158 + "num_speculative_tokens": 3,
159 + "enable_chunked_prefill": True,
160 + "max_num_batched_tokens": 4,
161 + "max_num_seqs": 4,
162 + "disable_logprobs_during_spec_decoding": False
163 + }])
164 @pytest.mark.parametrize(
165 "output_len",
166 [
@@ -192,6 +192,9 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
192 batch_size,
193 max_output_len=output_len,
194 seed=seed,
195 + prompt_logprobs=2,
196 + logprobs=2,
197 + disable_logprobs=False,
198 temperature=0.0,
199 ensure_all_accepted=ensure_all_accepted)
200
tests/spec_decode/e2e/test_ngram_correctness.py
+4 -9
@@ -26,6 +26,7 @@ for the target model outputs.
26
27 import pytest
28
29 +from ..utils import maybe_enable_chunked_prefill
30 from .conftest import run_equality_correctness_test
31
32
@@ -49,11 +50,13 @@ from .conftest import run_equality_correctness_test
50 "speculative_model": "[ngram]",
51 "num_speculative_tokens": 5,
52 "ngram_prompt_lookup_max": 3,
53 + "speculative_disable_mqa_scorer": False,
54 },
55 {
56 "speculative_model": "[ngram]",
57 "num_speculative_tokens": 5,
58 "ngram_prompt_lookup_max": 3,
59 + "speculative_disable_mqa_scorer": True,
60 },
61 ])
62 @pytest.mark.parametrize("output_len", [
@@ -68,15 +71,7 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
71 batch_size: int, output_len: int,
72 prefill_chunk_size: int, seed: int):
73 """Verify greedy equality on a tiny model with different batch size."""
71 - if prefill_chunk_size > 0:
72 - common_llm_kwargs.update(
73 - **{
74 - "enable_chunked_prefill": True,
75 - "max_num_batched_tokens": prefill_chunk_size,
76 - "max_num_seqs": prefill_chunk_size
77 - })
78 - else:
79 - common_llm_kwargs["enable_chunked_prefill"] = False
74 + maybe_enable_chunked_prefill(prefill_chunk_size, common_llm_kwargs)
75 run_equality_correctness_test(vllm_runner,
76 common_llm_kwargs,
77 per_test_common_llm_kwargs,
tests/spec_decode/test_scorer.py
+1
@@ -60,6 +60,7 @@ def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
60 num_gpu_blocks = 2048 // block_size
61 scorer_worker = create_worker(Worker, model_name, block_size,
62 num_gpu_blocks, seed)
63 + scorer_worker.model_runner.disable_logprobs = True # accessed by mqa_scorer
64 scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor = True
65 scorer_worker.model_runner.model.sampler.\
66 should_modify_greedy_probs_inplace = True
tests/spec_decode/test_spec_decode_worker.py
+1
@@ -754,6 +754,7 @@ def test_populate_seq_ids_with_bonus_tokens():
754 seq_group_metadata_list=seq_group_metadata_list,
755 accepted_token_ids=accepted_token_ids,
756 target_logprobs=target_token_logprobs,
757 + prompt_logprobs=None,
758 k=k,
759 stage_times=(0, 0, 0))
760 # Verify that _seq_with_bonus_token_in_last_step contains the following:
tests/spec_decode/utils.py
+12
@@ -274,3 +274,15 @@ def create_batch(batch_size,
274 prompts, num_gpu_blocks, block_size, final_prompt_lens,
275 prev_output_tokens, seq_ids)
276 return seq_group_metadata_list, prompts, prev_output_tokens
277 +
278 +
279 +def maybe_enable_chunked_prefill(prefill_chunk_size, llm_kwargs):
280 + if prefill_chunk_size > 0:
281 + llm_kwargs.update(
282 + **{
283 + "enable_chunked_prefill": True,
284 + "max_num_batched_tokens": prefill_chunk_size,
285 + "max_num_seqs": prefill_chunk_size
286 + })
287 + else:
288 + llm_kwargs["enable_chunked_prefill"] = False
vllm/config.py
+2 -7
@@ -1685,7 +1685,8 @@ class SpeculativeConfig:
1685 raise ValueError("Expect the batch size threshold of disabling "
1686 "speculative decoding is > 1, but got "
1687 f"{speculative_disable_by_batch_size=}")
1688 -
1688 + if (enable_chunked_prefill and speculative_model == "eagle"):
1689 + raise ValueError("Chunked prefill and EAGLE are not compatible.")
1690 # TODO: The user should be able to specify revision/max model len
1691 # for the draft model. It is not currently supported.
1692 draft_revision = None
@@ -1752,12 +1753,6 @@ class SpeculativeConfig:
1753 f"num_speculative_tokens={n_predict}, but "
1754 f"{num_speculative_tokens=} was provided.")
1755
1755 - if enable_chunked_prefill and draft_hf_config.model_type in (
1756 - "medusa", "mlp_speculator", "eagle"):
1757 - raise ValueError(
1758 - "Chunked prefill and hidden-state based draft models are "
1759 - "not compatible.")
1760 -
1756 speculative_draft_tensor_parallel_size = \
1757 SpeculativeConfig._verify_and_get_draft_model_tensor_parallel_size(
1758 target_parallel_config,
vllm/engine/llm_engine.py
+17 -2
@@ -1010,8 +1010,23 @@ class LLMEngine:
1010 self.speculative_config
1011 # Organize outputs by [step][sequence group] instead of
1012 # [sequence group][step].
1013 - outputs_by_sequence_group = create_output_by_sequence_group(
1014 - outputs, num_seq_groups=len(seq_group_metadata_list))
1013 + if self.scheduler_config.is_multi_step:
1014 + outputs_by_sequence_group = create_output_by_sequence_group(
1015 + outputs, len(seq_group_metadata_list))
1016 + elif self.speculative_config:
1017 + # Decodes are multi-steps while prefills are not, outputting at
1018 + # most 1 token. Separate them so that we can trigger chunk
1019 + # processing without having to pad or copy over prompts K times
1020 + # to match decodes structure (costly with prompt_logprobs).
1021 + num_prefills = sum(sg.is_prompt
1022 + for sg in seq_group_metadata_list)
1023 + prefills, decodes = outputs[:num_prefills], outputs[
1024 + num_prefills:]
1025 + outputs_by_sequence_group = create_output_by_sequence_group(
1026 + decodes,
1027 + num_seq_groups=len(seq_group_metadata_list) - num_prefills)
1028 + outputs_by_sequence_group = [p.outputs for p in prefills
1029 + ] + outputs_by_sequence_group
1030 # We have outputs for multiple steps submitted in a single burst,
1031 # so invalidate is_first_step_output.
1032 is_first_step_output = None
vllm/spec_decode/batch_expansion.py
+95 -38
@@ -83,13 +83,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
83
84 if not non_spec_indices:
85 # All sequence groups in batch have spec decoding enabled
86 - contracted = self._contract_batch_all_spec(
86 + return self._contract_batch_all_spec(
87 target_sampler_output=target_sampler_output,
88 proposals=proposals,
89 )
90 else:
91 # Batch has a mix of spec decode enabled and disabled seq groups
92 - contracted = self._contract_batch(
92 + return self._contract_batch(
93 execute_model_req.seq_group_metadata_list,
94 target_sampler_output=target_sampler_output,
95 proposals=proposals,
@@ -99,14 +99,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
99 k=execute_model_req.num_lookahead_slots,
100 )
101
102 - all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted
103 - return SpeculativeScores(
104 - probs=all_probs,
105 - token_ids=all_tokens,
106 - logprobs=spec_logprobs,
107 - hidden_states=all_hidden_states,
108 - )
109 -
102 def _expand_batch(
103 self,
104 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -143,13 +135,57 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
135 return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
136 num_scoring_tokens)
137
138 + def _contract_non_speculative(
139 + self, scores: SpeculativeScores,
140 + seq_group_metadata_list: List[SequenceGroupMetadata],
141 + non_spec_indices: List[int], non_spec_outputs: SpeculativeScores,
142 + has_prompt_log: bool) -> SpeculativeScores:
143 + """
144 + Augment input `scores` with non-speculative requests outputs.
145 + This includes decode requests with speculation turned off, as well
146 + as prefill requests when `enable_chunked_prefill` is set.
147 + For the latter, prefills are further separated into terminal and
148 + non-terminal chunks (from which no token is sampled).
149 + """
150 + if not non_spec_indices:
151 + return scores
152 +
153 + if has_prompt_log:
154 + # When prompt_logprobs is enabled, prefills yield output token
155 + # (and respective prob) in the last entry (prompt|out):
156 + # [.|.|.|prefill0_out|.|prefill1_out|decode0_out|..].
157 + # With chunked prefill, non-terminal chunks have -1 on each
158 + # position: they're still picked, but they're discarded later.
159 + seq_meta = seq_group_metadata_list
160 + nospec_sizes = torch.tensor([
161 + seq_meta[i].token_chunk_size if seq_meta[i].is_prompt else 1
162 + for i in non_spec_indices
163 + ])
164 + nospec_sampled_token_idxs = torch.cumsum(nospec_sizes, 0).add_(-1)
165 + else:
166 + # In this case only sampled tokens are returned, select all.
167 + nospec_sampled_token_idxs = list(
168 + range(len(non_spec_outputs.token_ids)))
169 +
170 + scores.token_ids[non_spec_indices, :1] = \
171 + non_spec_outputs.token_ids[nospec_sampled_token_idxs].unsqueeze(1)
172 + scores.probs[non_spec_indices, :1, :] = \
173 + non_spec_outputs.probs[nospec_sampled_token_idxs].unsqueeze(1)
174 + scores.logprobs[non_spec_indices, :1, :] = \
175 + non_spec_outputs.logprobs[nospec_sampled_token_idxs].unsqueeze(1)
176 + if scores.hidden_states is not None:
177 + assert non_spec_outputs.hidden_states is not None
178 + scores.hidden_states[non_spec_indices, :1, :] = \
179 + non_spec_outputs.hidden_states[nospec_sampled_token_idxs].unsqueeze(1)
180 + return scores
181 +
182 def _contract_batch(
147 - self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
148 - target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
149 - num_scoring_tokens: int, non_spec_indices: List[int],
150 - spec_indices: List[int], k: int
151 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
152 - Optional[torch.Tensor]]:
183 + self,
184 + contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
185 + target_sampler_output: SamplerOutput,
186 + proposals: SpeculativeProposals, num_scoring_tokens: int,
187 + non_spec_indices: List[int], spec_indices: List[int],
188 + k: int) -> SpeculativeScores:
189 """Contract the expanded batch back into its original size.
190 This maps the scores of speculative tokens back to their original
191 sequences.
@@ -195,23 +231,28 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
231 else:
232 all_hidden_states = None
233
198 - # Rule out prefills that produce no tokens.
199 - non_spec_indices = [
200 - idx for idx in non_spec_indices
201 - if contracted_seq_group_metadata_list[idx].do_sample
202 - ]
203 - if len(non_spec_indices):
204 - all_tokens[non_spec_indices, :1] = \
205 - non_spec_target_token_ids.unsqueeze(1)
206 - all_probs[non_spec_indices, :1, :] = \
207 - non_spec_target_probs.unsqueeze(1)
208 - all_logprobs[non_spec_indices, :1, :] = \
209 - non_spec_target_logprobs.unsqueeze(1)
210 - if all_hidden_states is not None:
211 - assert non_spec_target_hidden_states is not None
212 - all_hidden_states[non_spec_indices, :1, :] = \
213 - non_spec_target_hidden_states.unsqueeze(1)
214 -
234 + has_prompt_log = any((sg.sampling_params.prompt_logprobs
235 + and sg.sampling_params.prompt_logprobs > 0)
236 + for sg in contracted_seq_group_metadata_list)
237 + # When prompt logprobs is enabled, lens of returned tensors go from
238 + # n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
239 + # We adjust stride accordingly to get the generated tokens and
240 + # their probs, but pass on prompt_logprobs as is.
241 + prompt_logprobs = None
242 + if (not self._scorer_worker.model_runner.disable_logprobs\
243 + and has_prompt_log):
244 + prompt_logprobs = [
245 + o.prompt_logprobs for o in target_sampler_output.outputs
246 + ]
247 + elif not has_prompt_log:
248 + # When prompt logprobs are not to be returned,
249 + # we can ignore non-terminal chunks (no out token).
250 + non_spec_indices = [
251 + idx for idx in non_spec_indices
252 + if contracted_seq_group_metadata_list[idx].do_sample
253 + ]
254 +
255 + # "Contract" speculative.
256 if spec_indices:
257 all_tokens[spec_indices] = target_token_ids
258 all_probs[spec_indices] = target_probs
@@ -219,14 +260,27 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
260 if all_hidden_states is not None:
261 all_hidden_states[spec_indices] = target_hidden_states
262
222 - return all_tokens, all_probs, all_logprobs, all_hidden_states
263 + spec_scores = SpeculativeScores(probs=all_probs,
264 + token_ids=all_tokens,
265 + logprobs=all_logprobs,
266 + hidden_states=all_hidden_states,
267 + prompt_logprobs=prompt_logprobs)
268 +
269 + non_spec_outputs = SpeculativeScores(
270 + probs=non_spec_target_probs,
271 + token_ids=non_spec_target_token_ids,
272 + logprobs=non_spec_target_logprobs,
273 + hidden_states=non_spec_target_hidden_states)
274 + # Contract remaining nonspec entries based on non_spec_indices, if any.
275 + return self._contract_non_speculative(
276 + spec_scores, contracted_seq_group_metadata_list, non_spec_indices,
277 + non_spec_outputs, has_prompt_log)
278
279 def _contract_batch_all_spec(
280 self,
281 target_sampler_output: SamplerOutput,
282 proposals: SpeculativeProposals,
228 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
229 - Optional[torch.Tensor]]:
283 + ) -> SpeculativeScores:
284 """Contract the expanded batch back into its original size.
285 This maps the scores of speculative tokens back to their original
286 sequences.
@@ -250,8 +304,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
304 target_hidden_states = target_hidden_states.reshape(
305 *target_token_ids.shape, target_hidden_states.shape[-1])
306
253 - return (target_token_ids, target_probs, target_logprobs,
254 - target_hidden_states)
307 + return SpeculativeScores(probs=target_probs,
308 + token_ids=target_token_ids,
309 + logprobs=target_logprobs,
310 + hidden_states=target_hidden_states,
311 + prompt_logprobs=None)
312
313 def _create_scoring_model_input(
314 self,
vllm/spec_decode/interfaces.py
+6 -2
@@ -1,10 +1,10 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 -from typing import Optional, Set, Union
3 +from typing import List, Optional, Set, Union
4
5 import torch
6
7 -from vllm.sequence import ExecuteModelRequest
7 +from vllm.sequence import ExecuteModelRequest, PromptLogprobs
8 from vllm.worker.worker_base import WorkerBase
9
10
@@ -54,6 +54,10 @@ class SpeculativeScores:
54 # Optional last hidden states from the scoring model.
55 hidden_states: Optional[torch.Tensor] = None
56
57 + # Scoring model may also return logprobs for prompt tokens
58 + # for each request, when chunked prefill is enabled.
59 + prompt_logprobs: Optional[List[PromptLogprobs]] = None
60 +
61 def __repr__(self):
62 return (f"SpeculativeScores("
63 f"probs={self.probs.shape}, "
vllm/spec_decode/mqa_scorer.py
+56 -12
@@ -72,9 +72,15 @@ class MQAScorer(SpeculativeScorer):
72 target_token_ids = target_sampler_output.sampled_token_ids
73 target_probs = target_sampler_output.sampled_token_probs
74 target_logprobs = target_sampler_output.logprobs
75 + prompt_logprobs = None
76 +
77 # If all requests have the same number of query tokens, we can avoid
78 # the for loop to build output for better performance.
79 if min(all_proposal_lengths) == k:
80 + # Regular decodes only.
81 + assert all(not sg.is_prompt
82 + for sg in target_seq_group_metadata_list
83 + if sg.is_prompt)
84 bs, _ = proposals.proposal_token_ids.shape
85 all_tokens = target_token_ids.reshape(bs, k + 1)
86 all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
@@ -88,19 +94,56 @@ class MQAScorer(SpeculativeScorer):
94 all_logprobs = target_logprobs.new_full(size=all_probs.shape,
95 fill_value=-float("inf"))
96 target_token_ids = target_token_ids.flatten()
91 - start_loc = 0
92 - for i, (proposed_len, seq_meta) in enumerate(
93 - zip(all_proposal_lengths, target_seq_group_metadata_list)):
97 +
98 + # When prompt logprobs is enabled, lens of returned tensors go from
99 + # n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
100 + # We adjust stride accordingly to get the generated tokens and
101 + # their probs, but pass on prompt_logprobs as is, since it may be
102 + # that n_prompts >> K.
103 + has_prompt_log = any((sg.sampling_params.prompt_logprobs
104 + and sg.sampling_params.prompt_logprobs > 0)
105 + for sg in target_seq_group_metadata_list)
106 + # TODO (NickLucche) we should surface `disable_logprobs` as to not
107 + # break abstraction to get its value.
108 + if (not self._scorer_worker.model_runner.disable_logprobs\
109 + and has_prompt_log):
110 + prompt_logprobs = [
111 + o.prompt_logprobs for o in target_sampler_output.outputs
112 + ]
113 +
114 + # Split loop into prefill|decode for readability.
115 + start_loc, i = 0, 0
116 + while i < len(target_seq_group_metadata_list
117 + ) and target_seq_group_metadata_list[i].is_prompt:
118 + seq_meta = target_seq_group_metadata_list[i]
119 + end_loc = start_loc
120 + if has_prompt_log:
121 + end_loc += seq_meta.token_chunk_size
122 + elif seq_meta.do_sample:
123 + end_loc += 1
124 +
125 # Skip chunks with no output tokens.
126 if seq_meta.do_sample:
96 - output_len = proposed_len + 1
97 - end_loc = start_loc + output_len
98 - all_tokens[
99 - i, :output_len] = target_token_ids[start_loc:end_loc]
100 - all_probs[i, :output_len] = target_probs[start_loc:end_loc]
101 - all_logprobs[
102 - i, :output_len] = target_logprobs[start_loc:end_loc]
103 - start_loc = end_loc
127 + # Get sampled token (last position in chunk) and its prob.
128 + all_tokens[i, 0] = target_token_ids[end_loc - 1]
129 + all_probs[i, 0] = target_probs[end_loc - 1]
130 + all_logprobs[i, 0] = target_logprobs[end_loc - 1]
131 +
132 + i += 1
133 + start_loc = end_loc
134 + # Decodes.
135 + while i < len(target_seq_group_metadata_list):
136 + proposed_len, seq_meta = all_proposal_lengths[
137 + i], target_seq_group_metadata_list[i]
138 + output_len = proposed_len + 1
139 + end_loc = start_loc + output_len
140 + all_tokens[
141 + i, :output_len] = target_token_ids[start_loc:end_loc]
142 + all_probs[i, :output_len] = target_probs[start_loc:end_loc]
143 + all_logprobs[
144 + i, :output_len] = target_logprobs[start_loc:end_loc]
145 + start_loc = end_loc
146 + i += 1
147
148 hidden_states = None
149 if target_sampler_output.hidden_states is not None:
@@ -110,4 +153,5 @@ class MQAScorer(SpeculativeScorer):
153 return SpeculativeScores(probs=all_probs,
154 token_ids=all_tokens,
155 logprobs=all_logprobs,
113 - hidden_states=hidden_states)
156 + hidden_states=hidden_states,
157 + prompt_logprobs=prompt_logprobs)
vllm/spec_decode/spec_decode_worker.py
+157 -54
@@ -563,50 +563,57 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
563 (seq_id, seq_data) for sg in \
564 execute_model_req.seq_group_metadata_list \
565 for seq_id, seq_data in sg.seq_data.items()
566 - if sg.do_sample # ignore empty token sequences
566 ]
567 completion_seq_group_output_list: List[
568 CompletionSequenceGroupOutput] = []
569 output_index = 0
570 # Make sure the non-terminal prefill chunks are still aligned with
571 # their own empty output.
573 - for seq_group_meta in execute_model_req.seq_group_metadata_list:
574 - # Since we can get chunks here, we dont always have a sampled token
575 - # (only on last chunk) but we still have to provide an output.
576 - if not seq_group_meta.do_sample:
577 - completion_seq_group_output_list.append(
578 - CompletionSequenceGroupOutput(samples=[],
579 - prompt_logprobs=None))
580 - else:
581 - # Sequence with output.
582 - seq_id, seq_data = seq_data_entries[output_index]
583 - needs_prompt_logprobs = seq_output_prompt_logprobs[
584 - output_index]
585 - if needs_prompt_logprobs:
586 - prompt_token_ids = seq_data.get_prompt_token_ids()
587 - prompt_logprobs = [
588 - create_logprobs_output(
589 - token_id=p_token_id,
590 - token_id_logprob_rank=-1,
591 - token_id_logprob=0.0,
592 - topk_token_ids=[],
593 - topk_logprobs=[],
594 - )
595 - # no prompt logprobs for the first token
596 - for p_token_id in prompt_token_ids[1:]
597 - ]
598 - else:
599 - prompt_logprobs = None
600 - completion_seq_group_output_list.append(
601 - create_sequence_group_output(
602 - token_id=sampled_token_ids_list[output_index][0],
572 + for idx, seq_group_meta in enumerate(
573 + execute_model_req.seq_group_metadata_list):
574 + needs_prompt_logprobs = seq_output_prompt_logprobs[idx]
575 + seq_id, seq_data = seq_data_entries[idx]
576 + if needs_prompt_logprobs:
577 + prompt_token_ids = seq_data.get_prompt_token_ids()
578 +
579 + # Some of these sequences may belong to non-terminal chunks,
580 + # which may still have to report logprobs for prompts.
581 + start = 1 if seq_data._num_computed_tokens == 0 \
582 + else seq_data._num_computed_tokens
583 + end = (seq_data._num_computed_tokens + \
584 + seq_group_meta.token_chunk_size)
585 + prompt_token_ids = prompt_token_ids[start:end]
586 + prompt_logprobs = [
587 + create_logprobs_output(
588 + token_id=p_token_id,
589 token_id_logprob_rank=-1,
590 token_id_logprob=0.0,
605 - seq_id=seq_id,
591 topk_token_ids=[],
592 topk_logprobs=[],
608 - prompt_logprobs=prompt_logprobs))
609 - output_index += 1
593 + ) for p_token_id in prompt_token_ids
594 + ]
595 + else:
596 + prompt_logprobs = None
597 +
598 + # Since we can get chunks here, we dont always have a sampled token
599 + # (only on last chunk) but we still have to provide an output.
600 + if not seq_group_meta.do_sample:
601 + completion_seq_group_output_list.append(
602 + CompletionSequenceGroupOutput(
603 + samples=[], prompt_logprobs=prompt_logprobs))
604 + continue
605 +
606 + # Sequence with output.
607 + completion_seq_group_output_list.append(
608 + create_sequence_group_output(
609 + token_id=sampled_token_ids_list[output_index][0],
610 + token_id_logprob_rank=-1,
611 + token_id_logprob=0.0,
612 + seq_id=seq_id,
613 + topk_token_ids=[],
614 + topk_logprobs=[],
615 + prompt_logprobs=prompt_logprobs))
616 + output_index += 1
617
618 return [SamplerOutput(outputs=completion_seq_group_output_list)]
619
@@ -624,24 +631,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
631 assert len(sampler_output) == 1
632 sampler_output = sampler_output[0]
633
627 - # Store hidden states from target model execution.
634 + # Store hidden states from target model execution, BxD.
635 hidden_states = sampler_output.hidden_states
636 if hidden_states is not None:
630 - # remove hidden_states for prompt tokens
631 - # TODO Enable `return_hidden_states`: prefill chunks hidden states
632 - # are pruned by the logits processor. Also, they should be arranged
633 - # back into full-prefill latent. Address it to enable MLPSpeculator.
634 - if any(seq.is_prompt
635 - for seq in execute_model_req.seq_group_metadata_list):
637 + # Only decodes and prefill terminal chunks need a hidden state.
638 + seq_group_meta_with_hidden = [
639 + sg for sg in execute_model_req.seq_group_metadata_list
640 + if sg.do_sample
641 + ]
642 + if any(seq.is_prompt for seq in seq_group_meta_with_hidden):
643 + # Drop hidden_states with no prediction (eg non-terminal chunks)
644 hidden_states = hidden_states[
645 torch.where(sampler_output.sampled_token_ids -
646 VLLM_INVALID_TOKEN_ID)[0]]
639 - if self.previous_hidden_states is None:
647 + if self.previous_hidden_states is None and len(
648 + seq_group_meta_with_hidden):
649 self.previous_hidden_states = HiddenStates(
641 - hidden_states, execute_model_req.seq_group_metadata_list)
642 - else:
643 - self.previous_hidden_states.update(
644 - hidden_states, execute_model_req.seq_group_metadata_list)
650 + hidden_states, seq_group_meta_with_hidden)
651 + elif self.previous_hidden_states and len(
652 + seq_group_meta_with_hidden):
653 + self.previous_hidden_states.update(hidden_states,
654 + seq_group_meta_with_hidden)
655
656 if not skip_proposer:
657 # We prepare the prefill hidden states here so that there no
@@ -752,13 +762,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
762 ]
763 if len(non_spec_indices):
764 all_hidden_states = proposal_scores.hidden_states
755 - # TODO fix `return_hidden_states`, same as in `_run_no_spec`
765 if all_hidden_states is not None:
766 prefill_hidden_states = all_hidden_states[non_spec_indices]
767 execute_model_req.previous_hidden_states = \
768 prepare_prefill_hidden_states(prefill_hidden_states)
769 # Sync proposer KV cache for prefills.
770 prefill_req = execute_model_req.clone(non_spec_seqs)
771 + # TODO avoid sampling here?
772 self.proposer_worker.execute_model(prefill_req)
773
774 with Timer() as verification_timer:
@@ -774,6 +784,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
784 execute_model_req.seq_group_metadata_list,
785 accepted_token_ids,
786 target_logprobs=target_logprobs,
787 + prompt_logprobs=proposal_scores.prompt_logprobs
788 + if not self._disable_logprobs else None,
789 k=execute_model_req.num_lookahead_slots,
790 stage_times=stage_times)
791
@@ -845,19 +857,32 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
857 # metadata.
858 accepted_token_ids[original_indices] = accepted_token_ids.clone()
859
860 + # B x K+1 x D
861 hidden_states = proposal_scores.hidden_states
862 if hidden_states is not None:
863 + # Only get terminal hidden states for next step
864 + terminal_metadata = [
865 + sg for sg in seq_group_metadata_list if sg.do_sample
866 + ]
867 +
868 # Contract hidden states based on accepted tokens
869 hs_size = hidden_states.shape[-1]
852 -
870 accepted_index = accepted_token_ids + 1 # Convert -1 to 0
854 - accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
855 - index = accepted_index[:, None, None].expand(-1, 1, hs_size)
871 + accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) # b
872 + # Drop non-terminal prefill chunks hidden states.
873 + hidden_states = hidden_states[
874 + accepted_index != VLLM_INVALID_TOKEN_ID]
875 + accepted_index = accepted_index[
876 + accepted_index != VLLM_INVALID_TOKEN_ID]
877 + assert len(accepted_index) == hidden_states.shape[0] == len(
878 + terminal_metadata)
879 + index = accepted_index[:, None, None].expand(-1, 1,
880 + hs_size) # b x 1 x d
881 second_last_token_hidden_states = hidden_states[:, -2] # b x d
882 hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
883 # Store hidden states from target model for subsequent decode step
884 self.previous_hidden_states = HiddenStates(
860 - hidden_states, seq_group_metadata_list,
885 + hidden_states, terminal_metadata,
886 second_last_token_hidden_states)
887 return accepted_token_ids, logprobs
888
@@ -866,6 +891,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
891 seq_group_metadata_list: List[SequenceGroupMetadata],
892 accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
893 target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
894 + prompt_logprobs: Optional[
895 + torch.Tensor], # shape: [nprompt_tokens, vocab_size]
896 k: int,
897 stage_times: Tuple[float, float, float],
898 ) -> List[SamplerOutput]:
@@ -909,15 +936,89 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
936
937 # Construct the output on a per-step, per-sequence basis.
938 # Non-terminal prefill chunks will end up here as rows with just -1s
912 - # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
939 + # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]] while
940 + # terminal chunks will only have one generated token at time 0.
941 sampler_output_list: List[SamplerOutput] = []
942 +
943 + # Prefills are not multi-step (return at most 1 token), in order to
944 + # avoid padding or repetition to fit decodes, we separate them.
945 + for i, sg in enumerate(seq_group_metadata_list):
946 + if not sg.is_prompt:
947 + # Requests are ordered as prefills|decodes=>no more prefills.
948 + break
949 + num_logprobs = num_logprobs_per_seq[i]
950 + seq_kwargs = dict(token_id=-1,
951 + token_id_logprob_rank=0,
952 + token_id_logprob=-float('inf'),
953 + topk_token_ids=[-1] * num_logprobs,
954 + topk_logprobs=[-float('inf')] * num_logprobs,
955 + seq_id=seq_ids[i])
956 + # Terminal chunk, has token.
957 + if sg.do_sample:
958 + seq_kwargs.update(
959 + dict(
960 + token_id=accepted_token_ids[i][0].item(),
961 + token_id_logprob_rank=accepted_token_id_ranks_by_step[
962 + 0][i],
963 + token_id_logprob=accepted_token_id_logprobs_by_step[0]
964 + [i],
965 + topk_token_ids=topk_indices_by_step[0][i]
966 + [:num_logprobs],
967 + # output only so step is 0
968 + topk_logprobs=topk_logprobs_by_step[0][i]
969 + [:num_logprobs],
970 + ))
971 + needs_plogs = (sg.sampling_params.prompt_logprobs
972 + and sg.sampling_params.prompt_logprobs > 0)
973 + plogs = None
974 + if prompt_logprobs is not None:
975 + # Even non-terminal prompt chunks can have logprobs here.
976 + plogs = prompt_logprobs[i]
977 + elif needs_plogs:
978 + # Prompt logprobs are requested but `_disable_logprobs` is set.
979 + seq_data = next(iter(sg.seq_data.values()))
980 + # Get only the tokens in this chunk!
981 + prompt_token_ids = seq_data.get_prompt_token_ids()
982 + prompt_token_ids = prompt_token_ids[
983 + seq_data.
984 + _num_computed_tokens:seq_data._num_computed_tokens +
985 + sg.token_chunk_size]
986 +
987 + is_first_chunk = seq_data._num_computed_tokens == 0
988 + # There's no prob generated for the first token in a sequence.
989 + if is_first_chunk:
990 + prompt_token_ids = prompt_token_ids[1:]
991 + plogs = [
992 + create_logprobs_output(
993 + token_id=p_token_id,
994 + token_id_logprob_rank=-1,
995 + token_id_logprob=0.0,
996 + topk_token_ids=[],
997 + topk_logprobs=[],
998 + ) for p_token_id in prompt_token_ids
999 + ]
1000 + seq_kwargs.update(dict(prompt_logprobs=plogs))
1001 +
1002 + sampler_output_list.append(
1003 + SamplerOutput(
1004 + outputs=[create_sequence_group_output(
1005 + **seq_kwargs)])) # type: ignore
1006 +
1007 + # Decodes, create one SamplerOutput per-step (at most K+1).
1008 for step_index in range(num_steps):
915 - if all(token_id == -1
916 - for token_id in accepted_token_ids_by_step[step_index]):
1009 + if all(token_id == -1 for sg, token_id in zip(
1010 + seq_group_metadata_list,
1011 + accepted_token_ids_by_step[step_index])
1012 + if not sg.is_prompt):
1013 break
1014
1015 step_output_token_ids: List[CompletionSequenceGroupOutput] = []
1016 for sequence_index in range(batch_size):
1017 + seq_meta = seq_group_metadata_list[sequence_index]
1018 + # Prompts already processed above.
1019 + if seq_meta.is_prompt:
1020 + continue
1021 +
1022 # Each sequence may have a different num_logprobs; retrieve it.
1023 num_logprobs = num_logprobs_per_seq[sequence_index]
1024 step_output_token_ids.append(
@@ -952,6 +1053,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1053 # This is periodic because the rejection sampler emits metrics
1054 # periodically.
1055 self._maybe_log_stage_times(*stage_times)
1056 + # First `n_prefills` entries will contain prefills SamplerOutput when
1057 + # chunked prefill is enabled, the rest is decodes in multi-step format.
1058 return sampler_output_list
1059
1060 def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,