A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Feature] [Spec decode]: Enable MLPSpeculator/Medusa and `prompt_logprobs` with ChunkedPrefill (#10132)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: wallashss <wallashss@ibm.com> Co-authored-by: wallashss <wallashss@ibm.com>
Nicolò Lucchesi committed
Jan 27, 2025 at 22:38 UTC
6116ca8cd79b642c64f4ae6f050a6bc12b96d037
16 files changed
+469
-166
tests/spec_decode/e2e/conftest.py
+16
-3
@@ -2,6 +2,7 @@ from itertools import cycle
2
from typing import List, Optional, Sequence, Tuple, Union
3
4
import pytest
5
+import torch
6
7
from vllm import LLM, SamplingParams
8
from vllm.distributed import cleanup_dist_env_and_memory
@@ -154,6 +155,8 @@ def _check_logprobs_when_output_disabled(
155
spec_pos_logprob) = next(iter(spec_pos_logprobs.items()))
156
assert spec_pos_logprob.rank == -1
157
assert spec_pos_logprob.logprob == 0.0
158
+ if isinstance(spec_pos_logprob_token_id, torch.Tensor):
159
+ spec_pos_logprob_token_id = spec_pos_logprob_token_id.item()
160
assert spec_pos_logprob_token_id in baseline_pos_logprobs
161
162
@@ -244,7 +247,8 @@ def run_equality_correctness_test_tp(model,
247
batch_size: int,
248
max_output_len: int,
249
seed: int = 0,
247
- temperature: float = 0.0):
250
+ temperature: float = 0.0,
251
+ logprobs: Optional[int] = None):
252
"""Helper method that compares the outputs of both the baseline LLM and
253
the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
254
the same when temperature is zero.
@@ -257,7 +261,6 @@ def run_equality_correctness_test_tp(model,
261
results = []
262
263
prompts = [prompt for prompt, _ in zip(cycle(PROMPTS), range(batch_size))]
260
-
264
for args, env in ((arg1, env1), (arg2, env2)):
265
with RemoteOpenAIServer(model,
266
args,
@@ -269,12 +272,14 @@ def run_equality_correctness_test_tp(model,
272
prompt=prompts,
273
max_tokens=max_output_len,
274
seed=seed,
272
- temperature=temperature)
275
+ temperature=temperature,
276
+ logprobs=logprobs)
277
278
results.append({
279
"test":
280
"seeded_sampling",
281
"text": [choice.text for choice in completion.choices],
282
+ "logprobs": [choice.logprobs for choice in completion.choices],
283
"finish_reason":
284
[choice.finish_reason for choice in completion.choices],
285
"usage":
@@ -284,7 +289,15 @@ def run_equality_correctness_test_tp(model,
289
n = len(results) // 2
290
arg1_results = results[:n]
291
arg2_results = results[n:]
292
+ # Separate logprobs to avoid asserting exact equality.
293
+ arg1_logprobs = [r.pop("logprobs") for r in arg1_results]
294
+ arg2_logprobs = [r.pop("logprobs") for r in arg2_results]
295
+
296
for arg1_result, arg2_result in zip(arg1_results, arg2_results):
297
assert arg1_result == arg2_result, (
298
f"Results for {model=} are not the same with {arg1=} and {arg2=}. "
299
f"{arg1_result=} != {arg2_result=}")
300
+ if logprobs:
301
+ for logs1, logs2 in zip(arg1_logprobs, arg2_logprobs):
302
+ for l1, l2 in zip(logs1, logs2):
303
+ assert l1.tokens == l2.tokens
tests/spec_decode/e2e/test_integration_dist_tp2.py
+9
-1
@@ -2,6 +2,8 @@
2
tensor parallelism.
3
"""
4
5
+from typing import Optional
6
+
7
import pytest
8
import torch
9
@@ -154,15 +156,20 @@ def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
156
"--speculative-draft-tensor-parallel-size",
157
"1",
158
])])
159
+@pytest.mark.parametrize("logprobs", [None, 2])
160
@pytest.mark.parametrize("batch_size", [2])
161
@pytest.mark.parametrize("seed", [1])
162
def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
163
per_test_common_llm_kwargs,
164
baseline_llm_kwargs, test_llm_kwargs,
165
+ logprobs: Optional[int],
166
batch_size: int, seed: int):
167
"""Verify spec decode works well with same and different TP size for
168
the draft model with chunked prefill.
169
"""
170
+ if logprobs:
171
+ test_llm_kwargs.extend(
172
+ ["--disable_logprobs_during_spec_decoding", "False"])
173
run_equality_correctness_test_tp(model,
174
common_llm_kwargs,
175
per_test_common_llm_kwargs,
@@ -171,4 +178,5 @@ def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
178
batch_size,
179
max_output_len=32,
180
seed=seed,
174
- temperature=0.0)
181
+ temperature=0.0,
182
+ logprobs=logprobs)
tests/spec_decode/e2e/test_logprobs.py
+10
-6
@@ -4,26 +4,27 @@ import pytest
4
5
from vllm import SamplingParams
6
7
+from ..utils import maybe_enable_chunked_prefill
8
from .conftest import run_equality_correctness_test
9
10
11
@pytest.mark.parametrize(
12
"common_llm_kwargs",
13
[{
13
- "model_name": "JackFram/llama-68m",
14
+ "model_name": "JackFram/llama-160m",
15
16
# Skip cuda graph recording for fast test.
16
- "enforce_eager": True,
17
+ "enforce_eager": True
18
}])
19
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
20
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
21
@pytest.mark.parametrize("test_llm_kwargs",
22
[{
22
- "speculative_model": "JackFram/llama-160m",
23
+ "speculative_model": "JackFram/llama-68m",
24
"num_speculative_tokens": 3,
25
"disable_logprobs_during_spec_decoding": False,
26
}, {
26
- "speculative_model": "JackFram/llama-160m",
27
+ "speculative_model": "JackFram/llama-68m",
28
"num_speculative_tokens": 3,
29
"disable_logprobs_during_spec_decoding": True,
30
}])
@@ -36,12 +37,15 @@ from .conftest import run_equality_correctness_test
37
])
38
@pytest.mark.parametrize("seed", [1])
39
@pytest.mark.parametrize("logprobs", [1, 6])
40
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4, 12])
41
def test_logprobs_equality(vllm_runner, common_llm_kwargs,
42
per_test_common_llm_kwargs, baseline_llm_kwargs,
43
test_llm_kwargs, batch_size: int, output_len: int,
42
- seed: int, logprobs: int):
43
- """Verify output logprobs are equal with and without speculative decoding.
44
+ seed: int, logprobs: int, prefill_chunk_size: int):
45
+ """Verify output logprobs are equal with and without speculative decoding,
46
+ as well as with and without chunked prefill.
47
"""
48
+ maybe_enable_chunked_prefill(prefill_chunk_size, common_llm_kwargs)
49
run_equality_correctness_test(vllm_runner,
50
common_llm_kwargs,
51
per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_medusa_correctness.py
+24
-7
@@ -21,6 +21,7 @@ correctess for the target model outputs.
21
22
import pytest
23
24
+from ..utils import maybe_enable_chunked_prefill
25
from .conftest import run_equality_correctness_test
26
27
# main model
@@ -67,12 +68,14 @@ PRECISION = "float32"
68
])
69
@pytest.mark.parametrize("batch_size", [1, 32])
70
@pytest.mark.parametrize("seed", [1])
71
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
72
def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
73
per_test_common_llm_kwargs,
74
baseline_llm_kwargs, test_llm_kwargs,
75
batch_size: int, output_len: int,
74
- seed: int):
76
+ seed: int, prefill_chunk_size: int):
77
"""Verify greedy equality with different batch size."""
78
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
79
run_equality_correctness_test(vllm_runner,
80
common_llm_kwargs,
81
per_test_common_llm_kwargs,
@@ -119,12 +122,15 @@ def test_medusa_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
122
@pytest.mark.parametrize("batch_size", [8])
123
@pytest.mark.parametrize("seed", [1])
124
@pytest.mark.parametrize("logprobs", [1, 6])
125
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
126
def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
127
per_test_common_llm_kwargs,
128
baseline_llm_kwargs, test_llm_kwargs,
129
batch_size: int, output_len: int,
126
- seed: int, logprobs: int):
130
+ seed: int, logprobs: int,
131
+ prefill_chunk_size: int):
132
"""Verify greedy equality with different batch size."""
133
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
134
run_equality_correctness_test(vllm_runner,
135
common_llm_kwargs,
136
per_test_common_llm_kwargs,
@@ -167,12 +173,14 @@ def test_medusa_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
173
])
174
@pytest.mark.parametrize("batch_size", [1, 32])
175
@pytest.mark.parametrize("seed", [1])
176
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
177
def test_medusa_e2e_greedy_correctness_cuda_graph(
178
vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
179
baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
173
- seed: int):
180
+ seed: int, prefill_chunk_size: int):
181
"""Verify greedy equality with cuda graph enabled and different
182
batch sizes."""
183
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
184
run_equality_correctness_test(vllm_runner,
185
common_llm_kwargs,
186
per_test_common_llm_kwargs,
@@ -217,13 +225,15 @@ def test_medusa_e2e_greedy_correctness_cuda_graph(
225
])
226
@pytest.mark.parametrize("batch_size", [4])
227
@pytest.mark.parametrize("seed", [1])
228
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
229
def test_medusa_e2e_greedy_correctness_with_preemption(
230
vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
231
baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
223
- seed: int):
232
+ seed: int, prefill_chunk_size: int):
233
"""Verify greedy equality, even when some sequences are preempted mid-
234
generation.
235
"""
236
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
237
run_equality_correctness_test(vllm_runner,
238
common_llm_kwargs,
239
per_test_common_llm_kwargs,
@@ -267,13 +277,15 @@ def test_medusa_e2e_greedy_correctness_with_preemption(
277
32,
278
])
279
@pytest.mark.parametrize("seed", [1])
280
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
281
def test_medusa_different_k(vllm_runner, common_llm_kwargs,
282
per_test_common_llm_kwargs, baseline_llm_kwargs,
283
test_llm_kwargs, batch_size: int, output_len: int,
273
- seed: int):
284
+ seed: int, prefill_chunk_size: int):
285
"""Verify that medusa speculative decoding produces exact equality
286
to without spec decode with different values of num_speculative_tokens.
287
"""
288
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
289
run_equality_correctness_test(vllm_runner,
290
common_llm_kwargs,
291
per_test_common_llm_kwargs,
@@ -313,14 +325,17 @@ def test_medusa_different_k(vllm_runner, common_llm_kwargs,
325
32,
326
])
327
@pytest.mark.parametrize("seed", [1])
328
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
329
def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
330
per_test_common_llm_kwargs, baseline_llm_kwargs,
331
test_llm_kwargs, batch_size: int,
319
- output_len: int, seed: int):
332
+ output_len: int, seed: int,
333
+ prefill_chunk_size: int):
334
"""Verify that medusa speculative decoding produces exact equality
335
to without spec decode when speculation is disabled for large
336
batch sizes.
337
"""
338
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
339
run_equality_correctness_test(vllm_runner,
340
common_llm_kwargs,
341
per_test_common_llm_kwargs,
@@ -361,12 +376,14 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
376
32,
377
])
378
@pytest.mark.parametrize("seed", [1])
379
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
380
def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
381
baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
366
- output_len: int, seed: int):
382
+ output_len: int, seed: int, prefill_chunk_size: int):
383
"""Verify that speculative decoding generates the same output
384
with batch expansion scorer and mqa scorer.
385
"""
386
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
387
run_equality_correctness_test(vllm_runner,
388
common_llm_kwargs,
389
per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_mlp_correctness.py
+42
-11
@@ -25,6 +25,7 @@ import pytest
25
26
from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
27
28
+from ..utils import maybe_enable_chunked_prefill
29
from .conftest import run_equality_correctness_test
30
31
# main model
@@ -66,14 +67,16 @@ PRECISION = "float32"
67
@pytest.mark.parametrize("output_len", [
68
128,
69
])
69
-@pytest.mark.parametrize("batch_size", [1, 32])
70
+@pytest.mark.parametrize("batch_size", [4, 32])
71
@pytest.mark.parametrize("seed", [1])
72
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 32])
73
def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
74
per_test_common_llm_kwargs,
75
baseline_llm_kwargs, test_llm_kwargs,
76
batch_size: int, output_len: int,
75
- seed: int):
77
+ seed: int, prefill_chunk_size: int):
78
"""Verify greedy equality with different batch size."""
79
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
80
run_equality_correctness_test(vllm_runner,
81
common_llm_kwargs,
82
per_test_common_llm_kwargs,
@@ -116,12 +119,19 @@ def test_mlp_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
119
@pytest.mark.parametrize("batch_size", [8])
120
@pytest.mark.parametrize("seed", [1])
121
@pytest.mark.parametrize("logprobs", [1, 6])
122
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
123
def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
124
per_test_common_llm_kwargs,
125
baseline_llm_kwargs, test_llm_kwargs,
126
batch_size: int, output_len: int, seed: int,
123
- logprobs: int):
127
+ logprobs: int, prefill_chunk_size: int):
128
"""Verify greedy equality with different batch size."""
129
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
130
+ # NOTE Test is sensitive enough st if we don't enable chunked prefill
131
+ # scheduling on baseline too, we get slightly different logprobs, ending
132
+ # up sampling different tokens at the tail (ie top tokens don't change).
133
+ # TL;DR: sd+cp == org+cp but sd+cp != org..is this expected?
134
+ maybe_enable_chunked_prefill(prefill_chunk_size, baseline_llm_kwargs)
135
run_equality_correctness_test(vllm_runner,
136
common_llm_kwargs,
137
per_test_common_llm_kwargs,
@@ -162,12 +172,15 @@ def test_mlp_e2e_greedy_logprobs(vllm_runner, common_llm_kwargs,
172
@pytest.mark.parametrize("output_len", [2048])
173
@pytest.mark.parametrize("batch_size", [1, 32])
174
@pytest.mark.parametrize("seed", [1])
175
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
176
def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
177
per_test_common_llm_kwargs,
178
baseline_llm_kwargs, test_llm_kwargs,
168
- batch_size: int, output_len: int, seed: int):
179
+ batch_size: int, output_len: int,
180
+ prefill_chunk_size: int, seed: int):
181
"""Verify acceptance rate with different batch size and large output
182
length."""
183
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
184
run_equality_correctness_test(vllm_runner,
185
common_llm_kwargs,
186
per_test_common_llm_kwargs,
@@ -204,13 +217,17 @@ def test_mlp_e2e_acceptance_rate(vllm_runner, common_llm_kwargs,
217
@pytest.mark.parametrize("output_len", [64])
218
@pytest.mark.parametrize("batch_size", [1, 32])
219
@pytest.mark.parametrize("temperature", [1.0])
220
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
221
@pytest.mark.parametrize("seed", [1])
222
def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
223
per_test_common_llm_kwargs,
224
baseline_llm_kwargs, test_llm_kwargs,
225
batch_size: int, output_len: int,
212
- temperature: float, seed: int):
226
+ temperature: float,
227
+ prefill_chunk_size: int, seed: int):
228
"""Verify seeded runs produce the same output."""
229
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
230
+ maybe_enable_chunked_prefill(prefill_chunk_size, baseline_llm_kwargs)
231
run_equality_correctness_test(vllm_runner,
232
common_llm_kwargs,
233
per_test_common_llm_kwargs,
@@ -266,14 +283,16 @@ def test_mlp_e2e_seeded_correctness(vllm_runner, common_llm_kwargs,
283
128,
284
])
285
@pytest.mark.parametrize("batch_size", [4])
286
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
287
@pytest.mark.parametrize("seed", [1])
288
def test_mlp_e2e_greedy_correctness_with_preemption(
289
vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
290
baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
273
- seed: int):
291
+ prefill_chunk_size: int, seed: int):
292
"""Verify greedy equality, even when some sequences are preempted mid-
293
generation.
294
"""
295
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
296
run_equality_correctness_test(vllm_runner,
297
common_llm_kwargs,
298
per_test_common_llm_kwargs,
@@ -317,12 +336,14 @@ def test_mlp_e2e_greedy_correctness_with_preemption(
336
])
337
@pytest.mark.parametrize("batch_size", [4])
338
@pytest.mark.parametrize("seed", [1])
339
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
340
def test_mlp_e2e_greedy_correctness_with_padding(
341
vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
342
baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,
323
- seed: int):
343
+ prefill_chunk_size: int, seed: int):
344
"""Verify greedy equality when the vocab dimension is padded
345
"""
346
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
347
348
# Default pad_to is 64, test model has vocab_size of 32000
349
def patched_pad_vocab_size(vocab_size, pad_to=None):
@@ -373,14 +394,16 @@ def test_mlp_e2e_greedy_correctness_with_padding(
394
# Use smaller output len for fast test.
395
32,
396
])
397
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
398
@pytest.mark.parametrize("seed", [1])
399
def test_mlp_different_k(vllm_runner, common_llm_kwargs,
400
per_test_common_llm_kwargs, baseline_llm_kwargs,
379
- test_llm_kwargs, batch_size: int, seed: int,
380
- output_len: int):
401
+ test_llm_kwargs, batch_size: int,
402
+ prefill_chunk_size: int, seed: int, output_len: int):
403
"""Verify that mlp speculative decoding produces exact equality
404
to without spec decode with different values of num_speculative_tokens.
405
"""
406
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
407
run_equality_correctness_test(vllm_runner,
408
common_llm_kwargs,
409
per_test_common_llm_kwargs,
@@ -418,15 +441,21 @@ def test_mlp_different_k(vllm_runner, common_llm_kwargs,
441
# Use smaller output len for fast test.
442
32,
443
])
444
+# Speculative decoding is disabled when sequences reach decoding and the batch
445
+# consists of single-token requests. Hence we set `max_num_seqs`
446
+# >= `speculative_disable_by_batch_size` to test feature interaction.
447
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
448
@pytest.mark.parametrize("seed", [1])
449
def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
450
per_test_common_llm_kwargs, baseline_llm_kwargs,
424
- test_llm_kwargs, batch_size: int, seed: int,
451
+ test_llm_kwargs, batch_size: int,
452
+ prefill_chunk_size: int, seed: int,
453
output_len: int):
454
"""Verify that mlp speculative decoding produces exact equality
455
to without spec decode when speculation is disabled for large
456
batch sizes.
457
"""
458
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
459
run_equality_correctness_test(vllm_runner,
460
common_llm_kwargs,
461
per_test_common_llm_kwargs,
@@ -460,13 +489,15 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
489
# Use smaller output len for fast test.
490
32,
491
])
492
+@pytest.mark.parametrize("prefill_chunk_size", [-1, 4])
493
@pytest.mark.parametrize("seed", [1])
494
def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
495
baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
466
- output_len: int, seed: int):
496
+ output_len: int, prefill_chunk_size: int, seed: int):
497
"""Verify that speculative decoding generates the same output
498
with batch expansion scorer and mqa scorer.
499
"""
500
+ maybe_enable_chunked_prefill(prefill_chunk_size, test_llm_kwargs)
501
run_equality_correctness_test(vllm_runner,
502
common_llm_kwargs,
503
per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_multistep_correctness.py
+17
-14
@@ -147,20 +147,20 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
147
},
148
])
149
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
150
-@pytest.mark.parametrize("test_llm_kwargs", [
151
- {
152
- "speculative_model": "JackFram/llama-68m",
153
- "num_speculative_tokens": 5,
154
- "enable_chunked_prefill": False,
155
- },
156
- {
157
- "speculative_model": "JackFram/llama-68m",
158
- "num_speculative_tokens": 5,
159
- "enable_chunked_prefill": True,
160
- "max_num_batched_tokens": 4,
161
- "max_num_seqs": 4,
162
- },
163
-])
150
+@pytest.mark.parametrize("test_llm_kwargs",
151
+ [{
152
+ "speculative_model": "JackFram/llama-68m",
153
+ "num_speculative_tokens": 5,
154
+ "enable_chunked_prefill": False,
155
+ "disable_logprobs_during_spec_decoding": False
156
+ }, {
157
+ "speculative_model": "JackFram/llama-68m",
158
+ "num_speculative_tokens": 3,
159
+ "enable_chunked_prefill": True,
160
+ "max_num_batched_tokens": 4,
161
+ "max_num_seqs": 4,
162
+ "disable_logprobs_during_spec_decoding": False
163
+ }])
164
@pytest.mark.parametrize(
165
"output_len",
166
[
@@ -192,6 +192,9 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
192
batch_size,
193
max_output_len=output_len,
194
seed=seed,
195
+ prompt_logprobs=2,
196
+ logprobs=2,
197
+ disable_logprobs=False,
198
temperature=0.0,
199
ensure_all_accepted=ensure_all_accepted)
200
tests/spec_decode/e2e/test_ngram_correctness.py
+4
-9
@@ -26,6 +26,7 @@ for the target model outputs.
26
27
import pytest
28
29
+from ..utils import maybe_enable_chunked_prefill
30
from .conftest import run_equality_correctness_test
31
32
@@ -49,11 +50,13 @@ from .conftest import run_equality_correctness_test
50
"speculative_model": "[ngram]",
51
"num_speculative_tokens": 5,
52
"ngram_prompt_lookup_max": 3,
53
+ "speculative_disable_mqa_scorer": False,
54
},
55
{
56
"speculative_model": "[ngram]",
57
"num_speculative_tokens": 5,
58
"ngram_prompt_lookup_max": 3,
59
+ "speculative_disable_mqa_scorer": True,
60
},
61
])
62
@pytest.mark.parametrize("output_len", [
@@ -68,15 +71,7 @@ def test_ngram_e2e_greedy_correctness(vllm_runner, common_llm_kwargs,
71
batch_size: int, output_len: int,
72
prefill_chunk_size: int, seed: int):
73
"""Verify greedy equality on a tiny model with different batch size."""
71
- if prefill_chunk_size > 0:
72
- common_llm_kwargs.update(
73
- **{
74
- "enable_chunked_prefill": True,
75
- "max_num_batched_tokens": prefill_chunk_size,
76
- "max_num_seqs": prefill_chunk_size
77
- })
78
- else:
79
- common_llm_kwargs["enable_chunked_prefill"] = False
74
+ maybe_enable_chunked_prefill(prefill_chunk_size, common_llm_kwargs)
75
run_equality_correctness_test(vllm_runner,
76
common_llm_kwargs,
77
per_test_common_llm_kwargs,
tests/spec_decode/test_scorer.py
+1
@@ -60,6 +60,7 @@ def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
60
num_gpu_blocks = 2048 // block_size
61
scorer_worker = create_worker(Worker, model_name, block_size,
62
num_gpu_blocks, seed)
63
+ scorer_worker.model_runner.disable_logprobs = True # accessed by mqa_scorer
64
scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor = True
65
scorer_worker.model_runner.model.sampler.\
66
should_modify_greedy_probs_inplace = True
tests/spec_decode/test_spec_decode_worker.py
+1
@@ -754,6 +754,7 @@ def test_populate_seq_ids_with_bonus_tokens():
754
seq_group_metadata_list=seq_group_metadata_list,
755
accepted_token_ids=accepted_token_ids,
756
target_logprobs=target_token_logprobs,
757
+ prompt_logprobs=None,
758
k=k,
759
stage_times=(0, 0, 0))
760
# Verify that _seq_with_bonus_token_in_last_step contains the following:
tests/spec_decode/utils.py
+12
@@ -274,3 +274,15 @@ def create_batch(batch_size,
274
prompts, num_gpu_blocks, block_size, final_prompt_lens,
275
prev_output_tokens, seq_ids)
276
return seq_group_metadata_list, prompts, prev_output_tokens
277
+
278
+
279
+def maybe_enable_chunked_prefill(prefill_chunk_size, llm_kwargs):
280
+ if prefill_chunk_size > 0:
281
+ llm_kwargs.update(
282
+ **{
283
+ "enable_chunked_prefill": True,
284
+ "max_num_batched_tokens": prefill_chunk_size,
285
+ "max_num_seqs": prefill_chunk_size
286
+ })
287
+ else:
288
+ llm_kwargs["enable_chunked_prefill"] = False
vllm/config.py
+2
-7
@@ -1685,7 +1685,8 @@ class SpeculativeConfig:
1685
raise ValueError("Expect the batch size threshold of disabling "
1686
"speculative decoding is > 1, but got "
1687
f"{speculative_disable_by_batch_size=}")
1688
-
1688
+ if (enable_chunked_prefill and speculative_model == "eagle"):
1689
+ raise ValueError("Chunked prefill and EAGLE are not compatible.")
1690
# TODO: The user should be able to specify revision/max model len
1691
# for the draft model. It is not currently supported.
1692
draft_revision = None
@@ -1752,12 +1753,6 @@ class SpeculativeConfig:
1753
f"num_speculative_tokens={n_predict}, but "
1754
f"{num_speculative_tokens=} was provided.")
1755
1755
- if enable_chunked_prefill and draft_hf_config.model_type in (
1756
- "medusa", "mlp_speculator", "eagle"):
1757
- raise ValueError(
1758
- "Chunked prefill and hidden-state based draft models are "
1759
- "not compatible.")
1760
-
1756
speculative_draft_tensor_parallel_size = \
1757
SpeculativeConfig._verify_and_get_draft_model_tensor_parallel_size(
1758
target_parallel_config,
vllm/engine/llm_engine.py
+17
-2
@@ -1010,8 +1010,23 @@ class LLMEngine:
1010
self.speculative_config
1011
# Organize outputs by [step][sequence group] instead of
1012
# [sequence group][step].
1013
- outputs_by_sequence_group = create_output_by_sequence_group(
1014
- outputs, num_seq_groups=len(seq_group_metadata_list))
1013
+ if self.scheduler_config.is_multi_step:
1014
+ outputs_by_sequence_group = create_output_by_sequence_group(
1015
+ outputs, len(seq_group_metadata_list))
1016
+ elif self.speculative_config:
1017
+ # Decodes are multi-steps while prefills are not, outputting at
1018
+ # most 1 token. Separate them so that we can trigger chunk
1019
+ # processing without having to pad or copy over prompts K times
1020
+ # to match decodes structure (costly with prompt_logprobs).
1021
+ num_prefills = sum(sg.is_prompt
1022
+ for sg in seq_group_metadata_list)
1023
+ prefills, decodes = outputs[:num_prefills], outputs[
1024
+ num_prefills:]
1025
+ outputs_by_sequence_group = create_output_by_sequence_group(
1026
+ decodes,
1027
+ num_seq_groups=len(seq_group_metadata_list) - num_prefills)
1028
+ outputs_by_sequence_group = [p.outputs for p in prefills
1029
+ ] + outputs_by_sequence_group
1030
# We have outputs for multiple steps submitted in a single burst,
1031
# so invalidate is_first_step_output.
1032
is_first_step_output = None
vllm/spec_decode/batch_expansion.py
+95
-38
@@ -83,13 +83,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
83
84
if not non_spec_indices:
85
# All sequence groups in batch have spec decoding enabled
86
- contracted = self._contract_batch_all_spec(
86
+ return self._contract_batch_all_spec(
87
target_sampler_output=target_sampler_output,
88
proposals=proposals,
89
)
90
else:
91
# Batch has a mix of spec decode enabled and disabled seq groups
92
- contracted = self._contract_batch(
92
+ return self._contract_batch(
93
execute_model_req.seq_group_metadata_list,
94
target_sampler_output=target_sampler_output,
95
proposals=proposals,
@@ -99,14 +99,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
99
k=execute_model_req.num_lookahead_slots,
100
)
101
102
- all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted
103
- return SpeculativeScores(
104
- probs=all_probs,
105
- token_ids=all_tokens,
106
- logprobs=spec_logprobs,
107
- hidden_states=all_hidden_states,
108
- )
109
-
102
def _expand_batch(
103
self,
104
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -143,13 +135,57 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
135
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
136
num_scoring_tokens)
137
138
+ def _contract_non_speculative(
139
+ self, scores: SpeculativeScores,
140
+ seq_group_metadata_list: List[SequenceGroupMetadata],
141
+ non_spec_indices: List[int], non_spec_outputs: SpeculativeScores,
142
+ has_prompt_log: bool) -> SpeculativeScores:
143
+ """
144
+ Augment input `scores` with non-speculative requests outputs.
145
+ This includes decode requests with speculation turned off, as well
146
+ as prefill requests when `enable_chunked_prefill` is set.
147
+ For the latter, prefills are further separated into terminal and
148
+ non-terminal chunks (from which no token is sampled).
149
+ """
150
+ if not non_spec_indices:
151
+ return scores
152
+
153
+ if has_prompt_log:
154
+ # When prompt_logprobs is enabled, prefills yield output token
155
+ # (and respective prob) in the last entry (prompt|out):
156
+ # [.|.|.|prefill0_out|.|prefill1_out|decode0_out|..].
157
+ # With chunked prefill, non-terminal chunks have -1 on each
158
+ # position: they're still picked, but they're discarded later.
159
+ seq_meta = seq_group_metadata_list
160
+ nospec_sizes = torch.tensor([
161
+ seq_meta[i].token_chunk_size if seq_meta[i].is_prompt else 1
162
+ for i in non_spec_indices
163
+ ])
164
+ nospec_sampled_token_idxs = torch.cumsum(nospec_sizes, 0).add_(-1)
165
+ else:
166
+ # In this case only sampled tokens are returned, select all.
167
+ nospec_sampled_token_idxs = list(
168
+ range(len(non_spec_outputs.token_ids)))
169
+
170
+ scores.token_ids[non_spec_indices, :1] = \
171
+ non_spec_outputs.token_ids[nospec_sampled_token_idxs].unsqueeze(1)
172
+ scores.probs[non_spec_indices, :1, :] = \
173
+ non_spec_outputs.probs[nospec_sampled_token_idxs].unsqueeze(1)
174
+ scores.logprobs[non_spec_indices, :1, :] = \
175
+ non_spec_outputs.logprobs[nospec_sampled_token_idxs].unsqueeze(1)
176
+ if scores.hidden_states is not None:
177
+ assert non_spec_outputs.hidden_states is not None
178
+ scores.hidden_states[non_spec_indices, :1, :] = \
179
+ non_spec_outputs.hidden_states[nospec_sampled_token_idxs].unsqueeze(1)
180
+ return scores
181
+
182
def _contract_batch(
147
- self, contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
148
- target_sampler_output: SamplerOutput, proposals: SpeculativeProposals,
149
- num_scoring_tokens: int, non_spec_indices: List[int],
150
- spec_indices: List[int], k: int
151
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
152
- Optional[torch.Tensor]]:
183
+ self,
184
+ contracted_seq_group_metadata_list: List[SequenceGroupMetadata],
185
+ target_sampler_output: SamplerOutput,
186
+ proposals: SpeculativeProposals, num_scoring_tokens: int,
187
+ non_spec_indices: List[int], spec_indices: List[int],
188
+ k: int) -> SpeculativeScores:
189
"""Contract the expanded batch back into its original size.
190
This maps the scores of speculative tokens back to their original
191
sequences.
@@ -195,23 +231,28 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
231
else:
232
all_hidden_states = None
233
198
- # Rule out prefills that produce no tokens.
199
- non_spec_indices = [
200
- idx for idx in non_spec_indices
201
- if contracted_seq_group_metadata_list[idx].do_sample
202
- ]
203
- if len(non_spec_indices):
204
- all_tokens[non_spec_indices, :1] = \
205
- non_spec_target_token_ids.unsqueeze(1)
206
- all_probs[non_spec_indices, :1, :] = \
207
- non_spec_target_probs.unsqueeze(1)
208
- all_logprobs[non_spec_indices, :1, :] = \
209
- non_spec_target_logprobs.unsqueeze(1)
210
- if all_hidden_states is not None:
211
- assert non_spec_target_hidden_states is not None
212
- all_hidden_states[non_spec_indices, :1, :] = \
213
- non_spec_target_hidden_states.unsqueeze(1)
214
-
234
+ has_prompt_log = any((sg.sampling_params.prompt_logprobs
235
+ and sg.sampling_params.prompt_logprobs > 0)
236
+ for sg in contracted_seq_group_metadata_list)
237
+ # When prompt logprobs is enabled, lens of returned tensors go from
238
+ # n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
239
+ # We adjust stride accordingly to get the generated tokens and
240
+ # their probs, but pass on prompt_logprobs as is.
241
+ prompt_logprobs = None
242
+ if (not self._scorer_worker.model_runner.disable_logprobs\
243
+ and has_prompt_log):
244
+ prompt_logprobs = [
245
+ o.prompt_logprobs for o in target_sampler_output.outputs
246
+ ]
247
+ elif not has_prompt_log:
248
+ # When prompt logprobs are not to be returned,
249
+ # we can ignore non-terminal chunks (no out token).
250
+ non_spec_indices = [
251
+ idx for idx in non_spec_indices
252
+ if contracted_seq_group_metadata_list[idx].do_sample
253
+ ]
254
+
255
+ # "Contract" speculative.
256
if spec_indices:
257
all_tokens[spec_indices] = target_token_ids
258
all_probs[spec_indices] = target_probs
@@ -219,14 +260,27 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
260
if all_hidden_states is not None:
261
all_hidden_states[spec_indices] = target_hidden_states
262
222
- return all_tokens, all_probs, all_logprobs, all_hidden_states
263
+ spec_scores = SpeculativeScores(probs=all_probs,
264
+ token_ids=all_tokens,
265
+ logprobs=all_logprobs,
266
+ hidden_states=all_hidden_states,
267
+ prompt_logprobs=prompt_logprobs)
268
+
269
+ non_spec_outputs = SpeculativeScores(
270
+ probs=non_spec_target_probs,
271
+ token_ids=non_spec_target_token_ids,
272
+ logprobs=non_spec_target_logprobs,
273
+ hidden_states=non_spec_target_hidden_states)
274
+ # Contract remaining nonspec entries based on non_spec_indices, if any.
275
+ return self._contract_non_speculative(
276
+ spec_scores, contracted_seq_group_metadata_list, non_spec_indices,
277
+ non_spec_outputs, has_prompt_log)
278
279
def _contract_batch_all_spec(
280
self,
281
target_sampler_output: SamplerOutput,
282
proposals: SpeculativeProposals,
228
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
229
- Optional[torch.Tensor]]:
283
+ ) -> SpeculativeScores:
284
"""Contract the expanded batch back into its original size.
285
This maps the scores of speculative tokens back to their original
286
sequences.
@@ -250,8 +304,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
304
target_hidden_states = target_hidden_states.reshape(
305
*target_token_ids.shape, target_hidden_states.shape[-1])
306
253
- return (target_token_ids, target_probs, target_logprobs,
254
- target_hidden_states)
307
+ return SpeculativeScores(probs=target_probs,
308
+ token_ids=target_token_ids,
309
+ logprobs=target_logprobs,
310
+ hidden_states=target_hidden_states,
311
+ prompt_logprobs=None)
312
313
def _create_scoring_model_input(
314
self,
vllm/spec_decode/interfaces.py
+6
-2
@@ -1,10 +1,10 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
-from typing import Optional, Set, Union
3
+from typing import List, Optional, Set, Union
4
5
import torch
6
7
-from vllm.sequence import ExecuteModelRequest
7
+from vllm.sequence import ExecuteModelRequest, PromptLogprobs
8
from vllm.worker.worker_base import WorkerBase
9
10
@@ -54,6 +54,10 @@ class SpeculativeScores:
54
# Optional last hidden states from the scoring model.
55
hidden_states: Optional[torch.Tensor] = None
56
57
+ # Scoring model may also return logprobs for prompt tokens
58
+ # for each request, when chunked prefill is enabled.
59
+ prompt_logprobs: Optional[List[PromptLogprobs]] = None
60
+
61
def __repr__(self):
62
return (f"SpeculativeScores("
63
f"probs={self.probs.shape}, "
vllm/spec_decode/mqa_scorer.py
+56
-12
@@ -72,9 +72,15 @@ class MQAScorer(SpeculativeScorer):
72
target_token_ids = target_sampler_output.sampled_token_ids
73
target_probs = target_sampler_output.sampled_token_probs
74
target_logprobs = target_sampler_output.logprobs
75
+ prompt_logprobs = None
76
+
77
# If all requests have the same number of query tokens, we can avoid
78
# the for loop to build output for better performance.
79
if min(all_proposal_lengths) == k:
80
+ # Regular decodes only.
81
+ assert all(not sg.is_prompt
82
+ for sg in target_seq_group_metadata_list
83
+ if sg.is_prompt)
84
bs, _ = proposals.proposal_token_ids.shape
85
all_tokens = target_token_ids.reshape(bs, k + 1)
86
all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
@@ -88,19 +94,56 @@ class MQAScorer(SpeculativeScorer):
94
all_logprobs = target_logprobs.new_full(size=all_probs.shape,
95
fill_value=-float("inf"))
96
target_token_ids = target_token_ids.flatten()
91
- start_loc = 0
92
- for i, (proposed_len, seq_meta) in enumerate(
93
- zip(all_proposal_lengths, target_seq_group_metadata_list)):
97
+
98
+ # When prompt logprobs is enabled, lens of returned tensors go from
99
+ # n_sampled (requests with do_sample=True) to n_prompt+n_prefills.
100
+ # We adjust stride accordingly to get the generated tokens and
101
+ # their probs, but pass on prompt_logprobs as is, since it may be
102
+ # that n_prompts >> K.
103
+ has_prompt_log = any((sg.sampling_params.prompt_logprobs
104
+ and sg.sampling_params.prompt_logprobs > 0)
105
+ for sg in target_seq_group_metadata_list)
106
+ # TODO (NickLucche) we should surface `disable_logprobs` as to not
107
+ # break abstraction to get its value.
108
+ if (not self._scorer_worker.model_runner.disable_logprobs\
109
+ and has_prompt_log):
110
+ prompt_logprobs = [
111
+ o.prompt_logprobs for o in target_sampler_output.outputs
112
+ ]
113
+
114
+ # Split loop into prefill|decode for readability.
115
+ start_loc, i = 0, 0
116
+ while i < len(target_seq_group_metadata_list
117
+ ) and target_seq_group_metadata_list[i].is_prompt:
118
+ seq_meta = target_seq_group_metadata_list[i]
119
+ end_loc = start_loc
120
+ if has_prompt_log:
121
+ end_loc += seq_meta.token_chunk_size
122
+ elif seq_meta.do_sample:
123
+ end_loc += 1
124
+
125
# Skip chunks with no output tokens.
126
if seq_meta.do_sample:
96
- output_len = proposed_len + 1
97
- end_loc = start_loc + output_len
98
- all_tokens[
99
- i, :output_len] = target_token_ids[start_loc:end_loc]
100
- all_probs[i, :output_len] = target_probs[start_loc:end_loc]
101
- all_logprobs[
102
- i, :output_len] = target_logprobs[start_loc:end_loc]
103
- start_loc = end_loc
127
+ # Get sampled token (last position in chunk) and its prob.
128
+ all_tokens[i, 0] = target_token_ids[end_loc - 1]
129
+ all_probs[i, 0] = target_probs[end_loc - 1]
130
+ all_logprobs[i, 0] = target_logprobs[end_loc - 1]
131
+
132
+ i += 1
133
+ start_loc = end_loc
134
+ # Decodes.
135
+ while i < len(target_seq_group_metadata_list):
136
+ proposed_len, seq_meta = all_proposal_lengths[
137
+ i], target_seq_group_metadata_list[i]
138
+ output_len = proposed_len + 1
139
+ end_loc = start_loc + output_len
140
+ all_tokens[
141
+ i, :output_len] = target_token_ids[start_loc:end_loc]
142
+ all_probs[i, :output_len] = target_probs[start_loc:end_loc]
143
+ all_logprobs[
144
+ i, :output_len] = target_logprobs[start_loc:end_loc]
145
+ start_loc = end_loc
146
+ i += 1
147
148
hidden_states = None
149
if target_sampler_output.hidden_states is not None:
@@ -110,4 +153,5 @@ class MQAScorer(SpeculativeScorer):
153
return SpeculativeScores(probs=all_probs,
154
token_ids=all_tokens,
155
logprobs=all_logprobs,
113
- hidden_states=hidden_states)
156
+ hidden_states=hidden_states,
157
+ prompt_logprobs=prompt_logprobs)
vllm/spec_decode/spec_decode_worker.py
+157
-54
@@ -563,50 +563,57 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
563
(seq_id, seq_data) for sg in \
564
execute_model_req.seq_group_metadata_list \
565
for seq_id, seq_data in sg.seq_data.items()
566
- if sg.do_sample # ignore empty token sequences
566
]
567
completion_seq_group_output_list: List[
568
CompletionSequenceGroupOutput] = []
569
output_index = 0
570
# Make sure the non-terminal prefill chunks are still aligned with
571
# their own empty output.
573
- for seq_group_meta in execute_model_req.seq_group_metadata_list:
574
- # Since we can get chunks here, we dont always have a sampled token
575
- # (only on last chunk) but we still have to provide an output.
576
- if not seq_group_meta.do_sample:
577
- completion_seq_group_output_list.append(
578
- CompletionSequenceGroupOutput(samples=[],
579
- prompt_logprobs=None))
580
- else:
581
- # Sequence with output.
582
- seq_id, seq_data = seq_data_entries[output_index]
583
- needs_prompt_logprobs = seq_output_prompt_logprobs[
584
- output_index]
585
- if needs_prompt_logprobs:
586
- prompt_token_ids = seq_data.get_prompt_token_ids()
587
- prompt_logprobs = [
588
- create_logprobs_output(
589
- token_id=p_token_id,
590
- token_id_logprob_rank=-1,
591
- token_id_logprob=0.0,
592
- topk_token_ids=[],
593
- topk_logprobs=[],
594
- )
595
- # no prompt logprobs for the first token
596
- for p_token_id in prompt_token_ids[1:]
597
- ]
598
- else:
599
- prompt_logprobs = None
600
- completion_seq_group_output_list.append(
601
- create_sequence_group_output(
602
- token_id=sampled_token_ids_list[output_index][0],
572
+ for idx, seq_group_meta in enumerate(
573
+ execute_model_req.seq_group_metadata_list):
574
+ needs_prompt_logprobs = seq_output_prompt_logprobs[idx]
575
+ seq_id, seq_data = seq_data_entries[idx]
576
+ if needs_prompt_logprobs:
577
+ prompt_token_ids = seq_data.get_prompt_token_ids()
578
+
579
+ # Some of these sequences may belong to non-terminal chunks,
580
+ # which may still have to report logprobs for prompts.
581
+ start = 1 if seq_data._num_computed_tokens == 0 \
582
+ else seq_data._num_computed_tokens
583
+ end = (seq_data._num_computed_tokens + \
584
+ seq_group_meta.token_chunk_size)
585
+ prompt_token_ids = prompt_token_ids[start:end]
586
+ prompt_logprobs = [
587
+ create_logprobs_output(
588
+ token_id=p_token_id,
589
token_id_logprob_rank=-1,
590
token_id_logprob=0.0,
605
- seq_id=seq_id,
591
topk_token_ids=[],
592
topk_logprobs=[],
608
- prompt_logprobs=prompt_logprobs))
609
- output_index += 1
593
+ ) for p_token_id in prompt_token_ids
594
+ ]
595
+ else:
596
+ prompt_logprobs = None
597
+
598
+ # Since we can get chunks here, we dont always have a sampled token
599
+ # (only on last chunk) but we still have to provide an output.
600
+ if not seq_group_meta.do_sample:
601
+ completion_seq_group_output_list.append(
602
+ CompletionSequenceGroupOutput(
603
+ samples=[], prompt_logprobs=prompt_logprobs))
604
+ continue
605
+
606
+ # Sequence with output.
607
+ completion_seq_group_output_list.append(
608
+ create_sequence_group_output(
609
+ token_id=sampled_token_ids_list[output_index][0],
610
+ token_id_logprob_rank=-1,
611
+ token_id_logprob=0.0,
612
+ seq_id=seq_id,
613
+ topk_token_ids=[],
614
+ topk_logprobs=[],
615
+ prompt_logprobs=prompt_logprobs))
616
+ output_index += 1
617
618
return [SamplerOutput(outputs=completion_seq_group_output_list)]
619
@@ -624,24 +631,27 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
631
assert len(sampler_output) == 1
632
sampler_output = sampler_output[0]
633
627
- # Store hidden states from target model execution.
634
+ # Store hidden states from target model execution, BxD.
635
hidden_states = sampler_output.hidden_states
636
if hidden_states is not None:
630
- # remove hidden_states for prompt tokens
631
- # TODO Enable `return_hidden_states`: prefill chunks hidden states
632
- # are pruned by the logits processor. Also, they should be arranged
633
- # back into full-prefill latent. Address it to enable MLPSpeculator.
634
- if any(seq.is_prompt
635
- for seq in execute_model_req.seq_group_metadata_list):
637
+ # Only decodes and prefill terminal chunks need a hidden state.
638
+ seq_group_meta_with_hidden = [
639
+ sg for sg in execute_model_req.seq_group_metadata_list
640
+ if sg.do_sample
641
+ ]
642
+ if any(seq.is_prompt for seq in seq_group_meta_with_hidden):
643
+ # Drop hidden_states with no prediction (eg non-terminal chunks)
644
hidden_states = hidden_states[
645
torch.where(sampler_output.sampled_token_ids -
646
VLLM_INVALID_TOKEN_ID)[0]]
639
- if self.previous_hidden_states is None:
647
+ if self.previous_hidden_states is None and len(
648
+ seq_group_meta_with_hidden):
649
self.previous_hidden_states = HiddenStates(
641
- hidden_states, execute_model_req.seq_group_metadata_list)
642
- else:
643
- self.previous_hidden_states.update(
644
- hidden_states, execute_model_req.seq_group_metadata_list)
650
+ hidden_states, seq_group_meta_with_hidden)
651
+ elif self.previous_hidden_states and len(
652
+ seq_group_meta_with_hidden):
653
+ self.previous_hidden_states.update(hidden_states,
654
+ seq_group_meta_with_hidden)
655
656
if not skip_proposer:
657
# We prepare the prefill hidden states here so that there no
@@ -752,13 +762,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
762
]
763
if len(non_spec_indices):
764
all_hidden_states = proposal_scores.hidden_states
755
- # TODO fix `return_hidden_states`, same as in `_run_no_spec`
765
if all_hidden_states is not None:
766
prefill_hidden_states = all_hidden_states[non_spec_indices]
767
execute_model_req.previous_hidden_states = \
768
prepare_prefill_hidden_states(prefill_hidden_states)
769
# Sync proposer KV cache for prefills.
770
prefill_req = execute_model_req.clone(non_spec_seqs)
771
+ # TODO avoid sampling here?
772
self.proposer_worker.execute_model(prefill_req)
773
774
with Timer() as verification_timer:
@@ -774,6 +784,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
784
execute_model_req.seq_group_metadata_list,
785
accepted_token_ids,
786
target_logprobs=target_logprobs,
787
+ prompt_logprobs=proposal_scores.prompt_logprobs
788
+ if not self._disable_logprobs else None,
789
k=execute_model_req.num_lookahead_slots,
790
stage_times=stage_times)
791
@@ -845,19 +857,32 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
857
# metadata.
858
accepted_token_ids[original_indices] = accepted_token_ids.clone()
859
860
+ # B x K+1 x D
861
hidden_states = proposal_scores.hidden_states
862
if hidden_states is not None:
863
+ # Only get terminal hidden states for next step
864
+ terminal_metadata = [
865
+ sg for sg in seq_group_metadata_list if sg.do_sample
866
+ ]
867
+
868
# Contract hidden states based on accepted tokens
869
hs_size = hidden_states.shape[-1]
852
-
870
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
854
- accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
855
- index = accepted_index[:, None, None].expand(-1, 1, hs_size)
871
+ accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) # b
872
+ # Drop non-terminal prefill chunks hidden states.
873
+ hidden_states = hidden_states[
874
+ accepted_index != VLLM_INVALID_TOKEN_ID]
875
+ accepted_index = accepted_index[
876
+ accepted_index != VLLM_INVALID_TOKEN_ID]
877
+ assert len(accepted_index) == hidden_states.shape[0] == len(
878
+ terminal_metadata)
879
+ index = accepted_index[:, None, None].expand(-1, 1,
880
+ hs_size) # b x 1 x d
881
second_last_token_hidden_states = hidden_states[:, -2] # b x d
882
hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
883
# Store hidden states from target model for subsequent decode step
884
self.previous_hidden_states = HiddenStates(
860
- hidden_states, seq_group_metadata_list,
885
+ hidden_states, terminal_metadata,
886
second_last_token_hidden_states)
887
return accepted_token_ids, logprobs
888
@@ -866,6 +891,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
891
seq_group_metadata_list: List[SequenceGroupMetadata],
892
accepted_token_ids: torch.Tensor, # shape: [batch_size, k+1]
893
target_logprobs: torch.Tensor, # shape: [batch_size, k+1, vocab_size]
894
+ prompt_logprobs: Optional[
895
+ torch.Tensor], # shape: [nprompt_tokens, vocab_size]
896
k: int,
897
stage_times: Tuple[float, float, float],
898
) -> List[SamplerOutput]:
@@ -909,15 +936,89 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
936
937
# Construct the output on a per-step, per-sequence basis.
938
# Non-terminal prefill chunks will end up here as rows with just -1s
912
- # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]]
939
+ # i.e mixed-batch [[-1, 1576], [-1, 29884], [-1, -1], [-1, -1]] while
940
+ # terminal chunks will only have one generated token at time 0.
941
sampler_output_list: List[SamplerOutput] = []
942
+
943
+ # Prefills are not multi-step (return at most 1 token), in order to
944
+ # avoid padding or repetition to fit decodes, we separate them.
945
+ for i, sg in enumerate(seq_group_metadata_list):
946
+ if not sg.is_prompt:
947
+ # Requests are ordered as prefills|decodes=>no more prefills.
948
+ break
949
+ num_logprobs = num_logprobs_per_seq[i]
950
+ seq_kwargs = dict(token_id=-1,
951
+ token_id_logprob_rank=0,
952
+ token_id_logprob=-float('inf'),
953
+ topk_token_ids=[-1] * num_logprobs,
954
+ topk_logprobs=[-float('inf')] * num_logprobs,
955
+ seq_id=seq_ids[i])
956
+ # Terminal chunk, has token.
957
+ if sg.do_sample:
958
+ seq_kwargs.update(
959
+ dict(
960
+ token_id=accepted_token_ids[i][0].item(),
961
+ token_id_logprob_rank=accepted_token_id_ranks_by_step[
962
+ 0][i],
963
+ token_id_logprob=accepted_token_id_logprobs_by_step[0]
964
+ [i],
965
+ topk_token_ids=topk_indices_by_step[0][i]
966
+ [:num_logprobs],
967
+ # output only so step is 0
968
+ topk_logprobs=topk_logprobs_by_step[0][i]
969
+ [:num_logprobs],
970
+ ))
971
+ needs_plogs = (sg.sampling_params.prompt_logprobs
972
+ and sg.sampling_params.prompt_logprobs > 0)
973
+ plogs = None
974
+ if prompt_logprobs is not None:
975
+ # Even non-terminal prompt chunks can have logprobs here.
976
+ plogs = prompt_logprobs[i]
977
+ elif needs_plogs:
978
+ # Prompt logprobs are requested but `_disable_logprobs` is set.
979
+ seq_data = next(iter(sg.seq_data.values()))
980
+ # Get only the tokens in this chunk!
981
+ prompt_token_ids = seq_data.get_prompt_token_ids()
982
+ prompt_token_ids = prompt_token_ids[
983
+ seq_data.
984
+ _num_computed_tokens:seq_data._num_computed_tokens +
985
+ sg.token_chunk_size]
986
+
987
+ is_first_chunk = seq_data._num_computed_tokens == 0
988
+ # There's no prob generated for the first token in a sequence.
989
+ if is_first_chunk:
990
+ prompt_token_ids = prompt_token_ids[1:]
991
+ plogs = [
992
+ create_logprobs_output(
993
+ token_id=p_token_id,
994
+ token_id_logprob_rank=-1,
995
+ token_id_logprob=0.0,
996
+ topk_token_ids=[],
997
+ topk_logprobs=[],
998
+ ) for p_token_id in prompt_token_ids
999
+ ]
1000
+ seq_kwargs.update(dict(prompt_logprobs=plogs))
1001
+
1002
+ sampler_output_list.append(
1003
+ SamplerOutput(
1004
+ outputs=[create_sequence_group_output(
1005
+ **seq_kwargs)])) # type: ignore
1006
+
1007
+ # Decodes, create one SamplerOutput per-step (at most K+1).
1008
for step_index in range(num_steps):
915
- if all(token_id == -1
916
- for token_id in accepted_token_ids_by_step[step_index]):
1009
+ if all(token_id == -1 for sg, token_id in zip(
1010
+ seq_group_metadata_list,
1011
+ accepted_token_ids_by_step[step_index])
1012
+ if not sg.is_prompt):
1013
break
1014
1015
step_output_token_ids: List[CompletionSequenceGroupOutput] = []
1016
for sequence_index in range(batch_size):
1017
+ seq_meta = seq_group_metadata_list[sequence_index]
1018
+ # Prompts already processed above.
1019
+ if seq_meta.is_prompt:
1020
+ continue
1021
+
1022
# Each sequence may have a different num_logprobs; retrieve it.
1023
num_logprobs = num_logprobs_per_seq[sequence_index]
1024
step_output_token_ids.append(
@@ -952,6 +1053,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
1053
# This is periodic because the rejection sampler emits metrics
1054
# periodically.
1055
self._maybe_log_stage_times(*stage_times)
1056
+ # First `n_prefills` entries will contain prefills SamplerOutput when
1057
+ # chunked prefill is enabled, the rest is decodes in multi-step format.
1058
return sampler_output_list
1059
1060
def _maybe_log_stage_times(self, average_time_per_proposal_tok_ms: float,