A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Speculative decoding 7/9] Speculative decoding end-to-end correctness tests. (#3951)
Cade Daniel committed
Apr 23, 2024 at 01:02 UTC
62b8aebc6f06b5c8fafa1f27893cd4f9bb11fa8b
22 files changed
+1164
-175
tests/samplers/test_rejection_sampler.py
+6
-2
@@ -91,12 +91,16 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
91
bonus_token_ids,
92
)
93
94
+ # Bonus tokens are currently disabled. Verify they're set to -1.
95
+ # See https://github.com/vllm-project/vllm/issues/4212
96
+ expected_bonus_token_ids = bonus_token_ids.clone() * 0 - 1
97
+
98
if which_tokens_accepted == "all_tokens_accepted":
99
# Expect all tokens to be equal to draft tokens.
100
assert torch.equal(output_token_ids[:, :-1], draft_token_ids)
101
102
# Expect all bonus tokens to be included.
99
- assert torch.equal(output_token_ids[:, -1:], bonus_token_ids)
103
+ assert torch.equal(output_token_ids[:, -1:], expected_bonus_token_ids)
104
elif which_tokens_accepted == "no_tokens_accepted":
105
# Expect first token to be equal to recovered tokens.
106
assert torch.equal(output_token_ids[:, 0], recovered_token_ids[:, 0])
@@ -106,7 +110,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
110
torch.ones_like(output_token_ids[:, 1:]) * -1)
111
elif which_tokens_accepted == "some_tokens_accepted":
112
recovered_plus_bonus = torch.cat(
109
- (recovered_token_ids, bonus_token_ids), dim=-1)
113
+ (recovered_token_ids, expected_bonus_token_ids), dim=-1)
114
# Assert first rejected token is a recovered token or bonus token.
115
assert torch.equal(
116
recovered_plus_bonus[torch.arange(0, batch_size),
tests/samplers/test_sampler.py
+2
-1
@@ -636,7 +636,8 @@ def test_sampler_top_k_top_p(seed: int, device: str):
636
def mock_sample(probs, *args, **kwargs):
637
nonlocal sample_probs
638
sample_probs = probs
639
- return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
639
+ return ([[prob.topk(1, dim=-1).indices.tolist(), [0]]
640
+ for prob in probs], None)
641
642
with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
643
sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
tests/spec_decode/e2e/__init__.py
tests/spec_decode/e2e/conftest.py
+35
-10
@@ -1,3 +1,5 @@
1
+from typing import List, Tuple
2
+
3
import pytest
4
5
from tests.conftest import cleanup
@@ -6,28 +8,34 @@ from vllm.model_executor.utils import set_random_seed
8
9
10
@pytest.fixture
9
-def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
10
- baseline_llm_kwargs, seed):
11
- return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
11
+def baseline_llm_generator(request, common_llm_kwargs,
12
+ per_test_common_llm_kwargs, baseline_llm_kwargs,
13
+ seed):
14
+ return create_llm_generator("baseline", request, common_llm_kwargs,
15
+ per_test_common_llm_kwargs,
16
baseline_llm_kwargs, seed)
17
18
19
@pytest.fixture
16
-def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
20
+def test_llm_generator(request, common_llm_kwargs, per_test_common_llm_kwargs,
21
test_llm_kwargs, seed):
18
- return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
19
- test_llm_kwargs, seed)
22
+ return create_llm_generator("test", request, common_llm_kwargs,
23
+ per_test_common_llm_kwargs, test_llm_kwargs,
24
+ seed)
25
26
22
-def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
23
- distinct_llm_kwargs, seed):
27
+def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
28
+ per_test_common_llm_kwargs, distinct_llm_kwargs,
29
+ seed):
30
kwargs = {
31
**common_llm_kwargs,
32
**per_test_common_llm_kwargs,
33
**distinct_llm_kwargs,
34
}
35
+ test_name = request.node.name
36
37
def generator_inner():
38
+ print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
39
llm = LLM(**kwargs)
40
41
set_random_seed(seed)
@@ -36,6 +44,23 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
44
del llm
45
cleanup()
46
39
- for llm in generator_inner():
40
- yield llm
47
+ def generator_outer():
48
+ for llm in generator_inner():
49
+ yield llm
50
+ del llm
51
+
52
+ return generator_outer
53
+
54
+
55
+def get_output_from_llm_generator(
56
+ llm_generator, prompts,
57
+ sampling_params) -> Tuple[List[str], List[List[int]]]:
58
+ tokens = []
59
+ token_ids = []
60
+ for llm in llm_generator():
61
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
62
+ token_ids = [output.outputs[0].token_ids for output in outputs]
63
+ tokens = [output.outputs[0].text for output in outputs]
64
del llm
65
+
66
+ return tokens, token_ids
tests/spec_decode/e2e/test_compatibility.py
new
+169
@@ -0,0 +1,169 @@
1
+import pytest
2
+
3
+from vllm import SamplingParams
4
+
5
+from .conftest import get_output_from_llm_generator
6
+
7
+
8
+@pytest.mark.parametrize(
9
+ "common_llm_kwargs",
10
+ [{
11
+ "model": "JackFram/llama-68m",
12
+ "speculative_model": "JackFram/llama-68m",
13
+ "num_speculative_tokens": 5,
14
+
15
+ # Required for spec decode.
16
+ "use_v2_block_manager": True
17
+ }])
18
+@pytest.mark.parametrize(
19
+ "per_test_common_llm_kwargs",
20
+ [
21
+ {
22
+ # Expect failure as spec decode not supported by
23
+ # Ray backend.
24
+ "worker_use_ray": True,
25
+ },
26
+ ])
27
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
28
+@pytest.mark.parametrize("seed", [1])
29
+def test_spec_decode_xfail_ray(test_llm_generator):
30
+ """Verify that speculative decoding with Ray fails.
31
+ """
32
+ output_len = 128
33
+ temperature = 0.0
34
+
35
+ prompts = [
36
+ "Hello, my name is",
37
+ ]
38
+
39
+ sampling_params = SamplingParams(
40
+ max_tokens=output_len,
41
+ ignore_eos=True,
42
+ temperature=temperature,
43
+ )
44
+
45
+ with pytest.raises(AssertionError,
46
+ match="Speculative decoding not yet supported for "):
47
+ get_output_from_llm_generator(test_llm_generator, prompts,
48
+ sampling_params)
49
+
50
+
51
+@pytest.mark.parametrize(
52
+ "common_llm_kwargs",
53
+ [{
54
+ "model": "JackFram/llama-68m",
55
+ "speculative_model": "JackFram/llama-68m",
56
+ "num_speculative_tokens": 5,
57
+
58
+ # Required for spec decode.
59
+ "use_v2_block_manager": True
60
+ }])
61
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
62
+ {
63
+ "enable_chunked_prefill": True,
64
+ },
65
+])
66
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
67
+@pytest.mark.parametrize("seed", [1])
68
+def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
69
+ """Verify that speculative decoding with chunked prefill fails.
70
+ """
71
+ output_len = 128
72
+ temperature = 0.0
73
+
74
+ prompts = [
75
+ "Hello, my name is",
76
+ ]
77
+
78
+ sampling_params = SamplingParams(
79
+ max_tokens=output_len,
80
+ ignore_eos=True,
81
+ temperature=temperature,
82
+ )
83
+
84
+ with pytest.raises(ValueError,
85
+ match="Speculative decoding and chunked prefill"):
86
+ get_output_from_llm_generator(test_llm_generator, prompts,
87
+ sampling_params)
88
+
89
+
90
+@pytest.mark.parametrize(
91
+ "common_llm_kwargs",
92
+ [{
93
+ "model": "meta-llama/Llama-2-7b-chat-hf",
94
+ "speculative_model": "JackFram/llama-68m",
95
+ "num_speculative_tokens": 5,
96
+
97
+ # Required for spec decode.
98
+ "use_v2_block_manager": True
99
+ }])
100
+@pytest.mark.parametrize(
101
+ "per_test_common_llm_kwargs",
102
+ [
103
+ {
104
+ # Speculative max model len > overridden max model len should raise.
105
+ "max_model_len": 128,
106
+ "speculative_max_model_len": 129,
107
+ },
108
+ {
109
+ # Speculative max model len > draft max model len should raise.
110
+ # https://huggingface.co/JackFram/llama-68m/blob/3b606af5198a0b26762d589a3ee3d26ee6fa6c85/config.json#L12
111
+ "speculative_max_model_len": 2048 + 1,
112
+ },
113
+ {
114
+ # Speculative max model len > target max model len should raise.
115
+ # https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/f5db02db724555f92da89c216ac04704f23d4590/config.json#L12
116
+ "speculative_max_model_len": 4096 + 1,
117
+ },
118
+ ])
119
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
120
+@pytest.mark.parametrize("seed", [1])
121
+def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
122
+ """Verify that speculative decoding validates speculative_max_model_len.
123
+ """
124
+ output_len = 128
125
+ temperature = 0.0
126
+
127
+ prompts = [
128
+ "Hello, my name is",
129
+ ]
130
+
131
+ sampling_params = SamplingParams(
132
+ max_tokens=output_len,
133
+ ignore_eos=True,
134
+ temperature=temperature,
135
+ )
136
+
137
+ with pytest.raises(ValueError, match="cannot be larger than"):
138
+ get_output_from_llm_generator(test_llm_generator, prompts,
139
+ sampling_params)
140
+
141
+
142
+@pytest.mark.parametrize("common_llm_kwargs", [{
143
+ "model": "JackFram/llama-68m",
144
+ "speculative_model": "JackFram/llama-68m",
145
+ "num_speculative_tokens": 5,
146
+}])
147
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
148
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
149
+@pytest.mark.parametrize("seed", [1])
150
+def test_spec_decode_xfail_block_manager_v1(test_llm_generator):
151
+ """Verify that speculative decoding with block manager v1 fails.
152
+ """
153
+ output_len = 128
154
+ temperature = 0.0
155
+
156
+ prompts = [
157
+ "Hello, my name is",
158
+ ]
159
+
160
+ sampling_params = SamplingParams(
161
+ max_tokens=output_len,
162
+ ignore_eos=True,
163
+ temperature=temperature,
164
+ )
165
+
166
+ with pytest.raises(ValueError,
167
+ match="Speculative decoding requires usage of the V2"):
168
+ get_output_from_llm_generator(test_llm_generator, prompts,
169
+ sampling_params)
tests/spec_decode/e2e/test_correctness.py
+493
-47
@@ -1,11 +1,42 @@
1
+"""The tests in this file verify end-to-end speculative decoding correctness.
2
+
3
+This docstring details important information on the testing methodology.
4
+
5
+Most of the tests rely on "greedy equality", where we expect the output of
6
+speculative decoding on a sequence to exactly match the output of normal non-
7
+speculative decoding.
8
+
9
+Since speculative decoding with rejection sampling guarantees that the output
10
+distribution matches the target model's output distribution (up to hardware
11
+numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
12
+equality. This gives us good coverage of temp=0.
13
+
14
+For temp>0, we rely on unit tests on the rejection sampler to verify that the
15
+output distribution is the same with spec decode vs. no spec decode (this would
16
+be prohibitively expensive to run with a real model).
17
+
18
+NOTE: Speculative decoding's distribution equality requires that the measured
19
+distributions of the target model and proposal model be deterministic given the
20
+same input. vLLM largely guarantees this.
21
+
22
+@cadedaniel has seen cases where the output probabilities of a draft/target
23
+model change slightly with certain batch sizes or prompts, even with Torch
24
+determinism flags set. It is unclear if this is a bug in vLLM, due to non-
25
+determinism in on-device batched operations, a bug in vLLM's spec decode
26
+implementation, or the "hardware numerics" limitations. Either way, rejection
27
+sampling ensures the output distribution matches the target model, but it breaks
28
+greedy-equality tests for those batch sizes/prompts.
29
+"""
30
+
31
from itertools import cycle
2
-from typing import List, Tuple
32
33
import pytest
34
from transformers import AutoTokenizer
35
36
from vllm import SamplingParams
37
38
+from .conftest import get_output_from_llm_generator
39
+
40
41
@pytest.mark.parametrize(
42
"common_llm_kwargs",
@@ -14,9 +45,6 @@ from vllm import SamplingParams
45
# Note this is repeated in the test body; to initialize a tokenizer.
46
"model": "JackFram/llama-68m",
47
17
- # Skip real loading for fast test.
18
- "load_format": "dummy",
19
-
48
# Skip cuda graph recording for fast test.
49
"enforce_eager": True,
50
@@ -31,22 +59,15 @@ from vllm import SamplingParams
59
"num_speculative_tokens": 5,
60
},
61
{
34
- "speculative_model": "JackFram/llama-68m",
35
- "num_speculative_tokens": 1,
36
- },
37
- {
38
- # No spec decode.
62
+ # Verify the detokenizer assertions in the test work when spec
63
+ # decode is disabled.
64
},
65
])
66
@pytest.mark.parametrize("test_llm_kwargs", [{}])
42
-@pytest.mark.parametrize("batch_size", [1])
43
-# NOTE: We should run more permutations of this test (more BS, more seeds). But
44
-# because our spec decode generates gibberish token ids, the likelihood of
45
-# emitting an invalid token combination is nontrivial. This causes divergence in
46
-# behavior of vLLM detokenization vs. hf tokenizer, for example when two "utf-
47
-# start" bytes are emitted.
67
+@pytest.mark.parametrize("batch_size", [1, 32])
68
@pytest.mark.parametrize("seed", [1])
49
-def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
69
+def test_spec_decode_e2e_with_detokenization(test_llm_generator,
70
+ batch_size: int):
71
"""Run generation with speculative decoding on a batch. Verify the engine
72
generates the correct number of tokens (via ignore_eos=True), and that the
73
detokenization matches HF transformers.
@@ -67,8 +88,6 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
88
max_tokens=output_len,
89
ignore_eos=True,
90
temperature=temperature,
70
- skip_special_tokens=True,
71
- spaces_between_special_tokens=False,
91
)
92
93
batch_tokens, batch_token_ids = get_output_from_llm_generator(
@@ -77,9 +96,10 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
96
# Expect a generation for each prompt in the batch.
97
assert len(batch_token_ids) == len(prompts)
98
80
- # Expect each generation to have expected number of tokens (note
81
- # ignore_eos=True).
82
- assert all(len(token_ids) == output_len for token_ids in batch_token_ids)
99
+ # Expect each generation to have expected number of tokens (note ignore_eos
100
+ # is True).
101
+ assert [len(token_ids)
102
+ for token_ids in batch_token_ids] == ([output_len] * batch_size)
103
104
# Expect detokenized string to match.
105
tok = AutoTokenizer.from_pretrained("JackFram/llama-68m")
@@ -92,14 +112,111 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
112
@pytest.mark.parametrize(
113
"common_llm_kwargs",
114
[{
95
- # Use a small model for a fast test.
96
- "model": "JackFram/llama-68m",
115
+ # Skip cuda graph recording for fast test.
116
+ "enforce_eager": True,
117
+
118
+ # Required for spec decode.
119
+ "use_v2_block_manager": True,
120
+
121
+ # Print spec metrics.
122
+ "disable_log_stats": False,
123
+ }])
124
+@pytest.mark.parametrize(
125
+ "per_test_common_llm_kwargs",
126
+ [
127
+ # Try two different tiny base models.
128
+ # Note that one is equal to the draft model, another isn't.
129
+ {
130
+ "model": "JackFram/llama-68m",
131
+ },
132
+ {
133
+ "model": "JackFram/llama-160m",
134
+ },
135
+ ])
136
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
137
+@pytest.mark.parametrize("test_llm_kwargs", [
138
+ {
139
"speculative_model": "JackFram/llama-68m",
140
"num_speculative_tokens": 5,
141
+ },
142
+])
143
+@pytest.mark.parametrize(
144
+ "output_len",
145
+ [
146
+ # Use long output len for the small model test.
147
+ 1536,
148
+ ])
149
+@pytest.mark.parametrize("batch_size", [1])
150
+@pytest.mark.parametrize("seed", [1])
151
+def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
152
+ baseline_llm_generator, test_llm_generator, batch_size: int,
153
+ output_len: int):
154
+ """Verify greedy equality on a tiny model with batch size of one.
155
+
156
+ Since this test is cheaper than other e2e correctness tests, we generate
157
+ with a higher output_len.
158
+ """
159
+ run_greedy_equality_correctness_test(baseline_llm_generator,
160
+ test_llm_generator,
161
+ batch_size,
162
+ max_output_len=output_len,
163
+ force_output_len=True)
164
100
- # Skip real loading for fast test.
101
- "load_format": "dummy",
165
166
+@pytest.mark.parametrize(
167
+ "common_llm_kwargs",
168
+ [{
169
+ # Skip cuda graph recording for fast test.
170
+ "enforce_eager": True,
171
+
172
+ # Required for spec decode.
173
+ "use_v2_block_manager": True,
174
+
175
+ # Print spec metrics.
176
+ "disable_log_stats": False,
177
+ }])
178
+@pytest.mark.parametrize(
179
+ "per_test_common_llm_kwargs",
180
+ [
181
+ # Try two different tiny base models.
182
+ # Note that one is equal to the draft model, another isn't.
183
+ {
184
+ "model": "JackFram/llama-68m",
185
+ },
186
+ {
187
+ "model": "JackFram/llama-160m",
188
+ },
189
+ ])
190
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
191
+@pytest.mark.parametrize("test_llm_kwargs", [
192
+ {
193
+ "speculative_model": "JackFram/llama-68m",
194
+ "num_speculative_tokens": 5,
195
+ },
196
+])
197
+@pytest.mark.parametrize(
198
+ "output_len",
199
+ [
200
+ # Use small output len for fast test.
201
+ 256,
202
+ ])
203
+@pytest.mark.parametrize("batch_size", [64])
204
+@pytest.mark.parametrize("seed", [1])
205
+def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
206
+ baseline_llm_generator, test_llm_generator, batch_size: int,
207
+ output_len: int):
208
+ """Verify greedy equality on a tiny model and large batch size.
209
+ """
210
+ run_greedy_equality_correctness_test(baseline_llm_generator,
211
+ test_llm_generator,
212
+ batch_size,
213
+ max_output_len=output_len,
214
+ force_output_len=True)
215
+
216
+
217
+@pytest.mark.parametrize(
218
+ "common_llm_kwargs",
219
+ [{
220
# Skip cuda graph recording for fast test.
221
"enforce_eager": True,
222
@@ -109,43 +226,372 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
226
@pytest.mark.parametrize(
227
"per_test_common_llm_kwargs",
228
[
229
+ # Try two different tiny base models.
230
+ # Note that one is equal to the draft model, another isn't.
231
{
113
- # Expect failure as spec decode not supported by
114
- # Ray backend.
115
- "worker_use_ray": True,
232
+ "model": "JackFram/llama-68m",
233
+ },
234
+ {
235
+ "model": "JackFram/llama-160m",
236
},
237
])
118
-@pytest.mark.parametrize("test_llm_kwargs", [{}])
238
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
239
+@pytest.mark.parametrize("test_llm_kwargs", [
240
+ {
241
+ "speculative_model": "JackFram/llama-68m",
242
+ "num_speculative_tokens": 5,
243
+ },
244
+])
245
+@pytest.mark.parametrize("max_output_len", [
246
+ 256,
247
+])
248
+@pytest.mark.parametrize("batch_size", [32])
249
+@pytest.mark.parametrize("seed", [1])
250
+def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
251
+ baseline_llm_generator, test_llm_generator, batch_size: int,
252
+ max_output_len: int):
253
+ """Verify greedy equality on a tiny model, with a large batch size, and when
254
+ sampling respects the EOS token.
255
+ """
256
+ run_greedy_equality_correctness_test(baseline_llm_generator,
257
+ test_llm_generator,
258
+ batch_size,
259
+ max_output_len,
260
+ force_output_len=False)
261
+
262
+
263
+@pytest.mark.parametrize(
264
+ "common_llm_kwargs",
265
+ [{
266
+ # A "real" model (not tiny).
267
+ "model": "meta-llama/Llama-2-7b-chat-hf",
268
+
269
+ # Skip cuda graph recording for fast test.
270
+ "enforce_eager": True,
271
+
272
+ # Required for spec decode.
273
+ "use_v2_block_manager": True,
274
+
275
+ # Print spec metrics.
276
+ "disable_log_stats": False,
277
+ }])
278
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
279
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
280
+@pytest.mark.parametrize("test_llm_kwargs", [
281
+ {
282
+ "speculative_model": "JackFram/llama-68m",
283
+ "num_speculative_tokens": 5,
284
+ },
285
+])
286
+@pytest.mark.parametrize("batch_size", [1])
287
+@pytest.mark.parametrize(
288
+ "output_len",
289
+ [
290
+ # Use decently long output len for a high quality test.
291
+ 256,
292
+ ])
293
+@pytest.mark.parametrize("seed", [1])
294
+def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
295
+ baseline_llm_generator, test_llm_generator, batch_size: int,
296
+ output_len: int):
297
+ """Verify greedy equality on a "real" model and batch size of 1. This is
298
+ separate from large BS tests to make identifying the source of bugs easier.
299
+ """
300
+ run_greedy_equality_correctness_test(baseline_llm_generator,
301
+ test_llm_generator,
302
+ batch_size,
303
+ max_output_len=output_len,
304
+ force_output_len=True)
305
+
306
+
307
+@pytest.mark.parametrize(
308
+ "common_llm_kwargs",
309
+ [{
310
+ # A "real" model (not tiny).
311
+ "model": "meta-llama/Llama-2-7b-chat-hf",
312
+
313
+ # Skip cuda graph recording for fast test.
314
+ "enforce_eager": True,
315
+
316
+ # Required for spec decode.
317
+ "use_v2_block_manager": True,
318
+
319
+ # Print spec metrics.
320
+ "disable_log_stats": False,
321
+ }])
322
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
323
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
324
+@pytest.mark.parametrize("test_llm_kwargs", [
325
+ {
326
+ "speculative_model": "JackFram/llama-68m",
327
+ "num_speculative_tokens": 5,
328
+ },
329
+])
330
+@pytest.mark.parametrize("batch_size", [32])
331
+@pytest.mark.parametrize(
332
+ "output_len",
333
+ [
334
+ # Use smaller output len for fast test.
335
+ 64,
336
+ ])
337
@pytest.mark.parametrize("seed", [1])
120
-def test_spec_decode_xfail(test_llm_generator):
121
- """Verify that speculative decoding with Ray fails.
338
+def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
339
+ baseline_llm_generator, test_llm_generator, batch_size: int,
340
+ output_len: int):
341
+ """Verify greedy equality with a "real" model on a nontrivial batch size.
342
+ This is the closest test to a real production workload.
343
+ """
344
+ run_greedy_equality_correctness_test(baseline_llm_generator,
345
+ test_llm_generator,
346
+ batch_size,
347
+ max_output_len=output_len,
348
+ force_output_len=True)
349
+
350
+
351
+@pytest.mark.parametrize(
352
+ "common_llm_kwargs",
353
+ [{
354
+ "block_size": 8,
355
+ # 2 for small prompt, 256//8 for generated.
356
+ "num_gpu_blocks_override": 2 + 256 // 8,
357
+ "max_model_len": (2 + 256 // 8) * 8,
358
+
359
+ # Skip cuda graph recording for fast test.
360
+ "enforce_eager": True,
361
+
362
+ # Required for spec decode.
363
+ "use_v2_block_manager": True
364
+ }])
365
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
366
+ {
367
+ "model": "JackFram/llama-160m",
368
+ },
369
+])
370
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
371
+@pytest.mark.parametrize("test_llm_kwargs", [
372
+ {
373
+ "speculative_model": "JackFram/llama-68m",
374
+ "num_speculative_tokens": 5,
375
+ },
376
+])
377
+@pytest.mark.parametrize(
378
+ "output_len",
379
+ [
380
+ # Use small output len for fast test.
381
+ 256,
382
+ ])
383
+@pytest.mark.parametrize("batch_size", [4])
384
+@pytest.mark.parametrize("seed", [1])
385
+def test_spec_decode_e2e_greedy_correctness_with_preemption(
386
+ baseline_llm_generator, test_llm_generator, batch_size: int,
387
+ output_len: int):
388
+ """Verify greedy equality, even when some sequences are preempted mid-
389
+ generation.
390
+ """
391
+ run_greedy_equality_correctness_test(baseline_llm_generator,
392
+ test_llm_generator,
393
+ batch_size,
394
+ max_output_len=output_len,
395
+ force_output_len=True)
396
+
397
+
398
+@pytest.mark.parametrize(
399
+ "common_llm_kwargs",
400
+ [{
401
+ "model": "JackFram/llama-160m",
402
+
403
+ # Skip cuda graph recording for fast test.
404
+ "enforce_eager": True,
405
+
406
+ # Required for spec decode.
407
+ "use_v2_block_manager": True
408
+ }])
409
+@pytest.mark.parametrize(
410
+ "per_test_common_llm_kwargs",
411
+ [
412
+ # As of this writing, vLLM only compiles with these 3 block sizes by
413
+ # default.
414
+ {
415
+ "block_size": 8,
416
+ },
417
+ {
418
+ "block_size": 16,
419
+ },
420
+ {
421
+ "block_size": 32,
422
+ },
423
+ ])
424
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
425
+@pytest.mark.parametrize("test_llm_kwargs", [
426
+ {
427
+ "speculative_model": "JackFram/llama-68m",
428
+ "num_speculative_tokens": 5,
429
+ },
430
+])
431
+@pytest.mark.parametrize("batch_size", [2])
432
+@pytest.mark.parametrize(
433
+ "output_len",
434
+ [
435
+ # Use smaller output len for fast test.
436
+ 32,
437
+ ])
438
+@pytest.mark.parametrize("seed", [1])
439
+def test_spec_decode_different_block_size(baseline_llm_generator,
440
+ test_llm_generator, batch_size: int,
441
+ output_len: int):
442
+ """Verify greedy equality over different block sizes.
443
+ """
444
+ run_greedy_equality_correctness_test(baseline_llm_generator,
445
+ test_llm_generator,
446
+ batch_size,
447
+ max_output_len=output_len,
448
+ force_output_len=True)
449
+
450
+
451
+@pytest.mark.parametrize(
452
+ "common_llm_kwargs",
453
+ [{
454
+ "model": "JackFram/llama-160m",
455
+
456
+ # Skip cuda graph recording for fast test.
457
+ "enforce_eager": True,
458
+
459
+ # Required for spec decode.
460
+ "use_v2_block_manager": True
461
+ }])
462
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
463
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
464
+@pytest.mark.parametrize(
465
+ "test_llm_kwargs",
466
+ [
467
+ {
468
+ "speculative_model": "JackFram/llama-68m",
469
+ "num_speculative_tokens": 5,
470
+
471
+ # Artificially limit the draft model max model len; this forces vLLM
472
+ # to skip speculation once the sequences grow beyond 32-k tokens.
473
+ "speculative_max_model_len": 32,
474
+ },
475
+ ])
476
+@pytest.mark.parametrize("batch_size", [8])
477
+@pytest.mark.parametrize(
478
+ "output_len",
479
+ [
480
+ # This must be a good bit larger than speculative_max_model_len so that
481
+ # we can test the case where all seqs are skipped, but still small to
482
+ # ensure fast test.
483
+ 64,
484
+ ])
485
+@pytest.mark.parametrize("seed", [1])
486
+def test_skip_speculation(baseline_llm_generator, test_llm_generator,
487
+ batch_size: int, output_len: int):
488
+ """Verify greedy equality when some (or all) sequences skip speculation.
489
+ We do this by setting the max model len of the draft model to an
490
+ artificially low value, such that when the sequences grow beyond it, they
491
+ are skipped in speculative decoding.
492
+ """
493
+ run_greedy_equality_correctness_test(baseline_llm_generator,
494
+ test_llm_generator,
495
+ batch_size,
496
+ max_output_len=output_len,
497
+ force_output_len=True)
498
+
499
+
500
+@pytest.mark.parametrize(
501
+ "common_llm_kwargs",
502
+ [{
503
+ "model": "JackFram/llama-68m",
504
+
505
+ # Skip cuda graph recording for fast test.
506
+ "enforce_eager": True,
507
+
508
+ # Required for spec decode.
509
+ "use_v2_block_manager": True
510
+ }])
511
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
512
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
513
+@pytest.mark.parametrize(
514
+ "test_llm_kwargs",
515
+ [
516
+ {
517
+ "speculative_model": "JackFram/llama-68m",
518
+ "num_speculative_tokens": k,
519
+ }
520
+ # Try a range of common k, as well as large speculation.
521
+ for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]
522
+ ])
523
+@pytest.mark.parametrize("batch_size", [2])
524
+@pytest.mark.parametrize(
525
+ "output_len",
526
+ [
527
+ # Use smaller output len for fast test.
528
+ 32,
529
+ ])
530
+@pytest.mark.parametrize("seed", [1])
531
+def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
532
+ output_len: int):
533
+ """Verify that speculative decoding produces exact equality to without spec
534
+ decode with many different values of k.
535
+ """
536
+ run_greedy_equality_correctness_test(baseline_llm_generator,
537
+ test_llm_generator,
538
+ batch_size,
539
+ max_output_len=output_len,
540
+ force_output_len=True)
541
+
542
+
543
+def run_greedy_equality_correctness_test(baseline_llm_generator,
544
+ test_llm_generator,
545
+ batch_size,
546
+ max_output_len,
547
+ force_output_len: bool,
548
+ print_tokens: bool = False):
549
+ """Helper method that compares the outputs of both the baseline LLM and
550
+ the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
551
+ the same when temperature is zero.
552
"""
123
- output_len = 128
553
temperature = 0.0
554
555
prompts = [
556
"Hello, my name is",
557
+ "The president of the United States is",
558
+ "The capital of France is",
559
+ "The future of AI is",
560
+ "San Francisco is know for its",
561
+ "Facebook was created in 2004 by",
562
+ "Curious George is a",
563
+ "Python 3.11 brings improvements to its",
564
]
565
566
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
567
+
568
+ # If the test requires that we generated max_output_len tokens, then set the
569
+ # sampling params to ignore eos token.
570
+ ignore_eos = force_output_len
571
+
572
sampling_params = SamplingParams(
131
- max_tokens=output_len,
132
- ignore_eos=True,
573
+ max_tokens=max_output_len,
574
+ ignore_eos=ignore_eos,
575
temperature=temperature,
576
)
577
136
- with pytest.raises(AssertionError,
137
- match="Speculative decoding not yet supported for "):
138
- get_output_from_llm_generator(test_llm_generator, prompts,
139
- sampling_params)
578
+ spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
579
+ test_llm_generator, prompts, sampling_params)
580
581
+ (baseline_batch_tokens,
582
+ baseline_batch_token_ids) = get_output_from_llm_generator(
583
+ baseline_llm_generator, prompts, sampling_params)
584
142
-def get_output_from_llm_generator(
143
- llm_generator, prompts,
144
- sampling_params) -> Tuple[List[str], List[List[int]]]:
145
- for llm in llm_generator:
146
- outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
147
- token_ids = [output.outputs[0].token_ids for output in outputs]
148
- tokens = [output.outputs[0].text for output in outputs]
149
- del llm
585
+ assert len(baseline_batch_token_ids) == len(prompts)
586
+ assert len(spec_batch_token_ids) == len(prompts)
587
151
- return tokens, token_ids
588
+ for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
589
+ spec_tokens) in enumerate(
590
+ zip(baseline_batch_token_ids, baseline_batch_tokens,
591
+ spec_batch_token_ids, spec_batch_tokens)):
592
+ if print_tokens:
593
+ print(f'{i=} {baseline_tokens=}')
594
+ print(f'{i=} {spec_tokens=}')
595
+ print(f'{i=} {baseline_token_ids=}')
596
+ print(f'{i=} {spec_token_ids=}')
597
+ assert baseline_token_ids == spec_token_ids
tests/spec_decode/test_metrics.py
+2
-2
@@ -119,7 +119,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
119
num_draft_tokens = 0
120
k = 5
121
122
- num_possible_tokens = AsyncMetricsCollector.get_max_num_accepted_tokens(
122
+ max_num_emitted_tokens = AsyncMetricsCollector.get_max_num_emitted_tokens(
123
num_draft_tokens, k)
124
125
rej_sampler = MagicMock()
@@ -153,7 +153,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
153
assert (metrics.draft_acceptance_rate == num_accepted_tokens /
154
num_draft_tokens)
155
assert (metrics.system_efficiency == num_emitted_tokens /
156
- num_possible_tokens)
156
+ max_num_emitted_tokens)
157
else:
158
assert math.isnan(metrics.draft_acceptance_rate)
159
assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py
+2
-2
@@ -344,8 +344,8 @@ def test_draft_proposals_no_speculations():
344
assert torch.is_tensor(proposals.proposal_token_ids)
345
assert torch.is_tensor(proposals.proposal_probs)
346
347
- assert proposals.proposal_token_ids.shape == torch.Size([0, k])
348
- assert proposals.proposal_probs.shape[:-1] == torch.Size([0, k])
347
+ assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
348
+ assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
349
350
assert proposals.proposal_lens.shape == torch.Size([batch_size])
351
assert proposals.proposal_lens.tolist() == [0 for _ in range(batch_size)]
tests/spec_decode/test_spec_decode_worker.py
+23
-17
@@ -1,4 +1,5 @@
1
import random
2
+from types import SimpleNamespace
3
from unittest.mock import MagicMock
4
5
import pytest
@@ -62,8 +63,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
63
"""Verify SpecDecodeWorker calls the target model with correct
64
inputs. Everything else is mocked out.
65
"""
65
- draft_worker = mock_worker(cls=MultiStepWorker)
66
- target_worker = mock_worker()
66
+ draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
67
+ target_worker = mock_worker(use_spec=False)
68
rejection_sampler = MagicMock(spec=RejectionSampler)
69
rejection_sampler.token_id_dtype = torch.int64
70
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -144,8 +145,10 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
145
"""
146
vocab_size = 32_000
147
147
- draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
148
- target_worker = mock_worker(vocab_size=vocab_size)
148
+ draft_worker = mock_worker(cls=MultiStepWorker,
149
+ vocab_size=vocab_size,
150
+ use_spec=False)
151
+ target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
152
rejection_sampler = MagicMock(spec=RejectionSampler)
153
rejection_sampler.token_id_dtype = torch.int64
154
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -202,17 +205,16 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
205
num_lookahead_slots=k)
206
207
assert len(rejection_sampler.call_args_list) == 1
205
- args, _ = rejection_sampler.call_args_list[0]
206
- (actual_proposal_scores, actual_bonus_token_ids, actual_proposal_probs,
207
- actual_proposal_token_ids) = args
208
+ _, kwargs = rejection_sampler.call_args_list[0]
209
+ actual = SimpleNamespace(**kwargs)
210
209
- assert torch.equal(actual_bonus_token_ids,
211
+ assert torch.equal(actual.bonus_token_ids,
212
target_token_ids.reshape(batch_size, k + 1)[:, -1:])
213
assert torch.equal(
212
- actual_proposal_scores,
214
+ actual.target_probs,
215
target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
214
- assert torch.equal(actual_proposal_token_ids, proposal_token_ids)
215
- assert torch.equal(actual_proposal_probs, proposal_probs)
216
+ assert torch.equal(actual.draft_token_ids, proposal_token_ids)
217
+ assert torch.equal(actual.draft_probs, proposal_probs)
218
219
220
@pytest.mark.parametrize('k', [1, 2, 6])
@@ -224,8 +226,10 @@ def test_correctly_formats_output(k: int, batch_size: int):
226
"""
227
vocab_size = 32_000
228
227
- draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
228
- target_worker = mock_worker(vocab_size=vocab_size)
229
+ draft_worker = mock_worker(cls=MultiStepWorker,
230
+ vocab_size=vocab_size,
231
+ use_spec=False)
232
+ target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
233
rejection_sampler = MagicMock(spec=RejectionSampler)
234
rejection_sampler.token_id_dtype = torch.int64
235
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -336,8 +340,10 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
340
"""
341
vocab_size = 32_000
342
339
- draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
340
- target_worker = mock_worker(vocab_size=vocab_size)
343
+ draft_worker = mock_worker(cls=MultiStepWorker,
344
+ vocab_size=vocab_size,
345
+ use_spec=False)
346
+ target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
347
rejection_sampler = MagicMock(spec=RejectionSampler)
348
rejection_sampler.token_id_dtype = torch.int64
349
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -500,8 +506,8 @@ def test_init_device():
506
"""Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
507
well as other GPU initialization.
508
"""
503
- draft_worker = mock_worker(cls=MultiStepWorker)
504
- target_worker = mock_worker()
509
+ draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
510
+ target_worker = mock_worker(use_spec=False)
511
rejection_sampler = MagicMock(spec=RejectionSampler)
512
rejection_sampler.token_id_dtype = torch.int64
513
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
tests/spec_decode/utils.py
+5
-2
@@ -63,11 +63,14 @@ def create_execute_model_data(
63
def mock_worker(cls=None,
64
vocab_size: int = 30_000,
65
max_model_len: int = 2048,
66
- rank: int = 0) -> MagicMock:
66
+ rank: int = 0,
67
+ use_spec: bool = True) -> MagicMock:
68
if cls is None:
69
cls = Worker
70
70
- worker = MagicMock(spec=cls)
71
+ spec = cls if use_spec else None
72
+
73
+ worker = MagicMock(spec=spec)
74
worker.vocab_size = vocab_size
75
worker.max_model_len = max_model_len
76
worker.rank = rank
vllm/config.py
+65
-2
@@ -655,6 +655,9 @@ class SpeculativeConfig:
655
target_dtype: str,
656
speculative_model: Optional[str],
657
num_speculative_tokens: Optional[int],
658
+ speculative_max_model_len: Optional[int],
659
+ enable_chunked_prefill: bool,
660
+ use_v2_block_manager: bool,
661
) -> Optional["SpeculativeConfig"]:
662
"""Create a SpeculativeConfig if possible, else return None.
663
@@ -672,6 +675,15 @@ class SpeculativeConfig:
675
model, if provided.
676
num_speculative_tokens (Optional[int]): The number of speculative
677
tokens, if provided.
678
+ speculative_max_model_len (Optional[int]): The maximum model len of
679
+ the speculative model. Used when testing the ability to skip
680
+ speculation for some sequences.
681
+ enable_chunked_prefill (bool): Whether vLLM is configured to use
682
+ chunked prefill or not. Used for raising an error since its not
683
+ yet compatible with spec decode.
684
+ use_v2_block_manager (bool): Whether vLLM is configured to use the
685
+ v2 block manager or not. Used for raising an error since the v2
686
+ block manager is required with spec decode.
687
688
Returns:
689
Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -690,12 +702,21 @@ class SpeculativeConfig:
702
assert (speculative_model is not None
703
and num_speculative_tokens is not None)
704
705
+ if enable_chunked_prefill:
706
+ raise ValueError(
707
+ "Speculative decoding and chunked prefill are "
708
+ f"currently mutually exclusive ({enable_chunked_prefill=}).")
709
+
710
+ if not use_v2_block_manager:
711
+ raise ValueError(
712
+ "Speculative decoding requires usage of the V2 "
713
+ "block manager. Enable it with --use-v2-block-manager.")
714
+
715
# TODO: The user should be able to specify revision/quantization/max
716
# model len for the draft model. It is not currently supported.
717
draft_revision = None
718
draft_code_revision = None
719
draft_quantization = None
698
- draft_max_model_len = None
720
721
draft_model_config = ModelConfig(
722
model=speculative_model,
@@ -707,7 +728,7 @@ class SpeculativeConfig:
728
revision=draft_revision,
729
code_revision=draft_code_revision,
730
tokenizer_revision=target_model_config.tokenizer_revision,
710
- max_model_len=draft_max_model_len,
731
+ max_model_len=None,
732
quantization=draft_quantization,
733
enforce_eager=target_model_config.enforce_eager,
734
max_context_len_to_capture=target_model_config.
@@ -715,6 +736,13 @@ class SpeculativeConfig:
736
max_logprobs=target_model_config.max_logprobs,
737
)
738
739
+ draft_model_config.max_model_len = (
740
+ SpeculativeConfig._maybe_override_draft_max_model_len(
741
+ speculative_max_model_len,
742
+ draft_model_config.max_model_len,
743
+ target_model_config.max_model_len,
744
+ ))
745
+
746
draft_parallel_config = (
747
SpeculativeConfig.create_draft_parallel_config(
748
target_parallel_config))
@@ -725,6 +753,41 @@ class SpeculativeConfig:
753
num_speculative_tokens,
754
)
755
756
+ @staticmethod
757
+ def _maybe_override_draft_max_model_len(
758
+ speculative_max_model_len: Optional[int],
759
+ draft_max_model_len: int,
760
+ target_max_model_len: int,
761
+ ) -> int:
762
+ """Determine the max sequence len for the draft model. This is usually
763
+ the draft_max_model_len, but may be the target_max_model_len if it is
764
+ less than the draft_max_model_len, or may be speculative_max_model_len
765
+ if it is specified.
766
+
767
+ This is necessary so that sequences do not exceed the capacity of the
768
+ draft model or the target model.
769
+
770
+ speculative_max_model_len is mainly used for testing that sequences can
771
+ skip speculation.
772
+ """
773
+
774
+ if speculative_max_model_len is not None:
775
+
776
+ if speculative_max_model_len > draft_max_model_len:
777
+ raise ValueError(f"{speculative_max_model_len=} cannot be "
778
+ f"larger than {draft_max_model_len=}")
779
+
780
+ if speculative_max_model_len > target_max_model_len:
781
+ raise ValueError(f"{speculative_max_model_len=} cannot be "
782
+ f"larger than {target_max_model_len=}")
783
+
784
+ return speculative_max_model_len
785
+
786
+ return min(
787
+ draft_max_model_len,
788
+ target_max_model_len,
789
+ )
790
+
791
@staticmethod
792
def create_draft_parallel_config(
793
target_parallel_config: ParallelConfig) -> ParallelConfig:
vllm/engine/arg_utils.py
+15
-3
@@ -73,6 +73,7 @@ class EngineArgs:
73
# Speculative decoding configuration.
74
speculative_model: Optional[str] = None
75
num_speculative_tokens: Optional[int] = None
76
+ speculative_max_model_len: Optional[int] = None
77
78
def __post_init__(self):
79
if self.tokenizer is None:
@@ -237,7 +238,7 @@ class EngineArgs:
238
parser.add_argument('--block-size',
239
type=int,
240
default=EngineArgs.block_size,
240
- choices=[8, 16, 32, 128],
241
+ choices=[8, 16, 32],
242
help='Token block size for contiguous chunks of '
243
'tokens.')
244
@@ -420,17 +421,25 @@ class EngineArgs:
421
parser.add_argument(
422
'--speculative-model',
423
type=str,
423
- default=None,
424
+ default=EngineArgs.speculative_model,
425
help=
426
'The name of the draft model to be used in speculative decoding.')
427
428
parser.add_argument(
429
'--num-speculative-tokens',
430
type=int,
430
- default=None,
431
+ default=EngineArgs.num_speculative_tokens,
432
help='The number of speculative tokens to sample from '
433
'the draft model in speculative decoding.')
434
435
+ parser.add_argument(
436
+ '--speculative-max-model-len',
437
+ type=str,
438
+ default=EngineArgs.speculative_max_model_len,
439
+ help='The maximum sequence length supported by the '
440
+ 'draft model. Sequences over this length will skip '
441
+ 'speculation.')
442
+
443
parser.add_argument('--model-loader-extra-config',
444
type=str,
445
default=EngineArgs.model_loader_extra_config,
@@ -481,6 +490,9 @@ class EngineArgs:
490
target_dtype=self.dtype,
491
speculative_model=self.speculative_model,
492
num_speculative_tokens=self.num_speculative_tokens,
493
+ speculative_max_model_len=self.speculative_max_model_len,
494
+ enable_chunked_prefill=self.enable_chunked_prefill,
495
+ use_v2_block_manager=self.use_v2_block_manager,
496
)
497
498
scheduler_config = SchedulerConfig(
vllm/engine/llm_engine.py
+30
-8
@@ -22,7 +22,7 @@ from vllm.lora.request import LoRARequest
22
from vllm.outputs import RequestOutput
23
from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25
- SequenceGroup)
25
+ SequenceGroup, SequenceStage)
26
from vllm.transformers_utils.detokenizer import Detokenizer
27
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28
get_tokenizer_group)
@@ -480,9 +480,12 @@ class LLMEngine:
480
seq_group = scheduled_seq_group.seq_group
481
seq_group.update_num_computed_tokens(
482
scheduled_seq_group.token_chunk_size)
483
- # If uncomputed tokens > 0, it means prefill is chunked.
484
- # We don't need to process outputs in that case.
485
- if seq_group.get_num_uncomputed_tokens() == 0:
483
+
484
+ # If all sequences in the sequence group are in DECODE, then we can
485
+ # process the output tokens. Otherwise, they are (chunked) prefill
486
+ # samples and should not be processed.
487
+ stages = [seq.data._stage for seq in seq_group.seqs_dict.values()]
488
+ if all(stage == SequenceStage.DECODE for stage in stages):
489
self.output_processor.process_outputs(seq_group, outputs)
490
491
# Free the finished sequence groups.
@@ -569,7 +572,8 @@ class LLMEngine:
572
573
# Log stats.
574
if self.log_stats:
572
- self.stat_logger.log(self._get_stats(scheduler_outputs))
575
+ self.stat_logger.log(
576
+ self._get_stats(scheduler_outputs, model_output=output))
577
578
return request_outputs
579
@@ -578,9 +582,18 @@ class LLMEngine:
582
if self.log_stats:
583
self.stat_logger.log(self._get_stats(scheduler_outputs=None))
584
581
- def _get_stats(self,
582
- scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
583
- """Get Stats to be Logged to Prometheus."""
585
+ def _get_stats(
586
+ self,
587
+ scheduler_outputs: Optional[SchedulerOutputs],
588
+ model_output: Optional[List[SamplerOutput]] = None) -> Stats:
589
+ """Get Stats to be Logged to Prometheus.
590
+
591
+ Args:
592
+ scheduler_outputs: Optional, used to populate metrics related to
593
+ the scheduled batch,
594
+ model_output: Optional, used to emit speculative decoding metrics
595
+ which are created by the workers.
596
+ """
597
now = time.time()
598
599
# KV Cache Usage in %.
@@ -637,6 +650,14 @@ class LLMEngine:
650
time_to_first_tokens = time_last_iters if prompt_run else []
651
time_per_output_tokens = [] if prompt_run else time_last_iters
652
653
+ # Spec decode, if enabled, emits specialized metrics from the worker in
654
+ # sampler output.
655
+ if model_output and (model_output[0].spec_decode_worker_metrics
656
+ is not None):
657
+ spec_decode_metrics = model_output[0].spec_decode_worker_metrics
658
+ else:
659
+ spec_decode_metrics = None
660
+
661
return Stats(
662
now=now,
663
num_running=num_running,
@@ -649,6 +670,7 @@ class LLMEngine:
670
time_to_first_tokens=time_to_first_tokens,
671
time_per_output_tokens=time_per_output_tokens,
672
time_e2e_requests=time_e2e_requests,
673
+ spec_decode_metrics=spec_decode_metrics,
674
)
675
676
def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/engine/metrics.py
+22
-1
@@ -1,6 +1,6 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import Dict, List, Protocol
3
+from typing import TYPE_CHECKING, Dict, List, Optional, Protocol
4
5
import numpy as np
6
from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -8,6 +8,9 @@ from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
8
9
from vllm.logger import init_logger
10
11
+if TYPE_CHECKING:
12
+ from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
13
+
14
logger = init_logger(__name__)
15
16
disable_created_metrics()
@@ -118,6 +121,8 @@ class Stats:
121
time_per_output_tokens: List[float]
122
time_e2e_requests: List[float]
123
124
+ spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
125
+
126
127
class SupportsMetricsInfo(Protocol):
128
@@ -235,3 +240,19 @@ class StatLogger:
240
self.num_prompt_tokens = []
241
self.num_generation_tokens = []
242
self.last_local_log = stats.now
243
+
244
+ if stats.spec_decode_metrics is not None:
245
+ logger.info(
246
+ self._format_spec_decode_metrics_str(
247
+ stats.spec_decode_metrics))
248
+
249
+ def _format_spec_decode_metrics_str(
250
+ self, metrics: "SpecDecodeWorkerMetrics") -> str:
251
+
252
+ return ("Speculative metrics: "
253
+ f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
254
+ f"System efficiency: {metrics.system_efficiency:.3f}, "
255
+ f"Number of speculative tokens: {metrics.num_spec_tokens}, "
256
+ f"Number of accepted tokens: {metrics.accepted_tokens}, "
257
+ f"Number of draft tokens tokens: {metrics.draft_tokens}, "
258
+ f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
vllm/executor/gpu_executor.py
+1
@@ -83,6 +83,7 @@ class GPUExecutor(ExecutorBase):
83
scheduler_config=self.scheduler_config,
84
device_config=self.device_config,
85
cache_config=self.cache_config,
86
+ # TODO allow draft-model specific load config.
87
load_config=self.load_config,
88
local_rank=0,
89
rank=0,
vllm/model_executor/layers/rejection_sampler.py
+7
@@ -144,6 +144,7 @@ class RejectionSampler(nn.Module):
144
recovered_probs = self._get_recovered_probs(
145
target_probs, draft_probs).reshape(batch_size * k, vocab_size)
146
147
+ # NOTE: the recovered_probs are overwritten by this method.
148
recovered_token_ids = _multinomial(recovered_probs,
149
num_samples=1).reshape(
150
batch_size, k)
@@ -307,6 +308,12 @@ class RejectionSampler(nn.Module):
308
output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
309
bonus_token_ids, -1)
310
311
+ # We disable bonus tokens because it causes corrupt KV cache for
312
+ # proposal methods that require KV cache. We can fix it by "prefilling"
313
+ # the bonus token in the proposer. The following issue tracks the fix.
314
+ # https://github.com/vllm-project/vllm/issues/4212
315
+ output_with_bonus_tokens[:, -1] = -1
316
+
317
# Fill the recovered token ids.
318
output.mul_(~after_false_mask).add_(
319
recovered_token_ids.mul(after_false_mask))
vllm/model_executor/layers/sampler.py
+167
-15
@@ -35,6 +35,14 @@ class Sampler(nn.Module):
35
in logits for each token in the input prompt.
36
"""
37
38
+ def __init__(self):
39
+ super().__init__()
40
+
41
+ # Whether or not the SamplerOutput should have on-device tensors
42
+ # containing the sampled token ids and probabilities. This is used by
43
+ # speculative decoding.
44
+ self.include_gpu_probs_tensor = False
45
+
46
def forward(
47
self,
48
logits: torch.Tensor,
@@ -79,13 +87,45 @@ class Sampler(nn.Module):
87
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
88
89
# Sample the next tokens.
82
- sample_results = _sample(probs, logprobs, sampling_metadata,
83
- sampling_tensors)
90
+ sample_results, maybe_sampled_tokens_tensor = _sample(
91
+ probs,
92
+ logprobs,
93
+ sampling_metadata,
94
+ sampling_tensors,
95
+ include_gpu_probs_tensor=self.include_gpu_probs_tensor,
96
+ modify_greedy_probs=self._should_modify_greedy_probs_inplace,
97
+ )
98
+
99
+ if self.include_gpu_probs_tensor:
100
+ assert maybe_sampled_tokens_tensor is not None
101
+ sampled_tokens_tensor = maybe_sampled_tokens_tensor
102
+ on_device_tensors = (probs, sampled_tokens_tensor)
103
+ else:
104
+ on_device_tensors = None
105
+
106
# Get the logprobs query results.
107
prompt_logprobs, sample_logprobs = _get_logprobs(
108
logprobs, sampling_metadata, sample_results)
87
- return _build_sampler_output(sample_results, sampling_metadata,
88
- prompt_logprobs, sample_logprobs)
109
+ return _build_sampler_output(sample_results,
110
+ sampling_metadata,
111
+ prompt_logprobs,
112
+ sample_logprobs,
113
+ on_device_tensors=on_device_tensors)
114
+
115
+ @property
116
+ def _should_modify_greedy_probs_inplace(self) -> bool:
117
+ """Whether or not the sampler should modify the probability distribution
118
+ of greedily-sampled tokens such that multinomial sampling would sample
119
+ the greedily-sampled token.
120
+
121
+ In other words, if True then we set the probability of the greedily-
122
+ sampled token to 1.
123
+
124
+ This is used by speculative decoding, which requires that the sampling
125
+ method be encoded into the probability distribution.
126
+ """
127
+ # Modify greedy probs if include_gpu_probs_tensor is set.
128
+ return self.include_gpu_probs_tensor
129
130
131
def _get_bin_counts_and_mask(
@@ -359,7 +399,9 @@ def _sample_with_torch(
399
probs: torch.Tensor,
400
logprobs: torch.Tensor,
401
sampling_metadata: SamplingMetadata,
362
-) -> List[Tuple[List[int], List[int]]]:
402
+ include_gpu_probs_tensor: bool,
403
+ modify_greedy_probs: bool,
404
+) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
405
categorized_seq_group_ids = {t: [] for t in SamplingType}
406
categorized_sample_indices = sampling_metadata.categorized_sample_indices
407
for i, seq_group in enumerate(sampling_metadata.seq_groups):
@@ -371,6 +413,15 @@ def _sample_with_torch(
413
sample_metadata = {}
414
multinomial_samples = {}
415
416
+ # Create output tensor for sampled token ids.
417
+ if include_gpu_probs_tensor:
418
+ sampled_token_ids_tensor = torch.empty(logprobs.shape[0],
419
+ 1,
420
+ dtype=torch.long,
421
+ device=logprobs.device)
422
+ else:
423
+ sampled_token_ids_tensor = None
424
+
425
# Counterintiutively, having two loops here is actually faster.
426
# The first loop can run without waiting on GPU<->CPU sync.
427
for sampling_type in SamplingType:
@@ -383,9 +434,25 @@ def _sample_with_torch(
434
is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
435
sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
436
is_prompts, sample_indices)
437
+ long_sample_indices = sample_indices.long()
438
+
439
if sampling_type == SamplingType.GREEDY:
387
- greedy_samples = torch.argmax(logprobs[sample_indices.long()],
440
+ greedy_samples = torch.argmax(logprobs[long_sample_indices],
441
dim=-1)
442
+
443
+ if include_gpu_probs_tensor:
444
+ # Store sampled tokens in output tensor.
445
+ sampled_token_ids_tensor[
446
+ long_sample_indices] = greedy_samples.unsqueeze(-1)
447
+
448
+ if modify_greedy_probs:
449
+ # If required, modify the probabilities such that sampling from
450
+ # the modified distribution would always sample the argmax
451
+ # token id.
452
+ _modify_greedy_probs_inplace(logprobs, probs,
453
+ long_sample_indices,
454
+ greedy_samples)
455
+
456
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
457
max_best_of_in_batch = 1
458
for seq_group, is_prompt in zip(seq_groups, is_prompts):
@@ -397,15 +464,23 @@ def _sample_with_torch(
464
"seq_groups": seq_groups,
465
"generators": sampling_metadata.generators,
466
}
467
+
468
multinomial_samples[sampling_type] = _multinomial(
401
- probs[sample_indices.long()], max_best_of_in_batch,
469
+ probs[long_sample_indices], max_best_of_in_batch,
470
**seeded_args)
471
+
472
+ if include_gpu_probs_tensor:
473
+ # Store sampled tokens in output tensor.
474
+ sampled_token_ids_tensor[
475
+ long_sample_indices] = multinomial_samples[sampling_type]
476
+
477
elif sampling_type == SamplingType.BEAM:
478
beam_search_logprobs = logprobs[sample_indices]
479
else:
480
raise ValueError(f"Unsupported sampling type: {sampling_type}")
481
482
# GPU<->CPU sync happens in the loop below.
483
+ # This also converts the sample output to Python objects.
484
485
for sampling_type in SamplingType:
486
if sampling_type not in sample_metadata:
@@ -427,7 +502,7 @@ def _sample_with_torch(
502
sample_results_dict[i]
503
for i in range(len(sampling_metadata.seq_groups))
504
]
430
- return sample_results
505
+ return sample_results, sampled_token_ids_tensor
506
507
508
def _sample_with_triton_kernel(
@@ -511,12 +586,17 @@ def _sample_with_triton_kernel(
586
587
588
def _sample(
514
- probs: torch.Tensor,
515
- logprobs: torch.Tensor,
516
- sampling_metadata: SamplingMetadata,
517
- sampling_tensors: SamplingTensors,
518
-) -> List[Tuple[List[int], List[int]]]:
519
- return _sample_with_torch(probs, logprobs, sampling_metadata)
589
+ probs: torch.Tensor, logprobs: torch.Tensor,
590
+ sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
591
+ include_gpu_probs_tensor: bool, modify_greedy_probs: bool
592
+) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
593
+ return _sample_with_torch(
594
+ probs,
595
+ logprobs,
596
+ sampling_metadata,
597
+ include_gpu_probs_tensor=include_gpu_probs_tensor,
598
+ modify_greedy_probs=modify_greedy_probs,
599
+ )
600
601
# TODO: Enable once Triton kernel & associated code is faster.
602
# return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
@@ -680,12 +760,73 @@ def _get_logprobs(
760
return result_prompt_logprobs, result_sample_logprobs
761
762
763
+def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
764
+ sample_indices: torch.Tensor,
765
+ greedy_samples: torch.Tensor) -> None:
766
+ """Modify the probability distributions of the greedily-sampled tokens such
767
+ that each sampled token has a "probability" of 1.0. This is required by
768
+ speculative decoding, which depends on the sampling method being encoded
769
+ within the probability distribution for correctness.
770
+
771
+ # Why do we only need to do this for greedy sampling?
772
+
773
+ vLLM's sampler performs the following steps for greedy or multinomial
774
+ (random) sampling:
775
+ 1. Get logits from model.
776
+ 2. Modify logits according to per-sequence sampling parameters.
777
+ - Multiply by temperature, top-k and top-p masking, penalize tokens
778
+ according to their frequency, etc.
779
+ 3. Sample a token.
780
+ - Random sampling simply samples from the modified probability
781
+ distribution.
782
+ - Greedy sampling performs `argmax` to obtain the token with the
783
+ highest likelihood.
784
+
785
+ Ignoring greedy sampling for a moment, we find that the computed probability
786
+ distribution has the following property: we can sample from it independently
787
+ and find that the token sampled by the Sampler has a frequency corresponding
788
+ to how often we see it in our sampling. In other words, for tokens sampled
789
+ with vLLM's random SamplingType, the computed probability distribution
790
+ encodes the sampling methodology completely.
791
+
792
+ Greedy sampling does not normally have this property. vLLM modifies logits
793
+ according to sampling params, then performs `argmax`, then returns the
794
+ sampled token and the computed probability distribution. If we sample from
795
+ the distribution, we'll find the likelihood of the greedily-sampled token
796
+ is not always 1.0.
797
+
798
+ Since lossless speculative decoding requires that the sampling methodology
799
+ be encoded within the probability distribution, we are motivated to modify
800
+ the probability distribution such that the sampled token has probability 1
801
+ when speculative decoding is used.
802
+
803
+ NOTE: Alternatively, we could use an extremely low temperature to achieve
804
+ greedy sampling using multinomial computation and unite the codepaths. This
805
+ has implications on the overall design of the sampler, e.g. how to record
806
+ accurate logprobs for the user, so this improvement is deferred to later.
807
+ """
808
+ logprobs[sample_indices, :] = -float('inf')
809
+ logprobs[sample_indices, greedy_samples] = 0.0
810
+ probs[sample_indices, :] = 0
811
+ probs[sample_indices, greedy_samples] = 1.0
812
+
813
+
814
def _build_sampler_output(
815
sample_results: List[Tuple[List[int], List[int]]],
816
sampling_metadata: SamplingMetadata,
817
prompt_logprobs: List[Optional[PromptLogprobs]],
818
sample_logprobs: List[SampleLogprobs],
819
+ on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor]],
820
) -> SamplerOutput:
821
+ """Construct Python objects with the output of sampling.
822
+
823
+ Args:
824
+ on_device_tensors: Tuple containing on-device tensors with the
825
+ probabilities used in sampling and the sampled token ids. This
826
+ allows post-processing without copies to CPU/serialization, e.g. in
827
+ speculative decoding rejection sampling.
828
+ """
829
+
830
sampler_output = []
831
for (seq_group, sample_result, group_prompt_logprobs,
832
group_sample_logprobs) in zip(sampling_metadata.seq_groups,
@@ -701,4 +842,15 @@ def _build_sampler_output(
842
SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
843
sampler_output.append(
844
SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
704
- return SamplerOutput(outputs=sampler_output)
845
+
846
+ # If not specified, store None values in SamplerOutput.
847
+ if on_device_tensors is not None:
848
+ sampled_token_probs, sampled_token_ids = on_device_tensors
849
+ else:
850
+ sampled_token_probs, sampled_token_ids = (None, None)
851
+
852
+ return SamplerOutput(
853
+ outputs=sampler_output,
854
+ sampled_token_probs=sampled_token_probs,
855
+ sampled_token_ids=sampled_token_ids,
856
+ )
vllm/spec_decode/batch_expansion.py
+42
-28
@@ -6,8 +6,8 @@ import torch
6
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeScorer, SpeculativeScores)
9
-from vllm.spec_decode.util import (get_all_seq_ids, maybe_mock_device_tensors,
10
- nvtx_range, sampler_output_to_torch,
9
+from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10
+ sampler_output_to_torch,
11
split_batch_by_proposal_len)
12
from vllm.worker.worker_base import WorkerBase
13
@@ -72,10 +72,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
72
proposal_lens_list = proposals.proposal_lens.tolist()
73
proposal_token_ids_list = proposals.proposal_token_ids.tolist()
74
75
+ # Filter the list to ignore -1 proposals.
76
+ proposal_token_ids_list_without_skips = [
77
+ proposals for proposals in proposal_token_ids_list
78
+ if -1 not in proposals
79
+ ]
80
+
81
(spec_indices, non_spec_indices, target_seq_group_metadata_list,
82
num_scoring_tokens) = self._expand_batch(
83
seq_group_metadata_list=seq_group_metadata_list,
78
- proposal_token_ids_list=proposal_token_ids_list,
84
+ proposal_token_ids_list=proposal_token_ids_list_without_skips,
85
proposal_lens_list=proposal_lens_list,
86
)
87
@@ -89,7 +95,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
95
target_sampler_output = target_sampler_output[0]
96
97
all_tokens, all_probs = self._contract_batch(
92
- original_bs=len(seq_group_metadata_list),
98
+ contracted_bs=len(seq_group_metadata_list),
99
target_sampler_output=target_sampler_output,
100
proposals=proposals,
101
num_scoring_tokens=num_scoring_tokens,
@@ -128,14 +134,21 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
134
select_proposal_len_zero=True)
135
136
target_seq_group_metadata_list = self._create_scoring_model_input(
131
- spec_seqs, proposal_token_ids_list)
137
+ seq_group_metadata_list=spec_seqs,
138
+ proposal_token_ids=proposal_token_ids_list,
139
+ # NOTE: We determine the seq ids in the expanded batch using the
140
+ # full seq_group_metadata_list, instead of only spec_seqs.
141
+ target_seq_ids_iter=self._create_target_seq_id_iterator(
142
+ seq_ids=get_all_seq_ids(seq_group_metadata_list)),
143
+ )
144
+
145
num_scoring_tokens = len(target_seq_group_metadata_list)
146
target_seq_group_metadata_list.extend(non_spec_seqs)
147
148
return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
149
num_scoring_tokens)
150
138
- def _contract_batch(self, original_bs: int,
151
+ def _contract_batch(self, contracted_bs: int,
152
target_sampler_output: List[SamplerOutput],
153
proposals: SpeculativeProposals,
154
num_scoring_tokens: int, non_spec_indices: List[int],
@@ -144,42 +157,41 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
157
"""Contract the expanded batch back into its original size.
158
This maps the scores of speculative tokens back to their original
159
sequences.
147
- """
148
-
149
- # We mock the device tensors until PR 7/9 is merged (e2e correctness).
150
- # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
151
- maybe_mock_device_tensors(
152
- sampler_output=target_sampler_output,
153
- batch_size=len(non_spec_indices) + num_scoring_tokens,
154
- vocab_size=self._vocab_size,
155
- device=self._device,
156
- )
160
161
+ contracted_bs is the original batch size, and the batch size that the
162
+ target_sampler_output will be contracted to.
163
+ """
164
(target_token_ids, target_probs, non_spec_target_token_ids,
165
non_spec_target_probs) = self._split_scoring_output(
166
target_sampler_output, num_scoring_tokens)
167
168
# Map distinct sequences used to score each token
169
# of shape [batch_size * k + 1] back to [batch_size, k + 1].
164
- batch_size, k = proposals.proposal_token_ids.shape
170
+ expanded_batch_size, k = proposals.proposal_token_ids.shape
171
+
172
+ # The number of tokens in the expanded batch used for speculation is
173
+ # equal to the total expanded batch size minus the number of samples for
174
+ # non-speculative sequences.
175
+ non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
176
+ spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
177
178
target_token_ids = target_token_ids.squeeze().reshape(
167
- batch_size, k + 1)
168
- target_probs = target_probs.squeeze().reshape(batch_size, k + 1,
179
+ spec_expanded_bs, k + 1)
180
+ target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
181
self._vocab_size)
182
171
- all_tokens = torch.full(size=(original_bs, k + 1),
183
+ all_tokens = torch.full(size=(contracted_bs, k + 1),
184
fill_value=-1,
185
device=self._device,
186
dtype=torch.long)
175
- all_probs = torch.zeros(original_bs,
187
+ all_probs = torch.zeros(contracted_bs,
188
k + 1,
189
self._vocab_size,
190
device=self._device,
191
dtype=torch.float32)
192
193
if non_spec_indices:
182
- all_tokens[non_spec_indices, 0] = non_spec_target_token_ids
194
+ all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
195
all_probs[non_spec_indices, :1, :] = non_spec_target_probs
196
197
if spec_indices:
@@ -189,20 +201,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
201
return all_tokens, all_probs
202
203
def _create_scoring_model_input(
192
- self,
193
- seq_group_metadata_list: List[SequenceGroupMetadata],
194
- proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
204
+ self,
205
+ seq_group_metadata_list: List[SequenceGroupMetadata],
206
+ proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
207
+ target_seq_ids_iter: Iterator[TargetSeqId],
208
) -> List[SequenceGroupMetadata]:
209
"""Given the original input sequences and proposed tokens from the draft
210
model, create a list of target sequences that can be used for scoring.
211
+
212
+ target_seq_ids_iter provides sequence ids for the expanded batch,
213
+ fulfilling the requirement that no seq id in the expanded batch is equal
214
+ to the seq id in the original batch.
215
"""
216
217
if not seq_group_metadata_list:
218
return []
219
203
- target_seq_ids_iter = self._create_target_seq_id_iterator(
204
- get_all_seq_ids(seq_group_metadata_list))
205
-
220
target_seq_group_metadata = list(
221
chain.from_iterable(
222
self._create_target_seq_group_metadata(
vllm/spec_decode/interfaces.py
+2
-2
@@ -24,9 +24,9 @@ class SpeculativeProposals:
24
25
def __repr__(self):
26
return (f"SpeculativeProposals("
27
- f"proposal_token_ids={self.proposal_token_ids.shape}, "
27
+ f"proposal_token_ids={self.proposal_token_ids}, "
28
f"proposal_probs={self.proposal_probs.shape}, "
29
- f"proposal_lens={self.proposal_lens.shape})")
29
+ f"proposal_lens={self.proposal_lens})")
30
31
32
@dataclass
vllm/spec_decode/metrics.py
+23
-8
@@ -147,15 +147,16 @@ class AsyncMetricsCollector:
147
emitted_tokens = self._aggregate_num_emitted_tokens.item()
148
draft_tokens = self._aggregate_num_draft_tokens
149
150
- num_possible_tokens = self.get_max_num_accepted_tokens(draft_tokens, k)
150
+ max_num_emitted_tokens = self.get_max_num_emitted_tokens(
151
+ draft_tokens, k)
152
153
if draft_tokens > 0:
154
draft_acceptance_rate = accepted_tokens / draft_tokens
155
else:
156
draft_acceptance_rate = float("nan")
157
157
- if num_possible_tokens > 0:
158
- system_efficiency = emitted_tokens / num_possible_tokens
158
+ if max_num_emitted_tokens > 0:
159
+ system_efficiency = emitted_tokens / max_num_emitted_tokens
160
else:
161
system_efficiency = float("nan")
162
@@ -169,8 +170,22 @@ class AsyncMetricsCollector:
170
)
171
172
@staticmethod
172
- def get_max_num_accepted_tokens(draft_tokens: int, k: int) -> int:
173
- # Divide by k since batch size can be variable.
174
- total_num_spec_seqs = draft_tokens / k
175
- num_accepted_per_seq_if_all_accepted = k + 1
176
- return int(total_num_spec_seqs / num_accepted_per_seq_if_all_accepted)
173
+ def get_max_num_emitted_tokens(draft_tokens: int, k: int) -> int:
174
+ """Calculate the number of emitted tokens, assuming all tokens are
175
+ accepted.
176
+
177
+ This is equal to the number of sequences that have been speculated on,
178
+ times (speculation len + 1). The +1 comes from the bonus token.
179
+ """
180
+ # Determine the number of sequences that have been speculated on. Since
181
+ # the batch size can be variable, we divide by k.
182
+ assert draft_tokens % k == 0
183
+ total_num_spec_seqs = draft_tokens // k
184
+
185
+ # A single sequence may emit k accepted tokens and one bonus token in
186
+ # the best case.
187
+ num_emitted_per_seq_if_all_accepted = k + 1
188
+
189
+ # The max num of emitted tokens is the number of speculated sequences
190
+ # times the max emitted per seq.
191
+ return total_num_spec_seqs * num_emitted_per_seq_if_all_accepted
vllm/spec_decode/multi_step_worker.py
+10
-19
@@ -6,8 +6,7 @@ import torch
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
-from vllm.spec_decode.util import (maybe_mock_device_tensors,
10
- sampler_output_to_torch)
9
+from vllm.spec_decode.util import sampler_output_to_torch
10
from vllm.worker.worker import Worker
11
12
@@ -329,12 +328,15 @@ class DraftModelTop1Proposer(SpeculativeProposer):
328
"""
329
if maybe_sampler_output is None:
330
# If no speculative tokens, the sampler output will be None.
332
- # In this case we return empty tensors.
333
- proposal_tokens = torch.zeros(0,
334
- max_proposal_len,
335
- dtype=torch.long,
336
- device=self._device)
337
- proposal_probs = torch.zeros(0,
331
+ # In this case we return empty proposals.
332
+ proposal_tokens = torch.full(size=(
333
+ batch_size,
334
+ max_proposal_len,
335
+ ),
336
+ fill_value=-1,
337
+ dtype=torch.long,
338
+ device=self._device)
339
+ proposal_probs = torch.zeros(batch_size,
340
max_proposal_len,
341
self._vocab_size,
342
dtype=torch.float32,
@@ -345,17 +347,6 @@ class DraftModelTop1Proposer(SpeculativeProposer):
347
return proposal_tokens, proposal_probs, proposal_lens_tensor
348
349
sampler_output = maybe_sampler_output
348
-
349
- # We mock the device tensors until PR 7/9 is merged (e2e correctness).
350
- # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
351
- for step_output in sampler_output:
352
- maybe_mock_device_tensors(
353
- sampler_output=step_output,
354
- batch_size=len(proposal_lens),
355
- vocab_size=self._vocab_size,
356
- device=self._device,
357
- )
358
-
350
proposal_tokens, proposal_probs = sampler_output_to_torch(
351
sampler_output)
352
vllm/spec_decode/spec_decode_worker.py
+43
-6
@@ -111,6 +111,32 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
111
device=self.device,
112
vocab_size=self._vocab_size)
113
114
+ self._configure_model_sampler_for_spec_decode()
115
+
116
+ def _configure_model_sampler_for_spec_decode(self):
117
+ """Configure model sampler to emit GPU tensors. This allows spec decode
118
+ to keep data on device without transferring to CPU and serializing,
119
+ which significantly reduces overhead of rejection sampling.
120
+
121
+ NOTE(cade): This breaks abstraction boundaries pretty badly. The better
122
+ design is to have the "move to CPU and serialize" sampling decision be
123
+ done outside of the model/sampler; this way the "last-mile" worker
124
+ object which interfaces with the scheduler can serialize and incur the
125
+ performance hit as necessary. This allows us to run the worker several
126
+ iterations in a row without incurring the "move to CPU and serialize"
127
+ performance penalty.
128
+
129
+ Since this requires a large change to vLLM, we defer it to later and
130
+ temporarily accept this broken abstraction boundary.
131
+
132
+ NOTE(cade): This will require a special check if the proposer worker
133
+ does not have a sampler (e.g. ngram speculation).
134
+ """
135
+ (self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
136
+ ) = True
137
+ (self.proposer_worker.model_runner.model.sampler.
138
+ include_gpu_probs_tensor) = True
139
+
140
def determine_num_available_blocks(self) -> Tuple[int, int]:
141
"""Determine the number of cache blocks to use.
142
@@ -286,15 +312,26 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
312
select_proposal_len_zero=True)
313
original_indices = spec_indices + non_spec_indices
314
289
- proposal_probs = proposal_scores.probs[spec_indices, :-1]
290
- bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
315
+ # Get probabilities of target model, excluding bonus token.
316
+ proposal_verifier_probs = proposal_scores.probs[spec_indices, :-1]
317
+
318
+ # Get non-speculative sampled tokens from target model.
319
non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
320
321
+ # Get bonus tokens from target model.
322
+ bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
323
+
324
+ # Get probabilities according to proposal method.
325
+ proposal_probs = proposals.proposal_probs[spec_indices]
326
+
327
+ # Get proposed tokens.
328
+ proposal_token_ids = proposals.proposal_token_ids[spec_indices]
329
+
330
accepted_token_ids = self.rejection_sampler(
294
- proposal_probs,
295
- bonus_token_ids,
296
- proposals.proposal_probs,
297
- proposals.proposal_token_ids,
331
+ target_probs=proposal_verifier_probs,
332
+ bonus_token_ids=bonus_token_ids,
333
+ draft_probs=proposal_probs,
334
+ draft_token_ids=proposal_token_ids,
335
)
336
337
# Append output tokens from non-speculative sequences to