@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 62b8aebc6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Speculative decoding 7/9] Speculative decoding end-to-end correctness tests. (#3951)

Cade Daniel committed Apr 23, 2024 at 01:02 UTC 62b8aebc6f06b5c8fafa1f27893cd4f9bb11fa8b
22 files changed +1164 -175
tests/samplers/test_rejection_sampler.py
+6 -2
@@ -91,12 +91,16 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
91 bonus_token_ids,
92 )
93
94 + # Bonus tokens are currently disabled. Verify they're set to -1.
95 + # See https://github.com/vllm-project/vllm/issues/4212
96 + expected_bonus_token_ids = bonus_token_ids.clone() * 0 - 1
97 +
98 if which_tokens_accepted == "all_tokens_accepted":
99 # Expect all tokens to be equal to draft tokens.
100 assert torch.equal(output_token_ids[:, :-1], draft_token_ids)
101
102 # Expect all bonus tokens to be included.
99 - assert torch.equal(output_token_ids[:, -1:], bonus_token_ids)
103 + assert torch.equal(output_token_ids[:, -1:], expected_bonus_token_ids)
104 elif which_tokens_accepted == "no_tokens_accepted":
105 # Expect first token to be equal to recovered tokens.
106 assert torch.equal(output_token_ids[:, 0], recovered_token_ids[:, 0])
@@ -106,7 +110,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
110 torch.ones_like(output_token_ids[:, 1:]) * -1)
111 elif which_tokens_accepted == "some_tokens_accepted":
112 recovered_plus_bonus = torch.cat(
109 - (recovered_token_ids, bonus_token_ids), dim=-1)
113 + (recovered_token_ids, expected_bonus_token_ids), dim=-1)
114 # Assert first rejected token is a recovered token or bonus token.
115 assert torch.equal(
116 recovered_plus_bonus[torch.arange(0, batch_size),
tests/samplers/test_sampler.py
+2 -1
@@ -636,7 +636,8 @@ def test_sampler_top_k_top_p(seed: int, device: str):
636 def mock_sample(probs, *args, **kwargs):
637 nonlocal sample_probs
638 sample_probs = probs
639 - return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
639 + return ([[prob.topk(1, dim=-1).indices.tolist(), [0]]
640 + for prob in probs], None)
641
642 with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
643 sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
tests/spec_decode/e2e/__init__.py
tests/spec_decode/e2e/conftest.py
+35 -10
@@ -1,3 +1,5 @@
1 +from typing import List, Tuple
2 +
3 import pytest
4
5 from tests.conftest import cleanup
@@ -6,28 +8,34 @@ from vllm.model_executor.utils import set_random_seed
8
9
10 @pytest.fixture
9 -def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
10 - baseline_llm_kwargs, seed):
11 - return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
11 +def baseline_llm_generator(request, common_llm_kwargs,
12 + per_test_common_llm_kwargs, baseline_llm_kwargs,
13 + seed):
14 + return create_llm_generator("baseline", request, common_llm_kwargs,
15 + per_test_common_llm_kwargs,
16 baseline_llm_kwargs, seed)
17
18
19 @pytest.fixture
16 -def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
20 +def test_llm_generator(request, common_llm_kwargs, per_test_common_llm_kwargs,
21 test_llm_kwargs, seed):
18 - return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
19 - test_llm_kwargs, seed)
22 + return create_llm_generator("test", request, common_llm_kwargs,
23 + per_test_common_llm_kwargs, test_llm_kwargs,
24 + seed)
25
26
22 -def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
23 - distinct_llm_kwargs, seed):
27 +def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
28 + per_test_common_llm_kwargs, distinct_llm_kwargs,
29 + seed):
30 kwargs = {
31 **common_llm_kwargs,
32 **per_test_common_llm_kwargs,
33 **distinct_llm_kwargs,
34 }
35 + test_name = request.node.name
36
37 def generator_inner():
38 + print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
39 llm = LLM(**kwargs)
40
41 set_random_seed(seed)
@@ -36,6 +44,23 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
44 del llm
45 cleanup()
46
39 - for llm in generator_inner():
40 - yield llm
47 + def generator_outer():
48 + for llm in generator_inner():
49 + yield llm
50 + del llm
51 +
52 + return generator_outer
53 +
54 +
55 +def get_output_from_llm_generator(
56 + llm_generator, prompts,
57 + sampling_params) -> Tuple[List[str], List[List[int]]]:
58 + tokens = []
59 + token_ids = []
60 + for llm in llm_generator():
61 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
62 + token_ids = [output.outputs[0].token_ids for output in outputs]
63 + tokens = [output.outputs[0].text for output in outputs]
64 del llm
65 +
66 + return tokens, token_ids
tests/spec_decode/e2e/test_compatibility.py new
+169
@@ -0,0 +1,169 @@
1 +import pytest
2 +
3 +from vllm import SamplingParams
4 +
5 +from .conftest import get_output_from_llm_generator
6 +
7 +
8 +@pytest.mark.parametrize(
9 + "common_llm_kwargs",
10 + [{
11 + "model": "JackFram/llama-68m",
12 + "speculative_model": "JackFram/llama-68m",
13 + "num_speculative_tokens": 5,
14 +
15 + # Required for spec decode.
16 + "use_v2_block_manager": True
17 + }])
18 +@pytest.mark.parametrize(
19 + "per_test_common_llm_kwargs",
20 + [
21 + {
22 + # Expect failure as spec decode not supported by
23 + # Ray backend.
24 + "worker_use_ray": True,
25 + },
26 + ])
27 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
28 +@pytest.mark.parametrize("seed", [1])
29 +def test_spec_decode_xfail_ray(test_llm_generator):
30 + """Verify that speculative decoding with Ray fails.
31 + """
32 + output_len = 128
33 + temperature = 0.0
34 +
35 + prompts = [
36 + "Hello, my name is",
37 + ]
38 +
39 + sampling_params = SamplingParams(
40 + max_tokens=output_len,
41 + ignore_eos=True,
42 + temperature=temperature,
43 + )
44 +
45 + with pytest.raises(AssertionError,
46 + match="Speculative decoding not yet supported for "):
47 + get_output_from_llm_generator(test_llm_generator, prompts,
48 + sampling_params)
49 +
50 +
51 +@pytest.mark.parametrize(
52 + "common_llm_kwargs",
53 + [{
54 + "model": "JackFram/llama-68m",
55 + "speculative_model": "JackFram/llama-68m",
56 + "num_speculative_tokens": 5,
57 +
58 + # Required for spec decode.
59 + "use_v2_block_manager": True
60 + }])
61 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
62 + {
63 + "enable_chunked_prefill": True,
64 + },
65 +])
66 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
67 +@pytest.mark.parametrize("seed", [1])
68 +def test_spec_decode_xfail_chunked_prefill(test_llm_generator):
69 + """Verify that speculative decoding with chunked prefill fails.
70 + """
71 + output_len = 128
72 + temperature = 0.0
73 +
74 + prompts = [
75 + "Hello, my name is",
76 + ]
77 +
78 + sampling_params = SamplingParams(
79 + max_tokens=output_len,
80 + ignore_eos=True,
81 + temperature=temperature,
82 + )
83 +
84 + with pytest.raises(ValueError,
85 + match="Speculative decoding and chunked prefill"):
86 + get_output_from_llm_generator(test_llm_generator, prompts,
87 + sampling_params)
88 +
89 +
90 +@pytest.mark.parametrize(
91 + "common_llm_kwargs",
92 + [{
93 + "model": "meta-llama/Llama-2-7b-chat-hf",
94 + "speculative_model": "JackFram/llama-68m",
95 + "num_speculative_tokens": 5,
96 +
97 + # Required for spec decode.
98 + "use_v2_block_manager": True
99 + }])
100 +@pytest.mark.parametrize(
101 + "per_test_common_llm_kwargs",
102 + [
103 + {
104 + # Speculative max model len > overridden max model len should raise.
105 + "max_model_len": 128,
106 + "speculative_max_model_len": 129,
107 + },
108 + {
109 + # Speculative max model len > draft max model len should raise.
110 + # https://huggingface.co/JackFram/llama-68m/blob/3b606af5198a0b26762d589a3ee3d26ee6fa6c85/config.json#L12
111 + "speculative_max_model_len": 2048 + 1,
112 + },
113 + {
114 + # Speculative max model len > target max model len should raise.
115 + # https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/f5db02db724555f92da89c216ac04704f23d4590/config.json#L12
116 + "speculative_max_model_len": 4096 + 1,
117 + },
118 + ])
119 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
120 +@pytest.mark.parametrize("seed", [1])
121 +def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
122 + """Verify that speculative decoding validates speculative_max_model_len.
123 + """
124 + output_len = 128
125 + temperature = 0.0
126 +
127 + prompts = [
128 + "Hello, my name is",
129 + ]
130 +
131 + sampling_params = SamplingParams(
132 + max_tokens=output_len,
133 + ignore_eos=True,
134 + temperature=temperature,
135 + )
136 +
137 + with pytest.raises(ValueError, match="cannot be larger than"):
138 + get_output_from_llm_generator(test_llm_generator, prompts,
139 + sampling_params)
140 +
141 +
142 +@pytest.mark.parametrize("common_llm_kwargs", [{
143 + "model": "JackFram/llama-68m",
144 + "speculative_model": "JackFram/llama-68m",
145 + "num_speculative_tokens": 5,
146 +}])
147 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
148 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
149 +@pytest.mark.parametrize("seed", [1])
150 +def test_spec_decode_xfail_block_manager_v1(test_llm_generator):
151 + """Verify that speculative decoding with block manager v1 fails.
152 + """
153 + output_len = 128
154 + temperature = 0.0
155 +
156 + prompts = [
157 + "Hello, my name is",
158 + ]
159 +
160 + sampling_params = SamplingParams(
161 + max_tokens=output_len,
162 + ignore_eos=True,
163 + temperature=temperature,
164 + )
165 +
166 + with pytest.raises(ValueError,
167 + match="Speculative decoding requires usage of the V2"):
168 + get_output_from_llm_generator(test_llm_generator, prompts,
169 + sampling_params)
tests/spec_decode/e2e/test_correctness.py
+493 -47
@@ -1,11 +1,42 @@
1 +"""The tests in this file verify end-to-end speculative decoding correctness.
2 +
3 +This docstring details important information on the testing methodology.
4 +
5 +Most of the tests rely on "greedy equality", where we expect the output of
6 +speculative decoding on a sequence to exactly match the output of normal non-
7 +speculative decoding.
8 +
9 +Since speculative decoding with rejection sampling guarantees that the output
10 +distribution matches the target model's output distribution (up to hardware
11 +numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
12 +equality. This gives us good coverage of temp=0.
13 +
14 +For temp>0, we rely on unit tests on the rejection sampler to verify that the
15 +output distribution is the same with spec decode vs. no spec decode (this would
16 +be prohibitively expensive to run with a real model).
17 +
18 +NOTE: Speculative decoding's distribution equality requires that the measured
19 +distributions of the target model and proposal model be deterministic given the
20 +same input. vLLM largely guarantees this.
21 +
22 +@cadedaniel has seen cases where the output probabilities of a draft/target
23 +model change slightly with certain batch sizes or prompts, even with Torch
24 +determinism flags set. It is unclear if this is a bug in vLLM, due to non-
25 +determinism in on-device batched operations, a bug in vLLM's spec decode
26 +implementation, or the "hardware numerics" limitations. Either way, rejection
27 +sampling ensures the output distribution matches the target model, but it breaks
28 +greedy-equality tests for those batch sizes/prompts.
29 +"""
30 +
31 from itertools import cycle
2 -from typing import List, Tuple
32
33 import pytest
34 from transformers import AutoTokenizer
35
36 from vllm import SamplingParams
37
38 +from .conftest import get_output_from_llm_generator
39 +
40
41 @pytest.mark.parametrize(
42 "common_llm_kwargs",
@@ -14,9 +45,6 @@ from vllm import SamplingParams
45 # Note this is repeated in the test body; to initialize a tokenizer.
46 "model": "JackFram/llama-68m",
47
17 - # Skip real loading for fast test.
18 - "load_format": "dummy",
19 -
48 # Skip cuda graph recording for fast test.
49 "enforce_eager": True,
50
@@ -31,22 +59,15 @@ from vllm import SamplingParams
59 "num_speculative_tokens": 5,
60 },
61 {
34 - "speculative_model": "JackFram/llama-68m",
35 - "num_speculative_tokens": 1,
36 - },
37 - {
38 - # No spec decode.
62 + # Verify the detokenizer assertions in the test work when spec
63 + # decode is disabled.
64 },
65 ])
66 @pytest.mark.parametrize("test_llm_kwargs", [{}])
42 -@pytest.mark.parametrize("batch_size", [1])
43 -# NOTE: We should run more permutations of this test (more BS, more seeds). But
44 -# because our spec decode generates gibberish token ids, the likelihood of
45 -# emitting an invalid token combination is nontrivial. This causes divergence in
46 -# behavior of vLLM detokenization vs. hf tokenizer, for example when two "utf-
47 -# start" bytes are emitted.
67 +@pytest.mark.parametrize("batch_size", [1, 32])
68 @pytest.mark.parametrize("seed", [1])
49 -def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
69 +def test_spec_decode_e2e_with_detokenization(test_llm_generator,
70 + batch_size: int):
71 """Run generation with speculative decoding on a batch. Verify the engine
72 generates the correct number of tokens (via ignore_eos=True), and that the
73 detokenization matches HF transformers.
@@ -67,8 +88,6 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
88 max_tokens=output_len,
89 ignore_eos=True,
90 temperature=temperature,
70 - skip_special_tokens=True,
71 - spaces_between_special_tokens=False,
91 )
92
93 batch_tokens, batch_token_ids = get_output_from_llm_generator(
@@ -77,9 +96,10 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
96 # Expect a generation for each prompt in the batch.
97 assert len(batch_token_ids) == len(prompts)
98
80 - # Expect each generation to have expected number of tokens (note
81 - # ignore_eos=True).
82 - assert all(len(token_ids) == output_len for token_ids in batch_token_ids)
99 + # Expect each generation to have expected number of tokens (note ignore_eos
100 + # is True).
101 + assert [len(token_ids)
102 + for token_ids in batch_token_ids] == ([output_len] * batch_size)
103
104 # Expect detokenized string to match.
105 tok = AutoTokenizer.from_pretrained("JackFram/llama-68m")
@@ -92,14 +112,111 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
112 @pytest.mark.parametrize(
113 "common_llm_kwargs",
114 [{
95 - # Use a small model for a fast test.
96 - "model": "JackFram/llama-68m",
115 + # Skip cuda graph recording for fast test.
116 + "enforce_eager": True,
117 +
118 + # Required for spec decode.
119 + "use_v2_block_manager": True,
120 +
121 + # Print spec metrics.
122 + "disable_log_stats": False,
123 + }])
124 +@pytest.mark.parametrize(
125 + "per_test_common_llm_kwargs",
126 + [
127 + # Try two different tiny base models.
128 + # Note that one is equal to the draft model, another isn't.
129 + {
130 + "model": "JackFram/llama-68m",
131 + },
132 + {
133 + "model": "JackFram/llama-160m",
134 + },
135 + ])
136 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
137 +@pytest.mark.parametrize("test_llm_kwargs", [
138 + {
139 "speculative_model": "JackFram/llama-68m",
140 "num_speculative_tokens": 5,
141 + },
142 +])
143 +@pytest.mark.parametrize(
144 + "output_len",
145 + [
146 + # Use long output len for the small model test.
147 + 1536,
148 + ])
149 +@pytest.mark.parametrize("batch_size", [1])
150 +@pytest.mark.parametrize("seed", [1])
151 +def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
152 + baseline_llm_generator, test_llm_generator, batch_size: int,
153 + output_len: int):
154 + """Verify greedy equality on a tiny model with batch size of one.
155 +
156 + Since this test is cheaper than other e2e correctness tests, we generate
157 + with a higher output_len.
158 + """
159 + run_greedy_equality_correctness_test(baseline_llm_generator,
160 + test_llm_generator,
161 + batch_size,
162 + max_output_len=output_len,
163 + force_output_len=True)
164
100 - # Skip real loading for fast test.
101 - "load_format": "dummy",
165
166 +@pytest.mark.parametrize(
167 + "common_llm_kwargs",
168 + [{
169 + # Skip cuda graph recording for fast test.
170 + "enforce_eager": True,
171 +
172 + # Required for spec decode.
173 + "use_v2_block_manager": True,
174 +
175 + # Print spec metrics.
176 + "disable_log_stats": False,
177 + }])
178 +@pytest.mark.parametrize(
179 + "per_test_common_llm_kwargs",
180 + [
181 + # Try two different tiny base models.
182 + # Note that one is equal to the draft model, another isn't.
183 + {
184 + "model": "JackFram/llama-68m",
185 + },
186 + {
187 + "model": "JackFram/llama-160m",
188 + },
189 + ])
190 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
191 +@pytest.mark.parametrize("test_llm_kwargs", [
192 + {
193 + "speculative_model": "JackFram/llama-68m",
194 + "num_speculative_tokens": 5,
195 + },
196 +])
197 +@pytest.mark.parametrize(
198 + "output_len",
199 + [
200 + # Use small output len for fast test.
201 + 256,
202 + ])
203 +@pytest.mark.parametrize("batch_size", [64])
204 +@pytest.mark.parametrize("seed", [1])
205 +def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
206 + baseline_llm_generator, test_llm_generator, batch_size: int,
207 + output_len: int):
208 + """Verify greedy equality on a tiny model and large batch size.
209 + """
210 + run_greedy_equality_correctness_test(baseline_llm_generator,
211 + test_llm_generator,
212 + batch_size,
213 + max_output_len=output_len,
214 + force_output_len=True)
215 +
216 +
217 +@pytest.mark.parametrize(
218 + "common_llm_kwargs",
219 + [{
220 # Skip cuda graph recording for fast test.
221 "enforce_eager": True,
222
@@ -109,43 +226,372 @@ def test_spec_decode_e2e_logical_flow(test_llm_generator, batch_size: int):
226 @pytest.mark.parametrize(
227 "per_test_common_llm_kwargs",
228 [
229 + # Try two different tiny base models.
230 + # Note that one is equal to the draft model, another isn't.
231 {
113 - # Expect failure as spec decode not supported by
114 - # Ray backend.
115 - "worker_use_ray": True,
232 + "model": "JackFram/llama-68m",
233 + },
234 + {
235 + "model": "JackFram/llama-160m",
236 },
237 ])
118 -@pytest.mark.parametrize("test_llm_kwargs", [{}])
238 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
239 +@pytest.mark.parametrize("test_llm_kwargs", [
240 + {
241 + "speculative_model": "JackFram/llama-68m",
242 + "num_speculative_tokens": 5,
243 + },
244 +])
245 +@pytest.mark.parametrize("max_output_len", [
246 + 256,
247 +])
248 +@pytest.mark.parametrize("batch_size", [32])
249 +@pytest.mark.parametrize("seed", [1])
250 +def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(
251 + baseline_llm_generator, test_llm_generator, batch_size: int,
252 + max_output_len: int):
253 + """Verify greedy equality on a tiny model, with a large batch size, and when
254 + sampling respects the EOS token.
255 + """
256 + run_greedy_equality_correctness_test(baseline_llm_generator,
257 + test_llm_generator,
258 + batch_size,
259 + max_output_len,
260 + force_output_len=False)
261 +
262 +
263 +@pytest.mark.parametrize(
264 + "common_llm_kwargs",
265 + [{
266 + # A "real" model (not tiny).
267 + "model": "meta-llama/Llama-2-7b-chat-hf",
268 +
269 + # Skip cuda graph recording for fast test.
270 + "enforce_eager": True,
271 +
272 + # Required for spec decode.
273 + "use_v2_block_manager": True,
274 +
275 + # Print spec metrics.
276 + "disable_log_stats": False,
277 + }])
278 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
279 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
280 +@pytest.mark.parametrize("test_llm_kwargs", [
281 + {
282 + "speculative_model": "JackFram/llama-68m",
283 + "num_speculative_tokens": 5,
284 + },
285 +])
286 +@pytest.mark.parametrize("batch_size", [1])
287 +@pytest.mark.parametrize(
288 + "output_len",
289 + [
290 + # Use decently long output len for a high quality test.
291 + 256,
292 + ])
293 +@pytest.mark.parametrize("seed", [1])
294 +def test_spec_decode_e2e_greedy_correctness_real_model_bs1(
295 + baseline_llm_generator, test_llm_generator, batch_size: int,
296 + output_len: int):
297 + """Verify greedy equality on a "real" model and batch size of 1. This is
298 + separate from large BS tests to make identifying the source of bugs easier.
299 + """
300 + run_greedy_equality_correctness_test(baseline_llm_generator,
301 + test_llm_generator,
302 + batch_size,
303 + max_output_len=output_len,
304 + force_output_len=True)
305 +
306 +
307 +@pytest.mark.parametrize(
308 + "common_llm_kwargs",
309 + [{
310 + # A "real" model (not tiny).
311 + "model": "meta-llama/Llama-2-7b-chat-hf",
312 +
313 + # Skip cuda graph recording for fast test.
314 + "enforce_eager": True,
315 +
316 + # Required for spec decode.
317 + "use_v2_block_manager": True,
318 +
319 + # Print spec metrics.
320 + "disable_log_stats": False,
321 + }])
322 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
323 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
324 +@pytest.mark.parametrize("test_llm_kwargs", [
325 + {
326 + "speculative_model": "JackFram/llama-68m",
327 + "num_speculative_tokens": 5,
328 + },
329 +])
330 +@pytest.mark.parametrize("batch_size", [32])
331 +@pytest.mark.parametrize(
332 + "output_len",
333 + [
334 + # Use smaller output len for fast test.
335 + 64,
336 + ])
337 @pytest.mark.parametrize("seed", [1])
120 -def test_spec_decode_xfail(test_llm_generator):
121 - """Verify that speculative decoding with Ray fails.
338 +def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
339 + baseline_llm_generator, test_llm_generator, batch_size: int,
340 + output_len: int):
341 + """Verify greedy equality with a "real" model on a nontrivial batch size.
342 + This is the closest test to a real production workload.
343 + """
344 + run_greedy_equality_correctness_test(baseline_llm_generator,
345 + test_llm_generator,
346 + batch_size,
347 + max_output_len=output_len,
348 + force_output_len=True)
349 +
350 +
351 +@pytest.mark.parametrize(
352 + "common_llm_kwargs",
353 + [{
354 + "block_size": 8,
355 + # 2 for small prompt, 256//8 for generated.
356 + "num_gpu_blocks_override": 2 + 256 // 8,
357 + "max_model_len": (2 + 256 // 8) * 8,
358 +
359 + # Skip cuda graph recording for fast test.
360 + "enforce_eager": True,
361 +
362 + # Required for spec decode.
363 + "use_v2_block_manager": True
364 + }])
365 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
366 + {
367 + "model": "JackFram/llama-160m",
368 + },
369 +])
370 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
371 +@pytest.mark.parametrize("test_llm_kwargs", [
372 + {
373 + "speculative_model": "JackFram/llama-68m",
374 + "num_speculative_tokens": 5,
375 + },
376 +])
377 +@pytest.mark.parametrize(
378 + "output_len",
379 + [
380 + # Use small output len for fast test.
381 + 256,
382 + ])
383 +@pytest.mark.parametrize("batch_size", [4])
384 +@pytest.mark.parametrize("seed", [1])
385 +def test_spec_decode_e2e_greedy_correctness_with_preemption(
386 + baseline_llm_generator, test_llm_generator, batch_size: int,
387 + output_len: int):
388 + """Verify greedy equality, even when some sequences are preempted mid-
389 + generation.
390 + """
391 + run_greedy_equality_correctness_test(baseline_llm_generator,
392 + test_llm_generator,
393 + batch_size,
394 + max_output_len=output_len,
395 + force_output_len=True)
396 +
397 +
398 +@pytest.mark.parametrize(
399 + "common_llm_kwargs",
400 + [{
401 + "model": "JackFram/llama-160m",
402 +
403 + # Skip cuda graph recording for fast test.
404 + "enforce_eager": True,
405 +
406 + # Required for spec decode.
407 + "use_v2_block_manager": True
408 + }])
409 +@pytest.mark.parametrize(
410 + "per_test_common_llm_kwargs",
411 + [
412 + # As of this writing, vLLM only compiles with these 3 block sizes by
413 + # default.
414 + {
415 + "block_size": 8,
416 + },
417 + {
418 + "block_size": 16,
419 + },
420 + {
421 + "block_size": 32,
422 + },
423 + ])
424 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
425 +@pytest.mark.parametrize("test_llm_kwargs", [
426 + {
427 + "speculative_model": "JackFram/llama-68m",
428 + "num_speculative_tokens": 5,
429 + },
430 +])
431 +@pytest.mark.parametrize("batch_size", [2])
432 +@pytest.mark.parametrize(
433 + "output_len",
434 + [
435 + # Use smaller output len for fast test.
436 + 32,
437 + ])
438 +@pytest.mark.parametrize("seed", [1])
439 +def test_spec_decode_different_block_size(baseline_llm_generator,
440 + test_llm_generator, batch_size: int,
441 + output_len: int):
442 + """Verify greedy equality over different block sizes.
443 + """
444 + run_greedy_equality_correctness_test(baseline_llm_generator,
445 + test_llm_generator,
446 + batch_size,
447 + max_output_len=output_len,
448 + force_output_len=True)
449 +
450 +
451 +@pytest.mark.parametrize(
452 + "common_llm_kwargs",
453 + [{
454 + "model": "JackFram/llama-160m",
455 +
456 + # Skip cuda graph recording for fast test.
457 + "enforce_eager": True,
458 +
459 + # Required for spec decode.
460 + "use_v2_block_manager": True
461 + }])
462 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
463 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
464 +@pytest.mark.parametrize(
465 + "test_llm_kwargs",
466 + [
467 + {
468 + "speculative_model": "JackFram/llama-68m",
469 + "num_speculative_tokens": 5,
470 +
471 + # Artificially limit the draft model max model len; this forces vLLM
472 + # to skip speculation once the sequences grow beyond 32-k tokens.
473 + "speculative_max_model_len": 32,
474 + },
475 + ])
476 +@pytest.mark.parametrize("batch_size", [8])
477 +@pytest.mark.parametrize(
478 + "output_len",
479 + [
480 + # This must be a good bit larger than speculative_max_model_len so that
481 + # we can test the case where all seqs are skipped, but still small to
482 + # ensure fast test.
483 + 64,
484 + ])
485 +@pytest.mark.parametrize("seed", [1])
486 +def test_skip_speculation(baseline_llm_generator, test_llm_generator,
487 + batch_size: int, output_len: int):
488 + """Verify greedy equality when some (or all) sequences skip speculation.
489 + We do this by setting the max model len of the draft model to an
490 + artificially low value, such that when the sequences grow beyond it, they
491 + are skipped in speculative decoding.
492 + """
493 + run_greedy_equality_correctness_test(baseline_llm_generator,
494 + test_llm_generator,
495 + batch_size,
496 + max_output_len=output_len,
497 + force_output_len=True)
498 +
499 +
500 +@pytest.mark.parametrize(
501 + "common_llm_kwargs",
502 + [{
503 + "model": "JackFram/llama-68m",
504 +
505 + # Skip cuda graph recording for fast test.
506 + "enforce_eager": True,
507 +
508 + # Required for spec decode.
509 + "use_v2_block_manager": True
510 + }])
511 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
512 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
513 +@pytest.mark.parametrize(
514 + "test_llm_kwargs",
515 + [
516 + {
517 + "speculative_model": "JackFram/llama-68m",
518 + "num_speculative_tokens": k,
519 + }
520 + # Try a range of common k, as well as large speculation.
521 + for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 63]
522 + ])
523 +@pytest.mark.parametrize("batch_size", [2])
524 +@pytest.mark.parametrize(
525 + "output_len",
526 + [
527 + # Use smaller output len for fast test.
528 + 32,
529 + ])
530 +@pytest.mark.parametrize("seed", [1])
531 +def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
532 + output_len: int):
533 + """Verify that speculative decoding produces exact equality to without spec
534 + decode with many different values of k.
535 + """
536 + run_greedy_equality_correctness_test(baseline_llm_generator,
537 + test_llm_generator,
538 + batch_size,
539 + max_output_len=output_len,
540 + force_output_len=True)
541 +
542 +
543 +def run_greedy_equality_correctness_test(baseline_llm_generator,
544 + test_llm_generator,
545 + batch_size,
546 + max_output_len,
547 + force_output_len: bool,
548 + print_tokens: bool = False):
549 + """Helper method that compares the outputs of both the baseline LLM and
550 + the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
551 + the same when temperature is zero.
552 """
123 - output_len = 128
553 temperature = 0.0
554
555 prompts = [
556 "Hello, my name is",
557 + "The president of the United States is",
558 + "The capital of France is",
559 + "The future of AI is",
560 + "San Francisco is know for its",
561 + "Facebook was created in 2004 by",
562 + "Curious George is a",
563 + "Python 3.11 brings improvements to its",
564 ]
565
566 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
567 +
568 + # If the test requires that we generated max_output_len tokens, then set the
569 + # sampling params to ignore eos token.
570 + ignore_eos = force_output_len
571 +
572 sampling_params = SamplingParams(
131 - max_tokens=output_len,
132 - ignore_eos=True,
573 + max_tokens=max_output_len,
574 + ignore_eos=ignore_eos,
575 temperature=temperature,
576 )
577
136 - with pytest.raises(AssertionError,
137 - match="Speculative decoding not yet supported for "):
138 - get_output_from_llm_generator(test_llm_generator, prompts,
139 - sampling_params)
578 + spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
579 + test_llm_generator, prompts, sampling_params)
580
581 + (baseline_batch_tokens,
582 + baseline_batch_token_ids) = get_output_from_llm_generator(
583 + baseline_llm_generator, prompts, sampling_params)
584
142 -def get_output_from_llm_generator(
143 - llm_generator, prompts,
144 - sampling_params) -> Tuple[List[str], List[List[int]]]:
145 - for llm in llm_generator:
146 - outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
147 - token_ids = [output.outputs[0].token_ids for output in outputs]
148 - tokens = [output.outputs[0].text for output in outputs]
149 - del llm
585 + assert len(baseline_batch_token_ids) == len(prompts)
586 + assert len(spec_batch_token_ids) == len(prompts)
587
151 - return tokens, token_ids
588 + for i, (baseline_token_ids, baseline_tokens, spec_token_ids,
589 + spec_tokens) in enumerate(
590 + zip(baseline_batch_token_ids, baseline_batch_tokens,
591 + spec_batch_token_ids, spec_batch_tokens)):
592 + if print_tokens:
593 + print(f'{i=} {baseline_tokens=}')
594 + print(f'{i=} {spec_tokens=}')
595 + print(f'{i=} {baseline_token_ids=}')
596 + print(f'{i=} {spec_token_ids=}')
597 + assert baseline_token_ids == spec_token_ids
tests/spec_decode/test_metrics.py
+2 -2
@@ -119,7 +119,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
119 num_draft_tokens = 0
120 k = 5
121
122 - num_possible_tokens = AsyncMetricsCollector.get_max_num_accepted_tokens(
122 + max_num_emitted_tokens = AsyncMetricsCollector.get_max_num_emitted_tokens(
123 num_draft_tokens, k)
124
125 rej_sampler = MagicMock()
@@ -153,7 +153,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
153 assert (metrics.draft_acceptance_rate == num_accepted_tokens /
154 num_draft_tokens)
155 assert (metrics.system_efficiency == num_emitted_tokens /
156 - num_possible_tokens)
156 + max_num_emitted_tokens)
157 else:
158 assert math.isnan(metrics.draft_acceptance_rate)
159 assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py
+2 -2
@@ -344,8 +344,8 @@ def test_draft_proposals_no_speculations():
344 assert torch.is_tensor(proposals.proposal_token_ids)
345 assert torch.is_tensor(proposals.proposal_probs)
346
347 - assert proposals.proposal_token_ids.shape == torch.Size([0, k])
348 - assert proposals.proposal_probs.shape[:-1] == torch.Size([0, k])
347 + assert proposals.proposal_token_ids.shape == torch.Size([batch_size, k])
348 + assert proposals.proposal_probs.shape[:-1] == torch.Size([batch_size, k])
349
350 assert proposals.proposal_lens.shape == torch.Size([batch_size])
351 assert proposals.proposal_lens.tolist() == [0 for _ in range(batch_size)]
tests/spec_decode/test_spec_decode_worker.py
+23 -17
@@ -1,4 +1,5 @@
1 import random
2 +from types import SimpleNamespace
3 from unittest.mock import MagicMock
4
5 import pytest
@@ -62,8 +63,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
63 """Verify SpecDecodeWorker calls the target model with correct
64 inputs. Everything else is mocked out.
65 """
65 - draft_worker = mock_worker(cls=MultiStepWorker)
66 - target_worker = mock_worker()
66 + draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
67 + target_worker = mock_worker(use_spec=False)
68 rejection_sampler = MagicMock(spec=RejectionSampler)
69 rejection_sampler.token_id_dtype = torch.int64
70 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -144,8 +145,10 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
145 """
146 vocab_size = 32_000
147
147 - draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
148 - target_worker = mock_worker(vocab_size=vocab_size)
148 + draft_worker = mock_worker(cls=MultiStepWorker,
149 + vocab_size=vocab_size,
150 + use_spec=False)
151 + target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
152 rejection_sampler = MagicMock(spec=RejectionSampler)
153 rejection_sampler.token_id_dtype = torch.int64
154 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -202,17 +205,16 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
205 num_lookahead_slots=k)
206
207 assert len(rejection_sampler.call_args_list) == 1
205 - args, _ = rejection_sampler.call_args_list[0]
206 - (actual_proposal_scores, actual_bonus_token_ids, actual_proposal_probs,
207 - actual_proposal_token_ids) = args
208 + _, kwargs = rejection_sampler.call_args_list[0]
209 + actual = SimpleNamespace(**kwargs)
210
209 - assert torch.equal(actual_bonus_token_ids,
211 + assert torch.equal(actual.bonus_token_ids,
212 target_token_ids.reshape(batch_size, k + 1)[:, -1:])
213 assert torch.equal(
212 - actual_proposal_scores,
214 + actual.target_probs,
215 target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
214 - assert torch.equal(actual_proposal_token_ids, proposal_token_ids)
215 - assert torch.equal(actual_proposal_probs, proposal_probs)
216 + assert torch.equal(actual.draft_token_ids, proposal_token_ids)
217 + assert torch.equal(actual.draft_probs, proposal_probs)
218
219
220 @pytest.mark.parametrize('k', [1, 2, 6])
@@ -224,8 +226,10 @@ def test_correctly_formats_output(k: int, batch_size: int):
226 """
227 vocab_size = 32_000
228
227 - draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
228 - target_worker = mock_worker(vocab_size=vocab_size)
229 + draft_worker = mock_worker(cls=MultiStepWorker,
230 + vocab_size=vocab_size,
231 + use_spec=False)
232 + target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
233 rejection_sampler = MagicMock(spec=RejectionSampler)
234 rejection_sampler.token_id_dtype = torch.int64
235 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -336,8 +340,10 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
340 """
341 vocab_size = 32_000
342
339 - draft_worker = mock_worker(cls=MultiStepWorker, vocab_size=vocab_size)
340 - target_worker = mock_worker(vocab_size=vocab_size)
343 + draft_worker = mock_worker(cls=MultiStepWorker,
344 + vocab_size=vocab_size,
345 + use_spec=False)
346 + target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
347 rejection_sampler = MagicMock(spec=RejectionSampler)
348 rejection_sampler.token_id_dtype = torch.int64
349 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
@@ -500,8 +506,8 @@ def test_init_device():
506 """Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
507 well as other GPU initialization.
508 """
503 - draft_worker = mock_worker(cls=MultiStepWorker)
504 - target_worker = mock_worker()
509 + draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
510 + target_worker = mock_worker(use_spec=False)
511 rejection_sampler = MagicMock(spec=RejectionSampler)
512 rejection_sampler.token_id_dtype = torch.int64
513 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
tests/spec_decode/utils.py
+5 -2
@@ -63,11 +63,14 @@ def create_execute_model_data(
63 def mock_worker(cls=None,
64 vocab_size: int = 30_000,
65 max_model_len: int = 2048,
66 - rank: int = 0) -> MagicMock:
66 + rank: int = 0,
67 + use_spec: bool = True) -> MagicMock:
68 if cls is None:
69 cls = Worker
70
70 - worker = MagicMock(spec=cls)
71 + spec = cls if use_spec else None
72 +
73 + worker = MagicMock(spec=spec)
74 worker.vocab_size = vocab_size
75 worker.max_model_len = max_model_len
76 worker.rank = rank
vllm/config.py
+65 -2
@@ -655,6 +655,9 @@ class SpeculativeConfig:
655 target_dtype: str,
656 speculative_model: Optional[str],
657 num_speculative_tokens: Optional[int],
658 + speculative_max_model_len: Optional[int],
659 + enable_chunked_prefill: bool,
660 + use_v2_block_manager: bool,
661 ) -> Optional["SpeculativeConfig"]:
662 """Create a SpeculativeConfig if possible, else return None.
663
@@ -672,6 +675,15 @@ class SpeculativeConfig:
675 model, if provided.
676 num_speculative_tokens (Optional[int]): The number of speculative
677 tokens, if provided.
678 + speculative_max_model_len (Optional[int]): The maximum model len of
679 + the speculative model. Used when testing the ability to skip
680 + speculation for some sequences.
681 + enable_chunked_prefill (bool): Whether vLLM is configured to use
682 + chunked prefill or not. Used for raising an error since its not
683 + yet compatible with spec decode.
684 + use_v2_block_manager (bool): Whether vLLM is configured to use the
685 + v2 block manager or not. Used for raising an error since the v2
686 + block manager is required with spec decode.
687
688 Returns:
689 Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
@@ -690,12 +702,21 @@ class SpeculativeConfig:
702 assert (speculative_model is not None
703 and num_speculative_tokens is not None)
704
705 + if enable_chunked_prefill:
706 + raise ValueError(
707 + "Speculative decoding and chunked prefill are "
708 + f"currently mutually exclusive ({enable_chunked_prefill=}).")
709 +
710 + if not use_v2_block_manager:
711 + raise ValueError(
712 + "Speculative decoding requires usage of the V2 "
713 + "block manager. Enable it with --use-v2-block-manager.")
714 +
715 # TODO: The user should be able to specify revision/quantization/max
716 # model len for the draft model. It is not currently supported.
717 draft_revision = None
718 draft_code_revision = None
719 draft_quantization = None
698 - draft_max_model_len = None
720
721 draft_model_config = ModelConfig(
722 model=speculative_model,
@@ -707,7 +728,7 @@ class SpeculativeConfig:
728 revision=draft_revision,
729 code_revision=draft_code_revision,
730 tokenizer_revision=target_model_config.tokenizer_revision,
710 - max_model_len=draft_max_model_len,
731 + max_model_len=None,
732 quantization=draft_quantization,
733 enforce_eager=target_model_config.enforce_eager,
734 max_context_len_to_capture=target_model_config.
@@ -715,6 +736,13 @@ class SpeculativeConfig:
736 max_logprobs=target_model_config.max_logprobs,
737 )
738
739 + draft_model_config.max_model_len = (
740 + SpeculativeConfig._maybe_override_draft_max_model_len(
741 + speculative_max_model_len,
742 + draft_model_config.max_model_len,
743 + target_model_config.max_model_len,
744 + ))
745 +
746 draft_parallel_config = (
747 SpeculativeConfig.create_draft_parallel_config(
748 target_parallel_config))
@@ -725,6 +753,41 @@ class SpeculativeConfig:
753 num_speculative_tokens,
754 )
755
756 + @staticmethod
757 + def _maybe_override_draft_max_model_len(
758 + speculative_max_model_len: Optional[int],
759 + draft_max_model_len: int,
760 + target_max_model_len: int,
761 + ) -> int:
762 + """Determine the max sequence len for the draft model. This is usually
763 + the draft_max_model_len, but may be the target_max_model_len if it is
764 + less than the draft_max_model_len, or may be speculative_max_model_len
765 + if it is specified.
766 +
767 + This is necessary so that sequences do not exceed the capacity of the
768 + draft model or the target model.
769 +
770 + speculative_max_model_len is mainly used for testing that sequences can
771 + skip speculation.
772 + """
773 +
774 + if speculative_max_model_len is not None:
775 +
776 + if speculative_max_model_len > draft_max_model_len:
777 + raise ValueError(f"{speculative_max_model_len=} cannot be "
778 + f"larger than {draft_max_model_len=}")
779 +
780 + if speculative_max_model_len > target_max_model_len:
781 + raise ValueError(f"{speculative_max_model_len=} cannot be "
782 + f"larger than {target_max_model_len=}")
783 +
784 + return speculative_max_model_len
785 +
786 + return min(
787 + draft_max_model_len,
788 + target_max_model_len,
789 + )
790 +
791 @staticmethod
792 def create_draft_parallel_config(
793 target_parallel_config: ParallelConfig) -> ParallelConfig:
vllm/engine/arg_utils.py
+15 -3
@@ -73,6 +73,7 @@ class EngineArgs:
73 # Speculative decoding configuration.
74 speculative_model: Optional[str] = None
75 num_speculative_tokens: Optional[int] = None
76 + speculative_max_model_len: Optional[int] = None
77
78 def __post_init__(self):
79 if self.tokenizer is None:
@@ -237,7 +238,7 @@ class EngineArgs:
238 parser.add_argument('--block-size',
239 type=int,
240 default=EngineArgs.block_size,
240 - choices=[8, 16, 32, 128],
241 + choices=[8, 16, 32],
242 help='Token block size for contiguous chunks of '
243 'tokens.')
244
@@ -420,17 +421,25 @@ class EngineArgs:
421 parser.add_argument(
422 '--speculative-model',
423 type=str,
423 - default=None,
424 + default=EngineArgs.speculative_model,
425 help=
426 'The name of the draft model to be used in speculative decoding.')
427
428 parser.add_argument(
429 '--num-speculative-tokens',
430 type=int,
430 - default=None,
431 + default=EngineArgs.num_speculative_tokens,
432 help='The number of speculative tokens to sample from '
433 'the draft model in speculative decoding.')
434
435 + parser.add_argument(
436 + '--speculative-max-model-len',
437 + type=str,
438 + default=EngineArgs.speculative_max_model_len,
439 + help='The maximum sequence length supported by the '
440 + 'draft model. Sequences over this length will skip '
441 + 'speculation.')
442 +
443 parser.add_argument('--model-loader-extra-config',
444 type=str,
445 default=EngineArgs.model_loader_extra_config,
@@ -481,6 +490,9 @@ class EngineArgs:
490 target_dtype=self.dtype,
491 speculative_model=self.speculative_model,
492 num_speculative_tokens=self.num_speculative_tokens,
493 + speculative_max_model_len=self.speculative_max_model_len,
494 + enable_chunked_prefill=self.enable_chunked_prefill,
495 + use_v2_block_manager=self.use_v2_block_manager,
496 )
497
498 scheduler_config = SchedulerConfig(
vllm/engine/llm_engine.py
+30 -8
@@ -22,7 +22,7 @@ from vllm.lora.request import LoRARequest
22 from vllm.outputs import RequestOutput
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25 - SequenceGroup)
25 + SequenceGroup, SequenceStage)
26 from vllm.transformers_utils.detokenizer import Detokenizer
27 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
28 get_tokenizer_group)
@@ -480,9 +480,12 @@ class LLMEngine:
480 seq_group = scheduled_seq_group.seq_group
481 seq_group.update_num_computed_tokens(
482 scheduled_seq_group.token_chunk_size)
483 - # If uncomputed tokens > 0, it means prefill is chunked.
484 - # We don't need to process outputs in that case.
485 - if seq_group.get_num_uncomputed_tokens() == 0:
483 +
484 + # If all sequences in the sequence group are in DECODE, then we can
485 + # process the output tokens. Otherwise, they are (chunked) prefill
486 + # samples and should not be processed.
487 + stages = [seq.data._stage for seq in seq_group.seqs_dict.values()]
488 + if all(stage == SequenceStage.DECODE for stage in stages):
489 self.output_processor.process_outputs(seq_group, outputs)
490
491 # Free the finished sequence groups.
@@ -569,7 +572,8 @@ class LLMEngine:
572
573 # Log stats.
574 if self.log_stats:
572 - self.stat_logger.log(self._get_stats(scheduler_outputs))
575 + self.stat_logger.log(
576 + self._get_stats(scheduler_outputs, model_output=output))
577
578 return request_outputs
579
@@ -578,9 +582,18 @@ class LLMEngine:
582 if self.log_stats:
583 self.stat_logger.log(self._get_stats(scheduler_outputs=None))
584
581 - def _get_stats(self,
582 - scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
583 - """Get Stats to be Logged to Prometheus."""
585 + def _get_stats(
586 + self,
587 + scheduler_outputs: Optional[SchedulerOutputs],
588 + model_output: Optional[List[SamplerOutput]] = None) -> Stats:
589 + """Get Stats to be Logged to Prometheus.
590 +
591 + Args:
592 + scheduler_outputs: Optional, used to populate metrics related to
593 + the scheduled batch,
594 + model_output: Optional, used to emit speculative decoding metrics
595 + which are created by the workers.
596 + """
597 now = time.time()
598
599 # KV Cache Usage in %.
@@ -637,6 +650,14 @@ class LLMEngine:
650 time_to_first_tokens = time_last_iters if prompt_run else []
651 time_per_output_tokens = [] if prompt_run else time_last_iters
652
653 + # Spec decode, if enabled, emits specialized metrics from the worker in
654 + # sampler output.
655 + if model_output and (model_output[0].spec_decode_worker_metrics
656 + is not None):
657 + spec_decode_metrics = model_output[0].spec_decode_worker_metrics
658 + else:
659 + spec_decode_metrics = None
660 +
661 return Stats(
662 now=now,
663 num_running=num_running,
@@ -649,6 +670,7 @@ class LLMEngine:
670 time_to_first_tokens=time_to_first_tokens,
671 time_per_output_tokens=time_per_output_tokens,
672 time_e2e_requests=time_e2e_requests,
673 + spec_decode_metrics=spec_decode_metrics,
674 )
675
676 def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/engine/metrics.py
+22 -1
@@ -1,6 +1,6 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import Dict, List, Protocol
3 +from typing import TYPE_CHECKING, Dict, List, Optional, Protocol
4
5 import numpy as np
6 from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -8,6 +8,9 @@ from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
8
9 from vllm.logger import init_logger
10
11 +if TYPE_CHECKING:
12 + from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
13 +
14 logger = init_logger(__name__)
15
16 disable_created_metrics()
@@ -118,6 +121,8 @@ class Stats:
121 time_per_output_tokens: List[float]
122 time_e2e_requests: List[float]
123
124 + spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
125 +
126
127 class SupportsMetricsInfo(Protocol):
128
@@ -235,3 +240,19 @@ class StatLogger:
240 self.num_prompt_tokens = []
241 self.num_generation_tokens = []
242 self.last_local_log = stats.now
243 +
244 + if stats.spec_decode_metrics is not None:
245 + logger.info(
246 + self._format_spec_decode_metrics_str(
247 + stats.spec_decode_metrics))
248 +
249 + def _format_spec_decode_metrics_str(
250 + self, metrics: "SpecDecodeWorkerMetrics") -> str:
251 +
252 + return ("Speculative metrics: "
253 + f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
254 + f"System efficiency: {metrics.system_efficiency:.3f}, "
255 + f"Number of speculative tokens: {metrics.num_spec_tokens}, "
256 + f"Number of accepted tokens: {metrics.accepted_tokens}, "
257 + f"Number of draft tokens tokens: {metrics.draft_tokens}, "
258 + f"Number of emitted tokens tokens: {metrics.emitted_tokens}.")
vllm/executor/gpu_executor.py
+1
@@ -83,6 +83,7 @@ class GPUExecutor(ExecutorBase):
83 scheduler_config=self.scheduler_config,
84 device_config=self.device_config,
85 cache_config=self.cache_config,
86 + # TODO allow draft-model specific load config.
87 load_config=self.load_config,
88 local_rank=0,
89 rank=0,
vllm/model_executor/layers/rejection_sampler.py
+7
@@ -144,6 +144,7 @@ class RejectionSampler(nn.Module):
144 recovered_probs = self._get_recovered_probs(
145 target_probs, draft_probs).reshape(batch_size * k, vocab_size)
146
147 + # NOTE: the recovered_probs are overwritten by this method.
148 recovered_token_ids = _multinomial(recovered_probs,
149 num_samples=1).reshape(
150 batch_size, k)
@@ -307,6 +308,12 @@ class RejectionSampler(nn.Module):
308 output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
309 bonus_token_ids, -1)
310
311 + # We disable bonus tokens because it causes corrupt KV cache for
312 + # proposal methods that require KV cache. We can fix it by "prefilling"
313 + # the bonus token in the proposer. The following issue tracks the fix.
314 + # https://github.com/vllm-project/vllm/issues/4212
315 + output_with_bonus_tokens[:, -1] = -1
316 +
317 # Fill the recovered token ids.
318 output.mul_(~after_false_mask).add_(
319 recovered_token_ids.mul(after_false_mask))
vllm/model_executor/layers/sampler.py
+167 -15
@@ -35,6 +35,14 @@ class Sampler(nn.Module):
35 in logits for each token in the input prompt.
36 """
37
38 + def __init__(self):
39 + super().__init__()
40 +
41 + # Whether or not the SamplerOutput should have on-device tensors
42 + # containing the sampled token ids and probabilities. This is used by
43 + # speculative decoding.
44 + self.include_gpu_probs_tensor = False
45 +
46 def forward(
47 self,
48 logits: torch.Tensor,
@@ -79,13 +87,45 @@ class Sampler(nn.Module):
87 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
88
89 # Sample the next tokens.
82 - sample_results = _sample(probs, logprobs, sampling_metadata,
83 - sampling_tensors)
90 + sample_results, maybe_sampled_tokens_tensor = _sample(
91 + probs,
92 + logprobs,
93 + sampling_metadata,
94 + sampling_tensors,
95 + include_gpu_probs_tensor=self.include_gpu_probs_tensor,
96 + modify_greedy_probs=self._should_modify_greedy_probs_inplace,
97 + )
98 +
99 + if self.include_gpu_probs_tensor:
100 + assert maybe_sampled_tokens_tensor is not None
101 + sampled_tokens_tensor = maybe_sampled_tokens_tensor
102 + on_device_tensors = (probs, sampled_tokens_tensor)
103 + else:
104 + on_device_tensors = None
105 +
106 # Get the logprobs query results.
107 prompt_logprobs, sample_logprobs = _get_logprobs(
108 logprobs, sampling_metadata, sample_results)
87 - return _build_sampler_output(sample_results, sampling_metadata,
88 - prompt_logprobs, sample_logprobs)
109 + return _build_sampler_output(sample_results,
110 + sampling_metadata,
111 + prompt_logprobs,
112 + sample_logprobs,
113 + on_device_tensors=on_device_tensors)
114 +
115 + @property
116 + def _should_modify_greedy_probs_inplace(self) -> bool:
117 + """Whether or not the sampler should modify the probability distribution
118 + of greedily-sampled tokens such that multinomial sampling would sample
119 + the greedily-sampled token.
120 +
121 + In other words, if True then we set the probability of the greedily-
122 + sampled token to 1.
123 +
124 + This is used by speculative decoding, which requires that the sampling
125 + method be encoded into the probability distribution.
126 + """
127 + # Modify greedy probs if include_gpu_probs_tensor is set.
128 + return self.include_gpu_probs_tensor
129
130
131 def _get_bin_counts_and_mask(
@@ -359,7 +399,9 @@ def _sample_with_torch(
399 probs: torch.Tensor,
400 logprobs: torch.Tensor,
401 sampling_metadata: SamplingMetadata,
362 -) -> List[Tuple[List[int], List[int]]]:
402 + include_gpu_probs_tensor: bool,
403 + modify_greedy_probs: bool,
404 +) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
405 categorized_seq_group_ids = {t: [] for t in SamplingType}
406 categorized_sample_indices = sampling_metadata.categorized_sample_indices
407 for i, seq_group in enumerate(sampling_metadata.seq_groups):
@@ -371,6 +413,15 @@ def _sample_with_torch(
413 sample_metadata = {}
414 multinomial_samples = {}
415
416 + # Create output tensor for sampled token ids.
417 + if include_gpu_probs_tensor:
418 + sampled_token_ids_tensor = torch.empty(logprobs.shape[0],
419 + 1,
420 + dtype=torch.long,
421 + device=logprobs.device)
422 + else:
423 + sampled_token_ids_tensor = None
424 +
425 # Counterintiutively, having two loops here is actually faster.
426 # The first loop can run without waiting on GPU<->CPU sync.
427 for sampling_type in SamplingType:
@@ -383,9 +434,25 @@ def _sample_with_torch(
434 is_prompts = [i < sampling_metadata.num_prompts for i in seq_group_ids]
435 sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
436 is_prompts, sample_indices)
437 + long_sample_indices = sample_indices.long()
438 +
439 if sampling_type == SamplingType.GREEDY:
387 - greedy_samples = torch.argmax(logprobs[sample_indices.long()],
440 + greedy_samples = torch.argmax(logprobs[long_sample_indices],
441 dim=-1)
442 +
443 + if include_gpu_probs_tensor:
444 + # Store sampled tokens in output tensor.
445 + sampled_token_ids_tensor[
446 + long_sample_indices] = greedy_samples.unsqueeze(-1)
447 +
448 + if modify_greedy_probs:
449 + # If required, modify the probabilities such that sampling from
450 + # the modified distribution would always sample the argmax
451 + # token id.
452 + _modify_greedy_probs_inplace(logprobs, probs,
453 + long_sample_indices,
454 + greedy_samples)
455 +
456 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
457 max_best_of_in_batch = 1
458 for seq_group, is_prompt in zip(seq_groups, is_prompts):
@@ -397,15 +464,23 @@ def _sample_with_torch(
464 "seq_groups": seq_groups,
465 "generators": sampling_metadata.generators,
466 }
467 +
468 multinomial_samples[sampling_type] = _multinomial(
401 - probs[sample_indices.long()], max_best_of_in_batch,
469 + probs[long_sample_indices], max_best_of_in_batch,
470 **seeded_args)
471 +
472 + if include_gpu_probs_tensor:
473 + # Store sampled tokens in output tensor.
474 + sampled_token_ids_tensor[
475 + long_sample_indices] = multinomial_samples[sampling_type]
476 +
477 elif sampling_type == SamplingType.BEAM:
478 beam_search_logprobs = logprobs[sample_indices]
479 else:
480 raise ValueError(f"Unsupported sampling type: {sampling_type}")
481
482 # GPU<->CPU sync happens in the loop below.
483 + # This also converts the sample output to Python objects.
484
485 for sampling_type in SamplingType:
486 if sampling_type not in sample_metadata:
@@ -427,7 +502,7 @@ def _sample_with_torch(
502 sample_results_dict[i]
503 for i in range(len(sampling_metadata.seq_groups))
504 ]
430 - return sample_results
505 + return sample_results, sampled_token_ids_tensor
506
507
508 def _sample_with_triton_kernel(
@@ -511,12 +586,17 @@ def _sample_with_triton_kernel(
586
587
588 def _sample(
514 - probs: torch.Tensor,
515 - logprobs: torch.Tensor,
516 - sampling_metadata: SamplingMetadata,
517 - sampling_tensors: SamplingTensors,
518 -) -> List[Tuple[List[int], List[int]]]:
519 - return _sample_with_torch(probs, logprobs, sampling_metadata)
589 + probs: torch.Tensor, logprobs: torch.Tensor,
590 + sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
591 + include_gpu_probs_tensor: bool, modify_greedy_probs: bool
592 +) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
593 + return _sample_with_torch(
594 + probs,
595 + logprobs,
596 + sampling_metadata,
597 + include_gpu_probs_tensor=include_gpu_probs_tensor,
598 + modify_greedy_probs=modify_greedy_probs,
599 + )
600
601 # TODO: Enable once Triton kernel & associated code is faster.
602 # return _sample_with_triton_kernel(probs, logprobs, sampling_metadata,
@@ -680,12 +760,73 @@ def _get_logprobs(
760 return result_prompt_logprobs, result_sample_logprobs
761
762
763 +def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
764 + sample_indices: torch.Tensor,
765 + greedy_samples: torch.Tensor) -> None:
766 + """Modify the probability distributions of the greedily-sampled tokens such
767 + that each sampled token has a "probability" of 1.0. This is required by
768 + speculative decoding, which depends on the sampling method being encoded
769 + within the probability distribution for correctness.
770 +
771 + # Why do we only need to do this for greedy sampling?
772 +
773 + vLLM's sampler performs the following steps for greedy or multinomial
774 + (random) sampling:
775 + 1. Get logits from model.
776 + 2. Modify logits according to per-sequence sampling parameters.
777 + - Multiply by temperature, top-k and top-p masking, penalize tokens
778 + according to their frequency, etc.
779 + 3. Sample a token.
780 + - Random sampling simply samples from the modified probability
781 + distribution.
782 + - Greedy sampling performs `argmax` to obtain the token with the
783 + highest likelihood.
784 +
785 + Ignoring greedy sampling for a moment, we find that the computed probability
786 + distribution has the following property: we can sample from it independently
787 + and find that the token sampled by the Sampler has a frequency corresponding
788 + to how often we see it in our sampling. In other words, for tokens sampled
789 + with vLLM's random SamplingType, the computed probability distribution
790 + encodes the sampling methodology completely.
791 +
792 + Greedy sampling does not normally have this property. vLLM modifies logits
793 + according to sampling params, then performs `argmax`, then returns the
794 + sampled token and the computed probability distribution. If we sample from
795 + the distribution, we'll find the likelihood of the greedily-sampled token
796 + is not always 1.0.
797 +
798 + Since lossless speculative decoding requires that the sampling methodology
799 + be encoded within the probability distribution, we are motivated to modify
800 + the probability distribution such that the sampled token has probability 1
801 + when speculative decoding is used.
802 +
803 + NOTE: Alternatively, we could use an extremely low temperature to achieve
804 + greedy sampling using multinomial computation and unite the codepaths. This
805 + has implications on the overall design of the sampler, e.g. how to record
806 + accurate logprobs for the user, so this improvement is deferred to later.
807 + """
808 + logprobs[sample_indices, :] = -float('inf')
809 + logprobs[sample_indices, greedy_samples] = 0.0
810 + probs[sample_indices, :] = 0
811 + probs[sample_indices, greedy_samples] = 1.0
812 +
813 +
814 def _build_sampler_output(
815 sample_results: List[Tuple[List[int], List[int]]],
816 sampling_metadata: SamplingMetadata,
817 prompt_logprobs: List[Optional[PromptLogprobs]],
818 sample_logprobs: List[SampleLogprobs],
819 + on_device_tensors: Optional[Tuple[torch.Tensor, torch.Tensor]],
820 ) -> SamplerOutput:
821 + """Construct Python objects with the output of sampling.
822 +
823 + Args:
824 + on_device_tensors: Tuple containing on-device tensors with the
825 + probabilities used in sampling and the sampled token ids. This
826 + allows post-processing without copies to CPU/serialization, e.g. in
827 + speculative decoding rejection sampling.
828 + """
829 +
830 sampler_output = []
831 for (seq_group, sample_result, group_prompt_logprobs,
832 group_sample_logprobs) in zip(sampling_metadata.seq_groups,
@@ -701,4 +842,15 @@ def _build_sampler_output(
842 SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
843 sampler_output.append(
844 SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
704 - return SamplerOutput(outputs=sampler_output)
845 +
846 + # If not specified, store None values in SamplerOutput.
847 + if on_device_tensors is not None:
848 + sampled_token_probs, sampled_token_ids = on_device_tensors
849 + else:
850 + sampled_token_probs, sampled_token_ids = (None, None)
851 +
852 + return SamplerOutput(
853 + outputs=sampler_output,
854 + sampled_token_probs=sampled_token_probs,
855 + sampled_token_ids=sampled_token_ids,
856 + )
vllm/spec_decode/batch_expansion.py
+42 -28
@@ -6,8 +6,8 @@ import torch
6 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeScorer, SpeculativeScores)
9 -from vllm.spec_decode.util import (get_all_seq_ids, maybe_mock_device_tensors,
10 - nvtx_range, sampler_output_to_torch,
9 +from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10 + sampler_output_to_torch,
11 split_batch_by_proposal_len)
12 from vllm.worker.worker_base import WorkerBase
13
@@ -72,10 +72,16 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
72 proposal_lens_list = proposals.proposal_lens.tolist()
73 proposal_token_ids_list = proposals.proposal_token_ids.tolist()
74
75 + # Filter the list to ignore -1 proposals.
76 + proposal_token_ids_list_without_skips = [
77 + proposals for proposals in proposal_token_ids_list
78 + if -1 not in proposals
79 + ]
80 +
81 (spec_indices, non_spec_indices, target_seq_group_metadata_list,
82 num_scoring_tokens) = self._expand_batch(
83 seq_group_metadata_list=seq_group_metadata_list,
78 - proposal_token_ids_list=proposal_token_ids_list,
84 + proposal_token_ids_list=proposal_token_ids_list_without_skips,
85 proposal_lens_list=proposal_lens_list,
86 )
87
@@ -89,7 +95,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
95 target_sampler_output = target_sampler_output[0]
96
97 all_tokens, all_probs = self._contract_batch(
92 - original_bs=len(seq_group_metadata_list),
98 + contracted_bs=len(seq_group_metadata_list),
99 target_sampler_output=target_sampler_output,
100 proposals=proposals,
101 num_scoring_tokens=num_scoring_tokens,
@@ -128,14 +134,21 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
134 select_proposal_len_zero=True)
135
136 target_seq_group_metadata_list = self._create_scoring_model_input(
131 - spec_seqs, proposal_token_ids_list)
137 + seq_group_metadata_list=spec_seqs,
138 + proposal_token_ids=proposal_token_ids_list,
139 + # NOTE: We determine the seq ids in the expanded batch using the
140 + # full seq_group_metadata_list, instead of only spec_seqs.
141 + target_seq_ids_iter=self._create_target_seq_id_iterator(
142 + seq_ids=get_all_seq_ids(seq_group_metadata_list)),
143 + )
144 +
145 num_scoring_tokens = len(target_seq_group_metadata_list)
146 target_seq_group_metadata_list.extend(non_spec_seqs)
147
148 return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
149 num_scoring_tokens)
150
138 - def _contract_batch(self, original_bs: int,
151 + def _contract_batch(self, contracted_bs: int,
152 target_sampler_output: List[SamplerOutput],
153 proposals: SpeculativeProposals,
154 num_scoring_tokens: int, non_spec_indices: List[int],
@@ -144,42 +157,41 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
157 """Contract the expanded batch back into its original size.
158 This maps the scores of speculative tokens back to their original
159 sequences.
147 - """
148 -
149 - # We mock the device tensors until PR 7/9 is merged (e2e correctness).
150 - # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
151 - maybe_mock_device_tensors(
152 - sampler_output=target_sampler_output,
153 - batch_size=len(non_spec_indices) + num_scoring_tokens,
154 - vocab_size=self._vocab_size,
155 - device=self._device,
156 - )
160
161 + contracted_bs is the original batch size, and the batch size that the
162 + target_sampler_output will be contracted to.
163 + """
164 (target_token_ids, target_probs, non_spec_target_token_ids,
165 non_spec_target_probs) = self._split_scoring_output(
166 target_sampler_output, num_scoring_tokens)
167
168 # Map distinct sequences used to score each token
169 # of shape [batch_size * k + 1] back to [batch_size, k + 1].
164 - batch_size, k = proposals.proposal_token_ids.shape
170 + expanded_batch_size, k = proposals.proposal_token_ids.shape
171 +
172 + # The number of tokens in the expanded batch used for speculation is
173 + # equal to the total expanded batch size minus the number of samples for
174 + # non-speculative sequences.
175 + non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
176 + spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
177
178 target_token_ids = target_token_ids.squeeze().reshape(
167 - batch_size, k + 1)
168 - target_probs = target_probs.squeeze().reshape(batch_size, k + 1,
179 + spec_expanded_bs, k + 1)
180 + target_probs = target_probs.squeeze().reshape(spec_expanded_bs, k + 1,
181 self._vocab_size)
182
171 - all_tokens = torch.full(size=(original_bs, k + 1),
183 + all_tokens = torch.full(size=(contracted_bs, k + 1),
184 fill_value=-1,
185 device=self._device,
186 dtype=torch.long)
175 - all_probs = torch.zeros(original_bs,
187 + all_probs = torch.zeros(contracted_bs,
188 k + 1,
189 self._vocab_size,
190 device=self._device,
191 dtype=torch.float32)
192
193 if non_spec_indices:
182 - all_tokens[non_spec_indices, 0] = non_spec_target_token_ids
194 + all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
195 all_probs[non_spec_indices, :1, :] = non_spec_target_probs
196
197 if spec_indices:
@@ -189,20 +201,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
201 return all_tokens, all_probs
202
203 def _create_scoring_model_input(
192 - self,
193 - seq_group_metadata_list: List[SequenceGroupMetadata],
194 - proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
204 + self,
205 + seq_group_metadata_list: List[SequenceGroupMetadata],
206 + proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
207 + target_seq_ids_iter: Iterator[TargetSeqId],
208 ) -> List[SequenceGroupMetadata]:
209 """Given the original input sequences and proposed tokens from the draft
210 model, create a list of target sequences that can be used for scoring.
211 +
212 + target_seq_ids_iter provides sequence ids for the expanded batch,
213 + fulfilling the requirement that no seq id in the expanded batch is equal
214 + to the seq id in the original batch.
215 """
216
217 if not seq_group_metadata_list:
218 return []
219
203 - target_seq_ids_iter = self._create_target_seq_id_iterator(
204 - get_all_seq_ids(seq_group_metadata_list))
205 -
220 target_seq_group_metadata = list(
221 chain.from_iterable(
222 self._create_target_seq_group_metadata(
vllm/spec_decode/interfaces.py
+2 -2
@@ -24,9 +24,9 @@ class SpeculativeProposals:
24
25 def __repr__(self):
26 return (f"SpeculativeProposals("
27 - f"proposal_token_ids={self.proposal_token_ids.shape}, "
27 + f"proposal_token_ids={self.proposal_token_ids}, "
28 f"proposal_probs={self.proposal_probs.shape}, "
29 - f"proposal_lens={self.proposal_lens.shape})")
29 + f"proposal_lens={self.proposal_lens})")
30
31
32 @dataclass
vllm/spec_decode/metrics.py
+23 -8
@@ -147,15 +147,16 @@ class AsyncMetricsCollector:
147 emitted_tokens = self._aggregate_num_emitted_tokens.item()
148 draft_tokens = self._aggregate_num_draft_tokens
149
150 - num_possible_tokens = self.get_max_num_accepted_tokens(draft_tokens, k)
150 + max_num_emitted_tokens = self.get_max_num_emitted_tokens(
151 + draft_tokens, k)
152
153 if draft_tokens > 0:
154 draft_acceptance_rate = accepted_tokens / draft_tokens
155 else:
156 draft_acceptance_rate = float("nan")
157
157 - if num_possible_tokens > 0:
158 - system_efficiency = emitted_tokens / num_possible_tokens
158 + if max_num_emitted_tokens > 0:
159 + system_efficiency = emitted_tokens / max_num_emitted_tokens
160 else:
161 system_efficiency = float("nan")
162
@@ -169,8 +170,22 @@ class AsyncMetricsCollector:
170 )
171
172 @staticmethod
172 - def get_max_num_accepted_tokens(draft_tokens: int, k: int) -> int:
173 - # Divide by k since batch size can be variable.
174 - total_num_spec_seqs = draft_tokens / k
175 - num_accepted_per_seq_if_all_accepted = k + 1
176 - return int(total_num_spec_seqs / num_accepted_per_seq_if_all_accepted)
173 + def get_max_num_emitted_tokens(draft_tokens: int, k: int) -> int:
174 + """Calculate the number of emitted tokens, assuming all tokens are
175 + accepted.
176 +
177 + This is equal to the number of sequences that have been speculated on,
178 + times (speculation len + 1). The +1 comes from the bonus token.
179 + """
180 + # Determine the number of sequences that have been speculated on. Since
181 + # the batch size can be variable, we divide by k.
182 + assert draft_tokens % k == 0
183 + total_num_spec_seqs = draft_tokens // k
184 +
185 + # A single sequence may emit k accepted tokens and one bonus token in
186 + # the best case.
187 + num_emitted_per_seq_if_all_accepted = k + 1
188 +
189 + # The max num of emitted tokens is the number of speculated sequences
190 + # times the max emitted per seq.
191 + return total_num_spec_seqs * num_emitted_per_seq_if_all_accepted
vllm/spec_decode/multi_step_worker.py
+10 -19
@@ -6,8 +6,7 @@ import torch
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 -from vllm.spec_decode.util import (maybe_mock_device_tensors,
10 - sampler_output_to_torch)
9 +from vllm.spec_decode.util import sampler_output_to_torch
10 from vllm.worker.worker import Worker
11
12
@@ -329,12 +328,15 @@ class DraftModelTop1Proposer(SpeculativeProposer):
328 """
329 if maybe_sampler_output is None:
330 # If no speculative tokens, the sampler output will be None.
332 - # In this case we return empty tensors.
333 - proposal_tokens = torch.zeros(0,
334 - max_proposal_len,
335 - dtype=torch.long,
336 - device=self._device)
337 - proposal_probs = torch.zeros(0,
331 + # In this case we return empty proposals.
332 + proposal_tokens = torch.full(size=(
333 + batch_size,
334 + max_proposal_len,
335 + ),
336 + fill_value=-1,
337 + dtype=torch.long,
338 + device=self._device)
339 + proposal_probs = torch.zeros(batch_size,
340 max_proposal_len,
341 self._vocab_size,
342 dtype=torch.float32,
@@ -345,17 +347,6 @@ class DraftModelTop1Proposer(SpeculativeProposer):
347 return proposal_tokens, proposal_probs, proposal_lens_tensor
348
349 sampler_output = maybe_sampler_output
348 -
349 - # We mock the device tensors until PR 7/9 is merged (e2e correctness).
350 - # https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
351 - for step_output in sampler_output:
352 - maybe_mock_device_tensors(
353 - sampler_output=step_output,
354 - batch_size=len(proposal_lens),
355 - vocab_size=self._vocab_size,
356 - device=self._device,
357 - )
358 -
350 proposal_tokens, proposal_probs = sampler_output_to_torch(
351 sampler_output)
352
vllm/spec_decode/spec_decode_worker.py
+43 -6
@@ -111,6 +111,32 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
111 device=self.device,
112 vocab_size=self._vocab_size)
113
114 + self._configure_model_sampler_for_spec_decode()
115 +
116 + def _configure_model_sampler_for_spec_decode(self):
117 + """Configure model sampler to emit GPU tensors. This allows spec decode
118 + to keep data on device without transferring to CPU and serializing,
119 + which significantly reduces overhead of rejection sampling.
120 +
121 + NOTE(cade): This breaks abstraction boundaries pretty badly. The better
122 + design is to have the "move to CPU and serialize" sampling decision be
123 + done outside of the model/sampler; this way the "last-mile" worker
124 + object which interfaces with the scheduler can serialize and incur the
125 + performance hit as necessary. This allows us to run the worker several
126 + iterations in a row without incurring the "move to CPU and serialize"
127 + performance penalty.
128 +
129 + Since this requires a large change to vLLM, we defer it to later and
130 + temporarily accept this broken abstraction boundary.
131 +
132 + NOTE(cade): This will require a special check if the proposer worker
133 + does not have a sampler (e.g. ngram speculation).
134 + """
135 + (self.scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor
136 + ) = True
137 + (self.proposer_worker.model_runner.model.sampler.
138 + include_gpu_probs_tensor) = True
139 +
140 def determine_num_available_blocks(self) -> Tuple[int, int]:
141 """Determine the number of cache blocks to use.
142
@@ -286,15 +312,26 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
312 select_proposal_len_zero=True)
313 original_indices = spec_indices + non_spec_indices
314
289 - proposal_probs = proposal_scores.probs[spec_indices, :-1]
290 - bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
315 + # Get probabilities of target model, excluding bonus token.
316 + proposal_verifier_probs = proposal_scores.probs[spec_indices, :-1]
317 +
318 + # Get non-speculative sampled tokens from target model.
319 non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
320
321 + # Get bonus tokens from target model.
322 + bonus_token_ids = proposal_scores.token_ids[spec_indices, -1:]
323 +
324 + # Get probabilities according to proposal method.
325 + proposal_probs = proposals.proposal_probs[spec_indices]
326 +
327 + # Get proposed tokens.
328 + proposal_token_ids = proposals.proposal_token_ids[spec_indices]
329 +
330 accepted_token_ids = self.rejection_sampler(
294 - proposal_probs,
295 - bonus_token_ids,
296 - proposals.proposal_probs,
297 - proposals.proposal_token_ids,
331 + target_probs=proposal_verifier_probs,
332 + bonus_token_ids=bonus_token_ids,
333 + draft_probs=proposal_probs,
334 + draft_token_ids=proposal_token_ids,
335 )
336
337 # Append output tokens from non-speculative sequences to