@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80ca1e6a3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Speculative Decoding 2/2 ] Integrate typical acceptance sampler into Spec Decode Worker (#5348)

sroy745 committed Jul 1, 2024 at 00:33 UTC 80ca1e6a3a28a0373dc00c5b4fe956c16de952fa
14 files changed +482 -210
tests/samplers/test_typical_acceptance_sampler.py
+64 -32
@@ -52,6 +52,19 @@ def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
52 return draft_token_ids
53
54
55 +def get_acceptance_sampler(
56 + posterior_threshold: float = 0.03,
57 + posterior_alpha: float = 0.9,
58 + disable_bonus_tokens: bool = False,
59 + strict_mode: bool = False,
60 +) -> TypicalAcceptanceSampler:
61 + """
62 + Initializes and returns a TypicalAcceptanceSampler.
63 + """
64 + return TypicalAcceptanceSampler(posterior_threshold, posterior_alpha,
65 + disable_bonus_tokens, strict_mode)
66 +
67 +
68 @pytest.mark.parametrize("k", list(range(1, 6)))
69 @pytest.mark.parametrize("vocab_size", [30_000, 50_000])
70 @pytest.mark.parametrize("batch_size", list(range(1, 32)))
@@ -64,7 +77,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
77 different combinations of k, vocab_size, batch_size and num devices.
78 """
79 torch.set_default_device(device)
67 - typical_acceptance_sampler = TypicalAcceptanceSampler()
80 + typical_acceptance_sampler = get_acceptance_sampler()
81 typical_acceptance_sampler.init_gpu_tensors(rank=0)
82 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
83 bonus_token_ids = torch.randint(low=0,
@@ -76,7 +89,10 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
89 size=(batch_size, k),
90 dtype=torch.int64)
91 # Verify that sampling succeeds for all cases.
79 - typical_acceptance_sampler(target_probs, bonus_token_ids, draft_token_ids)
92 + typical_acceptance_sampler(target_probs,
93 + bonus_token_ids,
94 + draft_probs=None,
95 + draft_token_ids=draft_token_ids)
96
97
98 @pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
@@ -94,7 +110,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
110 batch_size = 5
111 vocab_size = 30_000
112 torch.set_default_device(device)
97 - typical_acceptance_sampler = TypicalAcceptanceSampler(strict_mode=True)
113 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
114 typical_acceptance_sampler.init_gpu_tensors(rank=0)
115 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
116 bonus_token_ids = torch.randint(low=0,
@@ -125,8 +141,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
141 oob_token_ids[0][0] = rogue_token_id
142
143 with pytest.raises(AssertionError):
128 - typical_acceptance_sampler(target_probs, bonus_token_ids,
129 - draft_token_ids)
144 + typical_acceptance_sampler(target_probs,
145 + bonus_token_ids,
146 + draft_probs=None,
147 + draft_token_ids=draft_token_ids)
148
149
150 @pytest.mark.parametrize("seed", list(range(10)))
@@ -151,7 +169,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
169 batch_size = 5
170 vocab_size = 30_000
171 torch.set_default_device(device)
154 - typical_acceptance_sampler = TypicalAcceptanceSampler(
172 + typical_acceptance_sampler = get_acceptance_sampler(
173 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174 typical_acceptance_sampler.init_gpu_tensors(rank=0)
175 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -163,9 +181,11 @@ def test_uniform_target_distribution_accepts_all_tokens(
181 high=vocab_size,
182 size=(batch_size, 1),
183 dtype=torch.int64)
166 - output_token_ids = typical_acceptance_sampler(target_probs,
167 - bonus_token_ids,
168 - draft_token_ids)
184 + output_token_ids = typical_acceptance_sampler(
185 + target_probs,
186 + bonus_token_ids,
187 + draft_probs=None,
188 + draft_token_ids=draft_token_ids)
189 # We are using a uniform target probability distribution.
190 # For a uniform distribution the entropy is very high and it
191 # should lead to all draft tokens being accepted. Verify that.
@@ -203,7 +223,7 @@ def test_temperature_zero_target_distribution(seed: int,
223 vocab_size = 30_000
224 torch.set_default_device(device)
225
206 - typical_acceptance_sampler = TypicalAcceptanceSampler(
226 + typical_acceptance_sampler = get_acceptance_sampler(
227 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
228 typical_acceptance_sampler.init_gpu_tensors(rank=0)
229 # Simulate temperature 0 probability distribution for target probabilities
@@ -224,9 +244,11 @@ def test_temperature_zero_target_distribution(seed: int,
244 # 1.0 tokens in the target distribution we will reject all of them and
245 # fallback to the greedy sampling for selecting 1 token for each sequence.
246 # Verify the same.
227 - output_token_ids = typical_acceptance_sampler(target_probs,
228 - bonus_token_ids,
229 - draft_token_ids)
247 + output_token_ids = typical_acceptance_sampler(
248 + target_probs,
249 + bonus_token_ids,
250 + draft_probs=None,
251 + draft_token_ids=draft_token_ids)
252 assert output_token_ids.shape[0] == batch_size
253 assert output_token_ids.shape[1] == (k + 1)
254 assert torch.all(output_token_ids[:, -1] == -1)
@@ -261,7 +283,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
283 batch_size = 4
284 vocab_size = 30_000
285 torch.set_default_device(device)
264 - typical_acceptance_sampler = TypicalAcceptanceSampler(
286 + typical_acceptance_sampler = get_acceptance_sampler(
287 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
288 typical_acceptance_sampler.init_gpu_tensors(rank=0)
289 # For sequences 0 and 2 set the distribution to a temperature
@@ -277,9 +299,11 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
299 high=vocab_size,
300 size=(batch_size, 1),
301 dtype=torch.int64)
280 - output_token_ids = typical_acceptance_sampler(target_probs,
281 - bonus_token_ids,
282 - draft_token_ids)
302 + output_token_ids = typical_acceptance_sampler(
303 + target_probs,
304 + bonus_token_ids,
305 + draft_probs=None,
306 + draft_token_ids=draft_token_ids)
307 # verify the shape of output_token_ids
308 assert output_token_ids.shape[0] == batch_size
309 assert output_token_ids.shape[1] == (k + 1)
@@ -326,7 +350,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
350 batch_size = 1
351 vocab_size = 30_000
352 torch.set_default_device(device)
329 - typical_acceptance_sampler = TypicalAcceptanceSampler(
353 + typical_acceptance_sampler = get_acceptance_sampler(
354 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
355 typical_acceptance_sampler.init_gpu_tensors(rank=0)
356 # Create a temperature zero target probability distribution and ensure
@@ -339,9 +363,11 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
363 high=vocab_size,
364 size=(batch_size, 1),
365 dtype=torch.int64)
342 - output_token_ids = typical_acceptance_sampler(target_probs,
343 - bonus_token_ids,
344 - draft_token_ids)
366 + output_token_ids = typical_acceptance_sampler(
367 + target_probs,
368 + bonus_token_ids,
369 + draft_probs=None,
370 + draft_token_ids=draft_token_ids)
371 assert output_token_ids.shape[0] == batch_size
372 assert output_token_ids.shape[1] == (k + 1)
373 assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
@@ -357,9 +383,11 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
383 batch_size, k, vocab_size, zero_temperature_token_ids)
384 draft_token_ids = torch.cat(
385 (draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
360 - output_token_ids = typical_acceptance_sampler(target_probs,
361 - bonus_token_ids,
362 - draft_token_ids)
386 + output_token_ids = typical_acceptance_sampler(
387 + target_probs,
388 + bonus_token_ids,
389 + draft_probs=None,
390 + draft_token_ids=draft_token_ids)
391 assert output_token_ids.shape[0] == batch_size
392 assert output_token_ids.shape[1] == (k + 1)
393 assert torch.all(output_token_ids[:, :2] == draft_token_ids[:, :2])
@@ -384,7 +412,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
412 batch_size = 1
413 vocab_size = 30_000
414 torch.set_default_device(device)
387 - typical_acceptance_sampler = TypicalAcceptanceSampler(
415 + typical_acceptance_sampler = get_acceptance_sampler(
416 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
417 typical_acceptance_sampler.init_gpu_tensors(rank=0)
418 # Simulate temperature 0 probability distribution for target
@@ -402,9 +430,11 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
430 high=vocab_size,
431 size=(batch_size, 1),
432 dtype=torch.int64)
405 - output_token_ids = typical_acceptance_sampler(target_probs,
406 - bonus_token_ids,
407 - draft_token_ids)
433 + output_token_ids = typical_acceptance_sampler(
434 + target_probs,
435 + bonus_token_ids,
436 + draft_probs=None,
437 + draft_token_ids=draft_token_ids)
438 assert output_token_ids.shape[0] == batch_size
439 assert output_token_ids.shape[1] == (k + 1)
440 assert torch.all(output_token_ids[:, 1:-1] == -1)
@@ -418,9 +448,11 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
448 posterior_threshold=0.0,
449 posterior_alpha=0.0)
450 typical_acceptance_sampler.init_gpu_tensors(rank=0)
421 - output_token_ids = typical_acceptance_sampler(target_probs,
422 - bonus_token_ids,
423 - draft_token_ids)
451 + output_token_ids = typical_acceptance_sampler(
452 + target_probs,
453 + bonus_token_ids,
454 + draft_probs=None,
455 + draft_token_ids=draft_token_ids)
456 assert output_token_ids.shape[0] == batch_size
457 assert output_token_ids.shape[1] == (k + 1)
458 assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
@@ -451,7 +483,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
483 batch_size = 5
484 vocab_size = 30_000
485 torch.set_default_device(device)
454 - typical_acceptance_sampler = TypicalAcceptanceSampler(
486 + typical_acceptance_sampler = get_acceptance_sampler(
487 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
488 typical_acceptance_sampler.init_gpu_tensors(rank=0)
489 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
tests/spec_decode/e2e/test_multistep_correctness.py
+53 -1
@@ -11,9 +11,15 @@ distribution matches the target model's output distribution (up to hardware
11 numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
12 equality. This gives us good coverage of temp=0.
13
14 +At temp=0, the TypicalAcceptanceSampler ensures that only the tokens with the
15 +highest probability in the target distribution are accepted. Therefore, we can
16 +expect greedy equality for the TypicalAcceptanceSampler at temp=0.
17 +
18 For temp>0, we rely on unit tests on the rejection sampler to verify that the
19 output distribution is the same with spec decode vs. no spec decode (this would
16 -be prohibitively expensive to run with a real model).
20 +be prohibitively expensive to run with a real model). Similarly, for the
21 +TypicalAcceptance sampler also, we rely on unit tests to validate temp>0
22 +test cases.
23
24 NOTE: Speculative decoding's distribution equality requires that the measured
25 distributions of the target model and proposal model be deterministic given the
@@ -611,3 +617,49 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
617 batch_size,
618 max_output_len=output_len,
619 force_output_len=True)
620 +
621 +
622 +@pytest.mark.parametrize(
623 + "common_llm_kwargs",
624 + [{
625 + "model": "JackFram/llama-160m",
626 +
627 + # Skip cuda graph recording for fast test.
628 + "enforce_eager": True,
629 +
630 + # Required for spec decode.
631 + "use_v2_block_manager": True
632 + }])
633 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
634 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
635 +@pytest.mark.parametrize(
636 + "test_llm_kwargs",
637 + [
638 + {
639 + "speculative_model": "JackFram/llama-68m",
640 + "num_speculative_tokens": k,
641 + "spec_decoding_acceptance_method": "typical_acceptance_sampler"
642 + }
643 + # Try a range of common k.
644 + for k in [1, 2, 3]
645 + ])
646 +@pytest.mark.parametrize("batch_size", [1, 32])
647 +@pytest.mark.parametrize(
648 + "output_len",
649 + [
650 + # Use smaller output len for fast test.
651 + 32,
652 + ])
653 +@pytest.mark.parametrize("seed", [1])
654 +def test_typical_acceptance_sampling(baseline_llm_generator,
655 + test_llm_generator, batch_size: int,
656 + output_len: int):
657 + """Verify that speculative decoding produces exact equality to without spec
658 + decode with TypicalAcceptanceSampler as the draft token acceptance
659 + sampling method.
660 + """
661 + run_greedy_equality_correctness_test(baseline_llm_generator,
662 + test_llm_generator,
663 + batch_size,
664 + max_output_len=output_len,
665 + force_output_len=True)
tests/spec_decode/test_dynamic_spec_decode.py
+7 -5
@@ -3,33 +3,35 @@ from unittest.mock import MagicMock, patch
3 import pytest
4 import torch
5
6 -from vllm.model_executor.layers.rejection_sampler import RejectionSampler
6 from vllm.sequence import ExecuteModelRequest
7 from vllm.spec_decode.metrics import AsyncMetricsCollector
8 from vllm.spec_decode.multi_step_worker import MultiStepWorker
9 from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
10 from vllm.spec_decode.top1_proposer import Top1Proposer
11
12 +from .test_utils import mock_spec_decode_sampler
13 from .utils import create_batch, mock_worker
14
15
16 @pytest.mark.parametrize('queue_size', [4])
17 @pytest.mark.parametrize('batch_size', [1])
18 @pytest.mark.parametrize('k', [1])
19 +@pytest.mark.parametrize("acceptance_sampler_method",
20 + ["rejection_sampler", "typical_acceptance_sampler"])
21 @torch.inference_mode()
20 -def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
22 +def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
23 + acceptance_sampler_method: str):
24 """Verify that speculative tokens are disabled when the batch size
25 exceeds the threshold.
26 """
27 disable_by_batch_size = 3
25 -
28 draft_worker = mock_worker(cls=MultiStepWorker)
29 target_worker = mock_worker()
28 - rejection_sampler = MagicMock(spec=RejectionSampler)
30 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
31 worker = SpecDecodeWorker(proposer_worker=draft_worker,
32 scorer_worker=target_worker,
32 - rejection_sampler=rejection_sampler,
33 + spec_decode_sampler=mock_spec_decode_sampler(
34 + acceptance_sampler_method),
35 metrics_collector=metrics_collector,
36 disable_by_batch_size=disable_by_batch_size)
37
tests/spec_decode/test_metrics.py
+47 -47
@@ -10,16 +10,16 @@ from vllm.spec_decode.metrics import AsyncMetricsCollector
10 def test_initial_call_returns_none():
11 """Expect first call to get metrics to return None.
12 """
13 - rej_sampler = MagicMock()
14 - rej_sampler.num_accepted_tokens = torch.tensor(0,
15 - dtype=torch.long,
16 - device='cuda')
17 - rej_sampler.num_emitted_tokens = torch.tensor(0,
18 - dtype=torch.long,
19 - device='cuda')
20 - rej_sampler.num_draft_tokens = 0
21 -
22 - collector = AsyncMetricsCollector(rej_sampler)
13 + spec_decode_sampler = MagicMock()
14 + spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
15 + dtype=torch.long,
16 + device='cuda')
17 + spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
18 + dtype=torch.long,
19 + device='cuda')
20 + spec_decode_sampler.num_draft_tokens = 0
21 +
22 + collector = AsyncMetricsCollector(spec_decode_sampler)
23 collector.init_gpu_tensors(rank=0)
24 maybe_metrics = collector.maybe_collect_rejsample_metrics(k=5)
25 assert maybe_metrics is None
@@ -28,14 +28,14 @@ def test_initial_call_returns_none():
28 def test_second_call_returns_metrics():
29 """Expect second call to not return None.
30 """
31 - rej_sampler = MagicMock()
32 - rej_sampler.num_accepted_tokens = torch.tensor(0,
33 - dtype=torch.long,
34 - device='cuda')
35 - rej_sampler.num_emitted_tokens = torch.tensor(0,
36 - dtype=torch.long,
37 - device='cuda')
38 - rej_sampler.num_draft_tokens = 0
31 + spec_decode_sampler = MagicMock()
32 + spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
33 + dtype=torch.long,
34 + device='cuda')
35 + spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
36 + dtype=torch.long,
37 + device='cuda')
38 + spec_decode_sampler.num_draft_tokens = 0
39
40 collect_interval_s = 5.0
41 timer = MagicMock()
@@ -43,7 +43,7 @@ def test_second_call_returns_metrics():
43 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
44 ]
45
46 - collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
46 + collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
47 timer=timer,
48 collect_interval_s=collect_interval_s)
49 collector.init_gpu_tensors(rank=0)
@@ -56,16 +56,16 @@ def test_second_call_returns_metrics():
56 def test_nonzero_rank_noop(rank):
57 """Verify nonzero ranks don't collect metrics.
58 """
59 - rej_sampler = MagicMock()
60 - rej_sampler.num_accepted_tokens = torch.tensor(0,
61 - dtype=torch.long,
62 - device='cuda')
63 - rej_sampler.num_emitted_tokens = torch.tensor(0,
64 - dtype=torch.long,
65 - device='cuda')
66 - rej_sampler.num_draft_tokens = 0
67 -
68 - collector = AsyncMetricsCollector(rej_sampler)
59 + spec_decode_sampler = MagicMock()
60 + spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
61 + dtype=torch.long,
62 + device='cuda')
63 + spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
64 + dtype=torch.long,
65 + device='cuda')
66 + spec_decode_sampler.num_draft_tokens = 0
67 +
68 + collector = AsyncMetricsCollector(spec_decode_sampler)
69 collector.init_gpu_tensors(rank=rank)
70 _ = collector.maybe_collect_rejsample_metrics(k=5)
71 metrics = collector.maybe_collect_rejsample_metrics(k=5)
@@ -75,14 +75,14 @@ def test_nonzero_rank_noop(rank):
75 def test_noop_until_time():
76 """Verify metrics aren't collected until enough time passes.
77 """
78 - rej_sampler = MagicMock()
79 - rej_sampler.num_accepted_tokens = torch.tensor(0,
80 - dtype=torch.long,
81 - device='cuda')
82 - rej_sampler.num_emitted_tokens = torch.tensor(0,
83 - dtype=torch.long,
84 - device='cuda')
85 - rej_sampler.num_draft_tokens = 0
78 + spec_decode_sampler = MagicMock()
79 + spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
80 + dtype=torch.long,
81 + device='cuda')
82 + spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
83 + dtype=torch.long,
84 + device='cuda')
85 + spec_decode_sampler.num_draft_tokens = 0
86
87 collect_interval_s = 5.0
88 timer = MagicMock()
@@ -91,7 +91,7 @@ def test_noop_until_time():
91 collect_interval_s + 0.1, collect_interval_s + 0.1
92 ]
93
94 - collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
94 + collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
95 timer=timer,
96 collect_interval_s=collect_interval_s)
97 collector.init_gpu_tensors(rank=0)
@@ -122,14 +122,14 @@ def test_initial_metrics_has_correct_values(has_data: bool):
122 max_num_emitted_tokens = AsyncMetricsCollector.get_max_num_emitted_tokens(
123 num_draft_tokens, k)
124
125 - rej_sampler = MagicMock()
126 - rej_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127 - dtype=torch.long,
128 - device='cuda')
129 - rej_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130 - dtype=torch.long,
131 - device='cuda')
132 - rej_sampler.num_draft_tokens = num_draft_tokens
125 + spec_decode_sampler = MagicMock()
126 + spec_decode_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127 + dtype=torch.long,
128 + device='cuda')
129 + spec_decode_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130 + dtype=torch.long,
131 + device='cuda')
132 + spec_decode_sampler.num_draft_tokens = num_draft_tokens
133
134 collect_interval_s = 5.0
135 timer = MagicMock()
@@ -137,7 +137,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
137 0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
138 ]
139
140 - collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
140 + collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
141 timer=timer,
142 collect_interval_s=collect_interval_s)
143 collector.init_gpu_tensors(rank=0)
tests/spec_decode/test_spec_decode_worker.py
+85 -69
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock
6 import pytest
7 import torch
8
9 -from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9 from vllm.model_executor.utils import set_random_seed
10 from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
11 from vllm.spec_decode.interfaces import SpeculativeProposals
@@ -16,23 +15,26 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
15 from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
16 split_num_cache_blocks_evenly)
17
18 +from .test_utils import mock_spec_decode_sampler
19 from .utils import create_batch, create_sampler_output_list, mock_worker
20
21
22 @pytest.mark.parametrize('k', [1, 2, 6])
23 @pytest.mark.parametrize('batch_size', [1, 2, 32])
24 +@pytest.mark.parametrize("acceptance_sampler_method",
25 + ["rejection_sampler", "typical_acceptance_sampler"])
26 @torch.inference_mode()
25 -def test_correctly_calls_draft_model(k: int, batch_size: int):
27 +def test_correctly_calls_draft_model(k: int, batch_size: int,
28 + acceptance_sampler_method: str):
29 """Verify SpecDecodeWorker calls the draft worker with correct
30 inputs. Everything else is mocked out.
31 """
32 draft_worker = mock_worker(cls=MultiStepWorker)
33 target_worker = mock_worker()
31 - rejection_sampler = MagicMock(spec=RejectionSampler)
34 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
33 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
34 - metrics_collector)
35 -
35 + worker = SpecDecodeWorker(
36 + draft_worker, target_worker,
37 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
38 exception_secret = 'artificial stop'
39 draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
40
@@ -53,15 +55,16 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
55
56 @pytest.mark.parametrize('k', [1, 2, 6])
57 @pytest.mark.parametrize('batch_size', [1, 2, 32])
58 +@pytest.mark.parametrize("acceptance_sampler_method",
59 + ["rejection_sampler", "typical_acceptance_sampler"])
60 @torch.inference_mode()
57 -def test_correctly_calls_target_model(k: int, batch_size: int):
61 +def test_correctly_calls_target_model(k: int, batch_size: int,
62 + acceptance_sampler_method: str):
63 """Verify SpecDecodeWorker calls the target model with correct
64 inputs. Everything else is mocked out.
65 """
66 draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
67 target_worker = mock_worker(use_spec=False)
63 - rejection_sampler = MagicMock(spec=RejectionSampler)
64 - rejection_sampler.token_id_dtype = torch.int64
68 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
69
70 draft_worker.device = 'cuda'
@@ -69,8 +72,9 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
72
73 set_random_seed(1)
74
72 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
73 - metrics_collector)
75 + worker = SpecDecodeWorker(
76 + draft_worker, target_worker,
77 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
78 worker.init_device()
79
80 vocab_size = 32_000
@@ -133,8 +137,11 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
137
138 @pytest.mark.parametrize('k', [1, 2, 6])
139 @pytest.mark.parametrize('batch_size', [1, 2, 32])
140 +@pytest.mark.parametrize("acceptance_sampler_method",
141 + ["rejection_sampler", "typical_acceptance_sampler"])
142 @torch.inference_mode()
137 -def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
143 +def test_correctly_calls_spec_decode_sampler(k: int, batch_size: int,
144 + acceptance_sampler_method: str):
145 """Verify SpecDecodeWorker calls the rejection sampler with
146 correct inputs. Everything else is mocked out.
147 """
@@ -144,15 +151,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
151 vocab_size=vocab_size,
152 use_spec=False)
153 target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
147 - rejection_sampler = MagicMock(spec=RejectionSampler)
148 - rejection_sampler.token_id_dtype = torch.int64
154 + spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
155 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
156 draft_worker.device = 'cuda'
157 target_worker.device = 'cuda'
158
159 set_random_seed(1)
160
155 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
161 + worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
162 metrics_collector)
163 worker.init_device()
164
@@ -199,15 +205,16 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
205 target_worker.execute_model.return_value = [target_output[0]]
206
207 exception_secret = 'artificial stop'
202 - rejection_sampler.side_effect = ValueError(exception_secret)
208 +
209 + spec_decode_sampler.side_effect = ValueError(exception_secret)
210
211 with pytest.raises(ValueError, match=exception_secret):
212 worker.execute_model(execute_model_req=ExecuteModelRequest(
213 seq_group_metadata_list=seq_group_metadata_list,
214 num_lookahead_slots=k))
215
209 - assert len(rejection_sampler.call_args_list) == 1
210 - _, kwargs = rejection_sampler.call_args_list[0]
216 + assert len(spec_decode_sampler.call_args_list) == 1
217 + _, kwargs = spec_decode_sampler.call_args_list[0]
218 actual = SimpleNamespace(**kwargs)
219
220 assert torch.equal(actual.bonus_token_ids,
@@ -221,8 +228,11 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
228
229 @pytest.mark.parametrize('k', [1, 2, 6])
230 @pytest.mark.parametrize('batch_size', [1, 2, 32])
231 +@pytest.mark.parametrize("acceptance_sampler_method",
232 + ["rejection_sampler", "typical_acceptance_sampler"])
233 @torch.inference_mode()
225 -def test_correctly_formats_output(k: int, batch_size: int):
234 +def test_correctly_formats_output(k: int, batch_size: int,
235 + acceptance_sampler_method: str):
236 """Verify SpecDecodeWorker formats sampler output correctly.
237 Everything else is mocked out.
238 """
@@ -232,15 +242,13 @@ def test_correctly_formats_output(k: int, batch_size: int):
242 vocab_size=vocab_size,
243 use_spec=False)
244 target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
235 - rejection_sampler = MagicMock(spec=RejectionSampler)
236 - rejection_sampler.token_id_dtype = torch.int64
245 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
246 draft_worker.device = 'cuda'
247 target_worker.device = 'cuda'
248
249 set_random_seed(1)
242 -
243 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
250 + spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
251 + worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
252 metrics_collector)
253 worker.init_device()
254
@@ -286,24 +294,23 @@ def test_correctly_formats_output(k: int, batch_size: int):
294
295 target_worker.execute_model.return_value = [target_output[0]]
296
289 - rejection_sampler_output = torch.randint(low=0,
290 - high=vocab_size,
291 - size=(batch_size, k + 1),
292 - dtype=torch.int64,
293 - device='cuda')
297 + spec_decode_sampler_output = torch.randint(low=0,
298 + high=vocab_size,
299 + size=(batch_size, k + 1),
300 + dtype=torch.int64,
301 + device='cuda')
302 for i in range(batch_size):
303 minimum_accepted_tokens = 1
296 - rejection_sampler_output[i][
304 + spec_decode_sampler_output[i][
305 -random.randint(minimum_accepted_tokens, k + 1):] = -1
306
299 - rejection_sampler.return_value = rejection_sampler_output
300 -
307 + spec_decode_sampler.return_value = spec_decode_sampler_output
308 output = worker.execute_model(execute_model_req=ExecuteModelRequest(
309 seq_group_metadata_list=seq_group_metadata_list,
310 num_lookahead_slots=k))
311
312 expected_output = create_sampler_output_list(
306 - token_ids=rejection_sampler_output.transpose(0, 1),
313 + token_ids=spec_decode_sampler_output.transpose(0, 1),
314 probs=[None for _ in range(k + 1)],
315 logprobs=[None for _ in range(k + 1)])
316
@@ -350,8 +357,11 @@ def test_correctly_formats_output(k: int, batch_size: int):
357 @pytest.mark.parametrize('k', [1, 2])
358 @pytest.mark.parametrize('batch_size', [1])
359 @pytest.mark.parametrize('returns_metrics', [True, False])
360 +@pytest.mark.parametrize("acceptance_sampler_method",
361 + ["rejection_sampler", "typical_acceptance_sampler"])
362 @torch.inference_mode()
354 -def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
363 +def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
364 + acceptance_sampler_method: str):
365 """Verify SpecDecodeWorker collects metrics.
366 """
367 vocab_size = 32_000
@@ -360,15 +370,14 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
370 vocab_size=vocab_size,
371 use_spec=False)
372 target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
363 - rejection_sampler = MagicMock(spec=RejectionSampler)
364 - rejection_sampler.token_id_dtype = torch.int64
373 + spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
374 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
375 draft_worker.device = 'cuda'
376 target_worker.device = 'cuda'
377
378 set_random_seed(1)
379
371 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
380 + worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
381 metrics_collector)
382 worker.init_device()
383
@@ -414,17 +423,16 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
423
424 target_worker.execute_model.return_value = [target_output[0]]
425
417 - rejection_sampler_output = torch.randint(low=0,
418 - high=vocab_size,
419 - size=(batch_size, k + 1),
420 - dtype=torch.int64,
421 - device='cuda')
426 + spec_decode_sampler_output = torch.randint(low=0,
427 + high=vocab_size,
428 + size=(batch_size, k + 1),
429 + dtype=torch.int64,
430 + device='cuda')
431 for i in range(batch_size):
432 minimum_accepted_tokens = 1
424 - rejection_sampler_output[i][
433 + spec_decode_sampler_output[i][
434 -random.randint(minimum_accepted_tokens, k + 1):] = -1
426 -
427 - rejection_sampler.return_value = rejection_sampler_output
435 + spec_decode_sampler.return_value = spec_decode_sampler_output
436
437 mock_rejsample_metrics = MagicMock(
438 spec=SpecDecodeWorkerMetrics) if returns_metrics else None
@@ -445,15 +453,16 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
453
454 @pytest.mark.parametrize('k', [0])
455 @pytest.mark.parametrize('batch_size', [1, 2, 32])
456 +@pytest.mark.parametrize("acceptance_sampler_method",
457 + ["rejection_sampler", "typical_acceptance_sampler"])
458 @torch.inference_mode()
449 -def test_k_equals_zero(k: int, batch_size: int):
459 +def test_k_equals_zero(k: int, batch_size: int,
460 + acceptance_sampler_method: str):
461 """Verify that the SpecDecodeWorker calls the draft and target workers
462 when k is zero. This happens during prefill.
463 """
464 draft_worker = mock_worker(cls=MultiStepWorker)
465 target_worker = mock_worker()
455 - rejection_sampler = MagicMock(spec=RejectionSampler)
456 - rejection_sampler.token_id_dtype = torch.int64
466 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
467
468 sampler_output = MagicMock(spec=SamplerOutput)
@@ -465,8 +474,9 @@ def test_k_equals_zero(k: int, batch_size: int):
474
475 set_random_seed(1)
476
468 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
469 - metrics_collector)
477 + worker = SpecDecodeWorker(
478 + draft_worker, target_worker,
479 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
480
481 seq_group_metadata_list, _, _ = create_batch(batch_size,
482 k,
@@ -487,16 +497,17 @@ def test_k_equals_zero(k: int, batch_size: int):
497
498 @pytest.mark.parametrize('k', [0, 5])
499 @pytest.mark.parametrize('batch_size', [0])
500 +@pytest.mark.parametrize("acceptance_sampler_method",
501 + ["rejection_sampler", "typical_acceptance_sampler"])
502 @torch.inference_mode()
491 -def test_empty_input_batch(k: int, batch_size: int):
503 +def test_empty_input_batch(k: int, batch_size: int,
504 + acceptance_sampler_method: str):
505 """Verify that the SpecDecodeWorker calls the draft and target workers
506 when the input batch is empty. This can happen if the engine communicates
507 to the workers information without scheduling a batch.
508 """
509 draft_worker = mock_worker(cls=MultiStepWorker)
510 target_worker = mock_worker()
498 - rejection_sampler = MagicMock(spec=RejectionSampler)
499 - rejection_sampler.token_id_dtype = torch.int64
511 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
512
513 sampler_output = MagicMock(spec=SamplerOutput)
@@ -508,8 +519,9 @@ def test_empty_input_batch(k: int, batch_size: int):
519
520 set_random_seed(1)
521
511 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
512 - metrics_collector)
522 + worker = SpecDecodeWorker(
523 + draft_worker, target_worker,
524 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
525
526 seq_group_metadata_list, _, _ = create_batch(batch_size,
527 k,
@@ -528,18 +540,19 @@ def test_empty_input_batch(k: int, batch_size: int):
540 target_worker.execute_model.assert_called_once_with(execute_model_req)
541
542
543 +@pytest.mark.parametrize("acceptance_sampler_method",
544 + ["rejection_sampler", "typical_acceptance_sampler"])
545 @pytest.mark.skip_global_cleanup
532 -def test_init_device():
546 +def test_init_device(acceptance_sampler_method: str):
547 """Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
548 well as other GPU initialization.
549 """
550 draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
551 target_worker = mock_worker(use_spec=False)
538 - rejection_sampler = MagicMock(spec=RejectionSampler)
539 - rejection_sampler.token_id_dtype = torch.int64
552 + spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
553 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
554
542 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
555 + worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
556 metrics_collector)
557
558 worker.init_device()
@@ -549,22 +562,23 @@ def test_init_device():
562 target_worker.init_device.assert_called_once()
563
564 metrics_collector.init_gpu_tensors.assert_called_once()
552 - rejection_sampler.init_gpu_tensors.assert_called_once()
565 + spec_decode_sampler.init_gpu_tensors.assert_called_once()
566
567
568 +@pytest.mark.parametrize("acceptance_sampler_method",
569 + ["rejection_sampler", "typical_acceptance_sampler"])
570 @torch.inference_mode()
556 -def test_initialize_cache():
571 +def test_initialize_cache(acceptance_sampler_method):
572 """Verify SpecDecodeWorker invokes initialize_cache on proposer/scorer
573 workers.
574 """
575 draft_worker = mock_worker(cls=MultiStepWorker)
576 target_worker = mock_worker()
562 - rejection_sampler = MagicMock(spec=RejectionSampler)
563 - rejection_sampler.token_id_dtype = torch.int64
577 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
578
566 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
567 - metrics_collector)
579 + worker = SpecDecodeWorker(
580 + draft_worker, target_worker,
581 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
582
583 kwargs = {"num_gpu_blocks": 1024, "num_cpu_blocks": 1023}
584 worker.initialize_cache(**kwargs)
@@ -577,19 +591,20 @@ def test_initialize_cache():
591 @pytest.mark.parametrize('available_cpu_blocks', [500])
592 @pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
593 @pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
594 +@pytest.mark.parametrize("acceptance_sampler_method",
595 + ["rejection_sampler", "typical_acceptance_sampler"])
596 @pytest.mark.skip_global_cleanup
597 def test_determine_num_available_blocks(available_gpu_blocks: int,
598 available_cpu_blocks: int,
599 target_cache_block_size_bytes: int,
584 - draft_kv_size_bytes: int):
600 + draft_kv_size_bytes: int,
601 + acceptance_sampler_method: str):
602 """Verify SpecDecodeWorker correctly profiles num available GPU blocks.
603 Specifically, it should run profiling in the scorer worker, and then evenly
604 split the blocks between proposer and scorer worker.
605 """
606 draft_worker = mock_worker(cls=MultiStepWorker)
607 target_worker = mock_worker()
591 - rejection_sampler = MagicMock(spec=RejectionSampler)
592 - rejection_sampler.token_id_dtype = torch.int64
608 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
609
610 target_worker.determine_num_available_blocks.return_value = (
@@ -598,8 +613,9 @@ def test_determine_num_available_blocks(available_gpu_blocks: int,
613 target_cache_block_size_bytes)
614 draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
615
601 - worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
602 - metrics_collector)
616 + worker = SpecDecodeWorker(
617 + draft_worker, target_worker,
618 + mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
619
620 num_gpu_blocks, num_cpu_blocks = worker.determine_num_available_blocks()
621
tests/spec_decode/test_utils.py
+22
@@ -1,7 +1,11 @@
1 from unittest.mock import MagicMock
2
3 import pytest
4 +import torch
5
6 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 +from vllm.model_executor.layers.typical_acceptance_sampler import (
8 + TypicalAcceptanceSampler)
9 from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
10 from vllm.spec_decode.util import split_batch_by_proposal_len
11
@@ -109,3 +113,21 @@ def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
113
114 assert filtered_groups == []
115 assert indices == []
116 +
117 +
118 +def mock_spec_decode_sampler(acceptance_sampler_method):
119 + """
120 + Returns either a RejectionSampler or TypicalAcceptanceSampler
121 + object depending on whether acceptance_sampler_method is
122 + 'rejection_sampler' or 'typical_acceptance_sampler' respectively.
123 + """
124 + if acceptance_sampler_method == "rejection_sampler":
125 + sampler = MagicMock(spec=RejectionSampler)
126 + sampler.token_id_dtype = torch.int64
127 + return sampler
128 + elif acceptance_sampler_method == "typical_acceptance_sampler":
129 + sampler = MagicMock(spec=TypicalAcceptanceSampler)
130 + sampler.token_id_dtype = torch.int64
131 + return sampler
132 + else:
133 + raise ValueError(f"Invalid sampler name {acceptance_sampler_method}")
vllm/config.py
+73 -2
@@ -753,7 +753,6 @@ class SchedulerConfig:
753 self.chunked_prefill_enabled = enable_chunked_prefill
754 self.embedding_mode = embedding_mode
755 self.preemption_mode = preemption_mode
756 -
756 self._verify_args()
757
758 def _verify_args(self) -> None:
@@ -834,6 +833,9 @@ class SpeculativeConfig:
833 speculative_disable_by_batch_size: Optional[int],
834 ngram_prompt_lookup_max: Optional[int],
835 ngram_prompt_lookup_min: Optional[int],
836 + draft_token_acceptance_method: str,
837 + typical_acceptance_sampler_posterior_threshold: Optional[float],
838 + typical_acceptance_sampler_posterior_alpha: Optional[float],
839 ) -> Optional["SpeculativeConfig"]:
840 """Create a SpeculativeConfig if possible, else return None.
841
@@ -870,7 +872,20 @@ class SpeculativeConfig:
872 window, if provided.
873 ngram_prompt_lookup_min (Optional[int]): Min size of ngram token
874 window, if provided.
873 -
875 + draft_token_acceptance_method (str): The method to use for
876 + accepting draft tokens. This can take two possible
877 + values 'rejection_sampler' and 'typical_acceptance_sampler'
878 + for RejectionSampler and TypicalAcceptanceSampler
879 + respectively.
880 + typical_acceptance_sampler_posterior_threshold (Optional[float]):
881 + A threshold value that sets a lower bound on the posterior
882 + probability of a token in the target model for it to be
883 + accepted. This threshold is used only when we use the
884 + TypicalAcceptanceSampler for token acceptance.
885 + typical_acceptance_sampler_posterior_alpha (Optional[float]):
886 + A scaling factor for the entropy-based threshold in the
887 + TypicalAcceptanceSampler.
888 +
889 Returns:
890 Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
891 the necessary conditions are met, else None.
@@ -984,6 +999,11 @@ class SpeculativeConfig:
999 "speculative_model unless the draft model config contains an "
1000 "n_predict parameter.")
1001
1002 + if typical_acceptance_sampler_posterior_threshold is None:
1003 + typical_acceptance_sampler_posterior_threshold = 0.09
1004 + if typical_acceptance_sampler_posterior_alpha is None:
1005 + typical_acceptance_sampler_posterior_alpha = 0.3
1006 +
1007 return SpeculativeConfig(
1008 draft_model_config,
1009 draft_parallel_config,
@@ -991,6 +1011,11 @@ class SpeculativeConfig:
1011 speculative_disable_by_batch_size,
1012 ngram_prompt_lookup_max,
1013 ngram_prompt_lookup_min,
1014 + draft_token_acceptance_method=draft_token_acceptance_method,
1015 + typical_acceptance_sampler_posterior_threshold=\
1016 + typical_acceptance_sampler_posterior_threshold,
1017 + typical_acceptance_sampler_posterior_alpha=\
1018 + typical_acceptance_sampler_posterior_alpha,
1019 )
1020
1021 @staticmethod
@@ -1072,6 +1097,9 @@ class SpeculativeConfig:
1097 speculative_disable_by_batch_size: Optional[int],
1098 ngram_prompt_lookup_max: Optional[int],
1099 ngram_prompt_lookup_min: Optional[int],
1100 + draft_token_acceptance_method: str,
1101 + typical_acceptance_sampler_posterior_threshold: float,
1102 + typical_acceptance_sampler_posterior_alpha: float,
1103 ):
1104 """Create a SpeculativeConfig object.
1105
@@ -1085,6 +1113,19 @@ class SpeculativeConfig:
1113 enqueue requests is larger than this value.
1114 ngram_prompt_lookup_max: Max size of ngram token window.
1115 ngram_prompt_lookup_min: Min size of ngram token window.
1116 + draft_token_acceptance_method (str): The method to use for
1117 + accepting draft tokens. This can take two possible
1118 + values 'rejection_sampler' and 'typical_acceptance_sampler'
1119 + for RejectionSampler and TypicalAcceptanceSampler
1120 + respectively.
1121 + typical_acceptance_sampler_posterior_threshold (Optional[float]):
1122 + A threshold value that sets a lower bound on the posterior
1123 + probability of a token in the target model for it to be
1124 + accepted. This threshold is used only when we use the
1125 + TypicalAcceptanceSampler for token acceptance.
1126 + typical_acceptance_sampler_posterior_alpha (Optional[float]):
1127 + A scaling factor for the entropy-based threshold in the
1128 + TypicalAcceptanceSampler.
1129 """
1130 self.draft_model_config = draft_model_config
1131 self.draft_parallel_config = draft_parallel_config
@@ -1093,6 +1134,11 @@ class SpeculativeConfig:
1134 speculative_disable_by_batch_size
1135 self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0
1136 self.ngram_prompt_lookup_min = ngram_prompt_lookup_min or 0
1137 + self.draft_token_acceptance_method = draft_token_acceptance_method
1138 + self.typical_acceptance_sampler_posterior_threshold = \
1139 + typical_acceptance_sampler_posterior_threshold
1140 + self.typical_acceptance_sampler_posterior_alpha = \
1141 + typical_acceptance_sampler_posterior_alpha
1142
1143 self._verify_args()
1144
@@ -1104,6 +1150,31 @@ class SpeculativeConfig:
1150 if self.draft_model_config:
1151 self.draft_model_config.verify_with_parallel_config(
1152 self.draft_parallel_config)
1153 + # Validate and set draft token acceptance related settings.
1154 +
1155 + if (self.draft_token_acceptance_method is None):
1156 + raise ValueError("draft_token_acceptance_method is not set. "
1157 + "Expected values are rejection_sampler or "
1158 + "typical_acceptance_sampler.")
1159 +
1160 + if (self.draft_token_acceptance_method != 'rejection_sampler'
1161 + and self.draft_token_acceptance_method !=
1162 + 'typical_acceptance_sampler'):
1163 + raise ValueError(
1164 + "Expected draft_token_acceptance_method to be either "
1165 + "rejection_sampler or typical_acceptance_sampler. Instead it "
1166 + f"is {self.draft_token_acceptance_method}")
1167 +
1168 + if (self.typical_acceptance_sampler_posterior_threshold < 0
1169 + or self.typical_acceptance_sampler_posterior_alpha < 0):
1170 + raise ValueError(
1171 + "Expected typical_acceptance_sampler_posterior_threshold "
1172 + "and typical_acceptance_sampler_posterior_alpha to be > 0. "
1173 + "Instead found "
1174 + f"typical_acceptance_sampler_posterior_threshold = "
1175 + f"{self.typical_acceptance_sampler_posterior_threshold} and "
1176 + f"typical_acceptance_sampler_posterior_alpha = "
1177 + f"{self.typical_acceptance_sampler_posterior_alpha}")
1178
1179 @property
1180 def num_lookahead_slots(self) -> int:
vllm/engine/arg_utils.py
+41 -1
@@ -100,7 +100,9 @@ class EngineArgs:
100 speculative_disable_by_batch_size: Optional[int] = None
101 ngram_prompt_lookup_max: Optional[int] = None
102 ngram_prompt_lookup_min: Optional[int] = None
103 -
103 + spec_decoding_acceptance_method: str = 'rejection_sampler'
104 + typical_acceptance_sampler_posterior_threshold: Optional[float] = None
105 + typical_acceptance_sampler_posterior_alpha: Optional[float] = None
106 qlora_adapter_name_or_path: Optional[str] = None
107
108 otlp_traces_endpoint: Optional[str] = None
@@ -577,6 +579,38 @@ class EngineArgs:
579 help='Min size of window for ngram prompt lookup in speculative '
580 'decoding.')
581
582 + parser.add_argument(
583 + '--spec-decoding-acceptance-method',
584 + type=str,
585 + default=EngineArgs.spec_decoding_acceptance_method,
586 + choices=['rejection_sampler', 'typical_acceptance_sampler'],
587 + help='Specify the acceptance method to use during draft token '
588 + 'verification in speculative decoding. Two types of acceptance '
589 + 'routines are supported: '
590 + '1) RejectionSampler which does not allow changing the '
591 + 'acceptance rate of draft tokens, '
592 + '2) TypicalAcceptanceSampler which is configurable, allowing for '
593 + 'a higher acceptance rate at the cost of lower quality, '
594 + 'and vice versa.')
595 +
596 + parser.add_argument(
597 + '--typical-acceptance-sampler-posterior-threshold',
598 + type=float,
599 + default=EngineArgs.typical_acceptance_sampler_posterior_threshold,
600 + help='Set the lower bound threshold for the posterior '
601 + 'probability of a token to be accepted. This threshold is '
602 + 'used by the TypicalAcceptanceSampler to make sampling decisions '
603 + 'during speculative decoding. Defaults to 0.09')
604 +
605 + parser.add_argument(
606 + '--typical-acceptance-sampler-posterior-alpha',
607 + type=float,
608 + default=EngineArgs.typical_acceptance_sampler_posterior_alpha,
609 + help='A scaling factor for the entropy-based threshold for token '
610 + 'acceptance in the TypicalAcceptanceSampler. Typically defaults '
611 + 'to sqrt of --typical-acceptance-sampler-posterior-threshold '
612 + 'i.e. 0.3')
613 +
614 parser.add_argument('--model-loader-extra-config',
615 type=nullable_str,
616 default=EngineArgs.model_loader_extra_config,
@@ -737,6 +771,12 @@ class EngineArgs:
771 use_v2_block_manager=self.use_v2_block_manager,
772 ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
773 ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
774 + draft_token_acceptance_method=\
775 + self.spec_decoding_acceptance_method,
776 + typical_acceptance_sampler_posterior_threshold=self.
777 + typical_acceptance_sampler_posterior_threshold,
778 + typical_acceptance_sampler_posterior_alpha=self.
779 + typical_acceptance_sampler_posterior_alpha,
780 )
781
782 scheduler_config = SchedulerConfig(
vllm/engine/metrics.py
+1 -1
@@ -457,4 +457,4 @@ class PrometheusStatLogger(StatLoggerBase):
457
458 class RayPrometheusStatLogger(PrometheusStatLogger):
459 """RayPrometheusStatLogger uses Ray metrics instead."""
460 - _metrics_cls = RayMetrics
460 + _metrics_cls = RayMetrics
\ No newline at end of file
vllm/model_executor/layers/rejection_sampler.py
+9 -9
@@ -3,13 +3,12 @@ from typing import Tuple
3
4 import torch
5 import torch.jit
6 -import torch.nn as nn
6
7 from vllm.model_executor.layers.spec_decode_base_sampler import (
8 SpecDecodeBaseSampler)
9
10
12 -class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
11 +class RejectionSampler(SpecDecodeBaseSampler):
12 """Apply modified rejection sampling as described in "Accelerating Large
13 Language Model Decoding with Speculative Sampling"
14 https://arxiv.org/pdf/2302.01318.pdf.
@@ -28,8 +27,8 @@ class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
27 during sampling. This catches correctness issues but adds
28 nontrivial latency.
29 """
31 - SpecDecodeBaseSampler.__init__(self, disable_bonus_tokens, strict_mode)
32 - nn.Module.__init__(self)
30 + super().__init__(disable_bonus_tokens=disable_bonus_tokens,
31 + strict_mode=strict_mode)
32
33 def forward(
34 self,
@@ -78,11 +77,12 @@ class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
77 self._raise_if_incorrect_input(target_probs, bonus_token_ids,
78 draft_probs, draft_token_ids)
79
81 - accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
82 - target_probs,
83 - draft_probs,
84 - draft_token_ids,
85 - )
80 + accepted, recovered_token_ids = (
81 + self._batch_modified_rejection_sampling(
82 + target_probs,
83 + draft_probs,
84 + draft_token_ids,
85 + ))
86
87 output_token_ids = self._create_output(
88 accepted,
vllm/model_executor/layers/spec_decode_base_sampler.py
+14 -1
@@ -1,9 +1,12 @@
1 +from abc import abstractmethod
2 from typing import Optional
3
4 import torch
5 +import torch.jit
6 +import torch.nn as nn
7
8
6 -class SpecDecodeBaseSampler():
9 +class SpecDecodeBaseSampler(nn.Module):
10 """Base class for samplers used for Speculative Decoding verification
11 step.
12 """
@@ -51,6 +54,16 @@ class SpecDecodeBaseSampler():
54 def token_id_dtype(self):
55 return torch.int64
56
57 + @abstractmethod
58 + def forward(
59 + self,
60 + target_probs: torch.Tensor,
61 + bonus_token_ids: torch.Tensor,
62 + draft_probs: torch.Tensor,
63 + draft_token_ids: torch.Tensor,
64 + ) -> torch.Tensor:
65 + raise NotImplementedError
66 +
67 def _create_output(
68 self,
69 accepted: torch.Tensor, # [batch_size, k]
vllm/model_executor/layers/typical_acceptance_sampler.py
+10 -12
@@ -1,12 +1,11 @@
1 import torch
2 import torch.jit
3 -import torch.nn as nn
3
4 from vllm.model_executor.layers.spec_decode_base_sampler import (
5 SpecDecodeBaseSampler)
6
7
9 -class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
8 +class TypicalAcceptanceSampler(SpecDecodeBaseSampler):
9 """Apply typical acceptance sampling as described in section 3.3.1 in
10 "MEDUSA: Simple LLM Inference Acceleration Framework with
11 Multiple Decoding Heads"
@@ -15,10 +14,10 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
14
15 def __init__(
16 self,
17 + posterior_threshold: float,
18 + posterior_alpha: float,
19 disable_bonus_tokens: bool = False,
20 strict_mode: bool = False,
20 - posterior_threshold: float = 0.09,
21 - posterior_alpha: float = 0.3,
21 ):
22 """Create a Typical Acceptance Sampler.
23
@@ -31,23 +30,20 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
30 nontrivial latency.
31 posterior_threshold : A threshold value that sets a lower bound
32 on the posterior probability of a token in target model for it
34 - to be accepted. Default is 0.09
33 + to be accepted.
34 posterior_alpha : A scaling factor for the entropy-based
36 - threshold in typical acceptance sampling. Typically defaults to
37 - sqrt of posterior_threshold and is set to 0.3.
35 + threshold in typical acceptance sampling.
36 """
39 - SpecDecodeBaseSampler.__init__(
40 - self,
41 - disable_bonus_tokens=disable_bonus_tokens,
42 - strict_mode=strict_mode)
43 - nn.Module.__init__(self)
37 self._posterior_threshold = posterior_threshold
38 self._posterior_alpha = posterior_alpha
39 + super().__init__(disable_bonus_tokens=disable_bonus_tokens,
40 + strict_mode=strict_mode)
41
42 def forward(
43 self,
44 target_probs: torch.Tensor,
45 bonus_token_ids: torch.Tensor,
46 + draft_probs: torch.Tensor,
47 draft_token_ids: torch.Tensor,
48 ) -> torch.Tensor:
49 """Sample token ids using typical acceptance sampling. This accepts
@@ -69,6 +65,8 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
65 speculative tokens in a sequence are accepted.
66 shape = [batch_size, num_bonus_tokens]
67
68 + draft_probs: This parameter is unused by the acceptance sampler.
69 +
70 draft_token_ids: The token ids that were sampled from the draft
71 probabilities.
72 shape = [batch_size, num_speculative_tokens]
vllm/spec_decode/metrics.py
+13 -11
@@ -4,7 +4,8 @@ from typing import Callable, Optional
4
5 import torch
6
7 -from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 +from vllm.model_executor.layers.spec_decode_base_sampler import (
8 + SpecDecodeBaseSampler)
9 from vllm.utils import is_pin_memory_available
10
11
@@ -46,15 +47,15 @@ Timer = Callable[[], float]
47
48
49 class AsyncMetricsCollector:
49 - """Class which copies rejection sampler metrics from the device to CPU on a
50 - non-default Torch stream.
50 + """Class which copies rejection/typical-acceptance sampler metrics
51 + from the device to CPU on a non-default Torch stream.
52 """
53
54 def __init__(self,
54 - rejection_sampler: RejectionSampler,
55 + spec_decode_sampler: SpecDecodeBaseSampler,
56 timer: Optional[Timer] = None,
57 collect_interval_s: float = 5.0):
57 - self._rejection_sampler = rejection_sampler
58 + self.spec_decode_sampler = spec_decode_sampler
59 self._timer = time.time if timer is None else timer
60
61 self._rank: Optional[int] = None
@@ -95,7 +96,7 @@ class AsyncMetricsCollector:
96 return None
97
98 def _should_collect_rejsample_metrics(self, now: float) -> bool:
98 - """Return whether or not this iteration should print rejection sampling
99 + """Return whether or not this iteration should print sampling
100 metrics.
101 """
102 if self._rank != 0:
@@ -107,8 +108,8 @@ class AsyncMetricsCollector:
108 return True
109
110 def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
110 - """Copy rejection sampling metrics (number of accepted tokens, etc) to
111 - CPU asynchronously.
111 + """Copy rejection/typical-acceptance sampling metrics
112 + (number of accepted tokens, etc) to CPU asynchronously.
113
114 Returns a CUDA event recording when the copy is complete.
115 """
@@ -117,13 +118,14 @@ class AsyncMetricsCollector:
118
119 with torch.cuda.stream(self._copy_stream):
120 self._aggregate_num_accepted_tokens.copy_(
120 - self._rejection_sampler.num_accepted_tokens, non_blocking=True)
121 + self.spec_decode_sampler.num_accepted_tokens,
122 + non_blocking=True)
123 self._aggregate_num_emitted_tokens.copy_(
122 - self._rejection_sampler.num_emitted_tokens, non_blocking=True)
124 + self.spec_decode_sampler.num_emitted_tokens, non_blocking=True)
125 # Number of draft tokens is calculated on CPU, so no copy is
126 # required.
127 self._aggregate_num_draft_tokens = (
126 - self._rejection_sampler.num_draft_tokens)
128 + self.spec_decode_sampler.num_draft_tokens)
129
130 aggregate_metrics_ready = torch.cuda.Event()
131 aggregate_metrics_ready.record(self._copy_stream)
vllm/spec_decode/spec_decode_worker.py
+43 -19
@@ -7,6 +7,10 @@ from vllm.config import ParallelConfig, SpeculativeConfig
7 from vllm.distributed.communication_op import broadcast_tensor_dict
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10 +from vllm.model_executor.layers.spec_decode_base_sampler import (
11 + SpecDecodeBaseSampler)
12 +from vllm.model_executor.layers.typical_acceptance_sampler import (
13 + TypicalAcceptanceSampler)
14 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
15 HiddenStates, SamplerOutput, SequenceGroupMetadata,
16 get_all_seq_ids)
@@ -56,7 +60,12 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
60 draft_worker_kwargs=draft_worker_kwargs,
61 disable_by_batch_size=speculative_config.
62 speculative_disable_by_batch_size,
59 - )
63 + draft_token_acceptance_method=speculative_config.
64 + draft_token_acceptance_method,
65 + typical_acceptance_sampler_posterior_threshold=speculative_config.
66 + typical_acceptance_sampler_posterior_threshold,
67 + typical_acceptance_sampler_posterior_alpha=speculative_config.
68 + typical_acceptance_sampler_posterior_alpha)
69
70 return spec_decode_worker
71
@@ -78,8 +87,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
87 welcome!).
88 * Only top-1 proposal and scoring are implemented. Tree-attention is left as
89 future work.
81 - * Only lossless rejection sampling is supported. Contributions adding lossy
82 - verification routines are welcome (e.g. Medusa's typical acceptance).
90 * All sequences in a batch must have the same proposal length, or zero. This
91 can be improved by having per-sequence speculation in the future.
92 * The scoring forward pass is done without an MQA kernel, which is
@@ -95,6 +102,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
102 scorer_worker: Worker,
103 draft_worker_kwargs: Dict[str, Any],
104 disable_by_batch_size: Optional[int],
105 + draft_token_acceptance_method: str,
106 + typical_acceptance_sampler_posterior_threshold: float,
107 + typical_acceptance_sampler_posterior_alpha: float,
108 ) -> "SpecDecodeWorker":
109
110 ngram_prompt_lookup_max = (
@@ -127,17 +137,30 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
137 logger.info("Configuring SpecDecodeWorker with proposer=%s",
138 type(proposer_worker))
139
140 + spec_decode_sampler: SpecDecodeBaseSampler = None
141 + if draft_token_acceptance_method == "rejection_sampler":
142 + spec_decode_sampler = RejectionSampler(
143 + disable_bonus_tokens=disable_bonus_tokens, )
144 + elif draft_token_acceptance_method == "typical_acceptance_sampler":
145 + spec_decode_sampler = TypicalAcceptanceSampler(
146 + disable_bonus_tokens=disable_bonus_tokens,
147 + posterior_threshold=\
148 + typical_acceptance_sampler_posterior_threshold,
149 + posterior_alpha=typical_acceptance_sampler_posterior_alpha,
150 + )
151 + logger.info("Configuring SpecDecodeWorker with sampler=%s",
152 + type(spec_decode_sampler))
153 +
154 return SpecDecodeWorker(proposer_worker,
155 scorer_worker,
156 disable_by_batch_size=disable_by_batch_size,
133 - rejection_sampler=RejectionSampler(
134 - disable_bonus_tokens=disable_bonus_tokens))
157 + spec_decode_sampler=spec_decode_sampler)
158
159 def __init__(
160 self,
161 proposer_worker: ProposerWorkerBase,
162 scorer_worker: WorkerBase,
140 - rejection_sampler: RejectionSampler,
163 + spec_decode_sampler: SpecDecodeBaseSampler,
164 metrics_collector: Optional[AsyncMetricsCollector] = None,
165 disable_by_batch_size: Optional[int] = None,
166 ):
@@ -150,8 +173,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
173 scorer_worker: A worker that produces probabilities of speculative
174 tokens according to some base model. Typically a vanilla vLLM
175 Worker.
153 - rejection_sampler: A Torch module used to perform modified rejection
154 - sampling for speculative decoding.
176 + spec_decode_sampler: A Torch module used to perform acceptance
177 + sampling of the draft tokens in the verification step of
178 + speculative decoding. Currently we support two different
179 + types of sampler namely RejectionSampler and
180 + TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
181 + instance of RejectionSampler or TypicalAcceptanceSampler.
182 disable_by_batch_size: If the batch size is larger than this,
183 disable speculative decoding for new incoming requests.
184 metrics_collector: Helper class for collecting metrics; can be set
@@ -160,15 +187,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
187 self.proposer_worker = proposer_worker
188 self.scorer_worker = scorer_worker
189 self.disable_by_batch_size = disable_by_batch_size or float("inf")
163 - self.rejection_sampler = rejection_sampler
164 -
190 + self.spec_decode_sampler = spec_decode_sampler
191 self._metrics = AsyncMetricsCollector(
166 - rejection_sampler
192 + self.spec_decode_sampler
193 ) if metrics_collector is None else metrics_collector
168 -
169 - self.probs_dtype = self.rejection_sampler.probs_dtype
170 - self.token_id_dtype = self.rejection_sampler.token_id_dtype
171 -
194 + self.probs_dtype = self.spec_decode_sampler.probs_dtype
195 + self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
196 # Lazy initiazliation.
197 self.scorer: SpeculativeScorer
198
@@ -189,7 +213,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
213 self.proposer_worker.load_model()
214
215 self._metrics.init_gpu_tensors(self.rank)
192 - self.rejection_sampler.init_gpu_tensors(self.rank)
216 + self.spec_decode_sampler.init_gpu_tensors(self.rank)
217 +
218 self.scorer = BatchExpansionTop1Scorer(
219 scorer_worker=self.scorer_worker,
220 device=self.device,
@@ -203,7 +228,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
228 def _configure_model_sampler_for_spec_decode(self):
229 """Configure model sampler to emit GPU tensors. This allows spec decode
230 to keep data on device without transferring to CPU and serializing,
206 - which significantly reduces overhead of rejection sampling.
231 + which significantly reduces overhead of sampling during verification.
232
233 NOTE(cade): This breaks abstraction boundaries pretty badly. The better
234 design is to have the "move to CPU and serialize" sampling decision be
@@ -481,7 +506,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
506 # Get proposed tokens.
507 proposal_token_ids = proposals.proposal_token_ids[spec_indices]
508
484 - accepted_token_ids = self.rejection_sampler(
509 + accepted_token_ids = self.spec_decode_sampler(
510 target_probs=proposal_verifier_probs,
511 bonus_token_ids=bonus_token_ids,
512 draft_probs=proposal_probs,
@@ -496,7 +521,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
521 accepted_token_ids = torch.cat(
522 [accepted_token_ids, non_spec_token_ids])
523 logprobs = proposal_scores.logprobs
499 -
524 # Rearrange so that results are in the order of the original seq group
525 # metadata.
526 accepted_token_ids[original_indices] = accepted_token_ids.clone()