A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Speculative Decoding 2/2 ] Integrate typical acceptance sampler into Spec Decode Worker (#5348)
sroy745 committed
Jul 1, 2024 at 00:33 UTC
80ca1e6a3a28a0373dc00c5b4fe956c16de952fa
14 files changed
+482
-210
tests/samplers/test_typical_acceptance_sampler.py
+64
-32
@@ -52,6 +52,19 @@ def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
52
return draft_token_ids
53
54
55
+def get_acceptance_sampler(
56
+ posterior_threshold: float = 0.03,
57
+ posterior_alpha: float = 0.9,
58
+ disable_bonus_tokens: bool = False,
59
+ strict_mode: bool = False,
60
+) -> TypicalAcceptanceSampler:
61
+ """
62
+ Initializes and returns a TypicalAcceptanceSampler.
63
+ """
64
+ return TypicalAcceptanceSampler(posterior_threshold, posterior_alpha,
65
+ disable_bonus_tokens, strict_mode)
66
+
67
+
68
@pytest.mark.parametrize("k", list(range(1, 6)))
69
@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
70
@pytest.mark.parametrize("batch_size", list(range(1, 32)))
@@ -64,7 +77,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
77
different combinations of k, vocab_size, batch_size and num devices.
78
"""
79
torch.set_default_device(device)
67
- typical_acceptance_sampler = TypicalAcceptanceSampler()
80
+ typical_acceptance_sampler = get_acceptance_sampler()
81
typical_acceptance_sampler.init_gpu_tensors(rank=0)
82
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
83
bonus_token_ids = torch.randint(low=0,
@@ -76,7 +89,10 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
89
size=(batch_size, k),
90
dtype=torch.int64)
91
# Verify that sampling succeeds for all cases.
79
- typical_acceptance_sampler(target_probs, bonus_token_ids, draft_token_ids)
92
+ typical_acceptance_sampler(target_probs,
93
+ bonus_token_ids,
94
+ draft_probs=None,
95
+ draft_token_ids=draft_token_ids)
96
97
98
@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
@@ -94,7 +110,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
110
batch_size = 5
111
vocab_size = 30_000
112
torch.set_default_device(device)
97
- typical_acceptance_sampler = TypicalAcceptanceSampler(strict_mode=True)
113
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
114
typical_acceptance_sampler.init_gpu_tensors(rank=0)
115
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
116
bonus_token_ids = torch.randint(low=0,
@@ -125,8 +141,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
141
oob_token_ids[0][0] = rogue_token_id
142
143
with pytest.raises(AssertionError):
128
- typical_acceptance_sampler(target_probs, bonus_token_ids,
129
- draft_token_ids)
144
+ typical_acceptance_sampler(target_probs,
145
+ bonus_token_ids,
146
+ draft_probs=None,
147
+ draft_token_ids=draft_token_ids)
148
149
150
@pytest.mark.parametrize("seed", list(range(10)))
@@ -151,7 +169,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
169
batch_size = 5
170
vocab_size = 30_000
171
torch.set_default_device(device)
154
- typical_acceptance_sampler = TypicalAcceptanceSampler(
172
+ typical_acceptance_sampler = get_acceptance_sampler(
173
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174
typical_acceptance_sampler.init_gpu_tensors(rank=0)
175
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -163,9 +181,11 @@ def test_uniform_target_distribution_accepts_all_tokens(
181
high=vocab_size,
182
size=(batch_size, 1),
183
dtype=torch.int64)
166
- output_token_ids = typical_acceptance_sampler(target_probs,
167
- bonus_token_ids,
168
- draft_token_ids)
184
+ output_token_ids = typical_acceptance_sampler(
185
+ target_probs,
186
+ bonus_token_ids,
187
+ draft_probs=None,
188
+ draft_token_ids=draft_token_ids)
189
# We are using a uniform target probability distribution.
190
# For a uniform distribution the entropy is very high and it
191
# should lead to all draft tokens being accepted. Verify that.
@@ -203,7 +223,7 @@ def test_temperature_zero_target_distribution(seed: int,
223
vocab_size = 30_000
224
torch.set_default_device(device)
225
206
- typical_acceptance_sampler = TypicalAcceptanceSampler(
226
+ typical_acceptance_sampler = get_acceptance_sampler(
227
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
228
typical_acceptance_sampler.init_gpu_tensors(rank=0)
229
# Simulate temperature 0 probability distribution for target probabilities
@@ -224,9 +244,11 @@ def test_temperature_zero_target_distribution(seed: int,
244
# 1.0 tokens in the target distribution we will reject all of them and
245
# fallback to the greedy sampling for selecting 1 token for each sequence.
246
# Verify the same.
227
- output_token_ids = typical_acceptance_sampler(target_probs,
228
- bonus_token_ids,
229
- draft_token_ids)
247
+ output_token_ids = typical_acceptance_sampler(
248
+ target_probs,
249
+ bonus_token_ids,
250
+ draft_probs=None,
251
+ draft_token_ids=draft_token_ids)
252
assert output_token_ids.shape[0] == batch_size
253
assert output_token_ids.shape[1] == (k + 1)
254
assert torch.all(output_token_ids[:, -1] == -1)
@@ -261,7 +283,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
283
batch_size = 4
284
vocab_size = 30_000
285
torch.set_default_device(device)
264
- typical_acceptance_sampler = TypicalAcceptanceSampler(
286
+ typical_acceptance_sampler = get_acceptance_sampler(
287
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
288
typical_acceptance_sampler.init_gpu_tensors(rank=0)
289
# For sequences 0 and 2 set the distribution to a temperature
@@ -277,9 +299,11 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
299
high=vocab_size,
300
size=(batch_size, 1),
301
dtype=torch.int64)
280
- output_token_ids = typical_acceptance_sampler(target_probs,
281
- bonus_token_ids,
282
- draft_token_ids)
302
+ output_token_ids = typical_acceptance_sampler(
303
+ target_probs,
304
+ bonus_token_ids,
305
+ draft_probs=None,
306
+ draft_token_ids=draft_token_ids)
307
# verify the shape of output_token_ids
308
assert output_token_ids.shape[0] == batch_size
309
assert output_token_ids.shape[1] == (k + 1)
@@ -326,7 +350,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
350
batch_size = 1
351
vocab_size = 30_000
352
torch.set_default_device(device)
329
- typical_acceptance_sampler = TypicalAcceptanceSampler(
353
+ typical_acceptance_sampler = get_acceptance_sampler(
354
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
355
typical_acceptance_sampler.init_gpu_tensors(rank=0)
356
# Create a temperature zero target probability distribution and ensure
@@ -339,9 +363,11 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
363
high=vocab_size,
364
size=(batch_size, 1),
365
dtype=torch.int64)
342
- output_token_ids = typical_acceptance_sampler(target_probs,
343
- bonus_token_ids,
344
- draft_token_ids)
366
+ output_token_ids = typical_acceptance_sampler(
367
+ target_probs,
368
+ bonus_token_ids,
369
+ draft_probs=None,
370
+ draft_token_ids=draft_token_ids)
371
assert output_token_ids.shape[0] == batch_size
372
assert output_token_ids.shape[1] == (k + 1)
373
assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
@@ -357,9 +383,11 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
383
batch_size, k, vocab_size, zero_temperature_token_ids)
384
draft_token_ids = torch.cat(
385
(draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
360
- output_token_ids = typical_acceptance_sampler(target_probs,
361
- bonus_token_ids,
362
- draft_token_ids)
386
+ output_token_ids = typical_acceptance_sampler(
387
+ target_probs,
388
+ bonus_token_ids,
389
+ draft_probs=None,
390
+ draft_token_ids=draft_token_ids)
391
assert output_token_ids.shape[0] == batch_size
392
assert output_token_ids.shape[1] == (k + 1)
393
assert torch.all(output_token_ids[:, :2] == draft_token_ids[:, :2])
@@ -384,7 +412,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
412
batch_size = 1
413
vocab_size = 30_000
414
torch.set_default_device(device)
387
- typical_acceptance_sampler = TypicalAcceptanceSampler(
415
+ typical_acceptance_sampler = get_acceptance_sampler(
416
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
417
typical_acceptance_sampler.init_gpu_tensors(rank=0)
418
# Simulate temperature 0 probability distribution for target
@@ -402,9 +430,11 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
430
high=vocab_size,
431
size=(batch_size, 1),
432
dtype=torch.int64)
405
- output_token_ids = typical_acceptance_sampler(target_probs,
406
- bonus_token_ids,
407
- draft_token_ids)
433
+ output_token_ids = typical_acceptance_sampler(
434
+ target_probs,
435
+ bonus_token_ids,
436
+ draft_probs=None,
437
+ draft_token_ids=draft_token_ids)
438
assert output_token_ids.shape[0] == batch_size
439
assert output_token_ids.shape[1] == (k + 1)
440
assert torch.all(output_token_ids[:, 1:-1] == -1)
@@ -418,9 +448,11 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
448
posterior_threshold=0.0,
449
posterior_alpha=0.0)
450
typical_acceptance_sampler.init_gpu_tensors(rank=0)
421
- output_token_ids = typical_acceptance_sampler(target_probs,
422
- bonus_token_ids,
423
- draft_token_ids)
451
+ output_token_ids = typical_acceptance_sampler(
452
+ target_probs,
453
+ bonus_token_ids,
454
+ draft_probs=None,
455
+ draft_token_ids=draft_token_ids)
456
assert output_token_ids.shape[0] == batch_size
457
assert output_token_ids.shape[1] == (k + 1)
458
assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
@@ -451,7 +483,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
483
batch_size = 5
484
vocab_size = 30_000
485
torch.set_default_device(device)
454
- typical_acceptance_sampler = TypicalAcceptanceSampler(
486
+ typical_acceptance_sampler = get_acceptance_sampler(
487
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
488
typical_acceptance_sampler.init_gpu_tensors(rank=0)
489
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
tests/spec_decode/e2e/test_multistep_correctness.py
+53
-1
@@ -11,9 +11,15 @@ distribution matches the target model's output distribution (up to hardware
11
numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
12
equality. This gives us good coverage of temp=0.
13
14
+At temp=0, the TypicalAcceptanceSampler ensures that only the tokens with the
15
+highest probability in the target distribution are accepted. Therefore, we can
16
+expect greedy equality for the TypicalAcceptanceSampler at temp=0.
17
+
18
For temp>0, we rely on unit tests on the rejection sampler to verify that the
19
output distribution is the same with spec decode vs. no spec decode (this would
16
-be prohibitively expensive to run with a real model).
20
+be prohibitively expensive to run with a real model). Similarly, for the
21
+TypicalAcceptance sampler also, we rely on unit tests to validate temp>0
22
+test cases.
23
24
NOTE: Speculative decoding's distribution equality requires that the measured
25
distributions of the target model and proposal model be deterministic given the
@@ -611,3 +617,49 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
617
batch_size,
618
max_output_len=output_len,
619
force_output_len=True)
620
+
621
+
622
+@pytest.mark.parametrize(
623
+ "common_llm_kwargs",
624
+ [{
625
+ "model": "JackFram/llama-160m",
626
+
627
+ # Skip cuda graph recording for fast test.
628
+ "enforce_eager": True,
629
+
630
+ # Required for spec decode.
631
+ "use_v2_block_manager": True
632
+ }])
633
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
634
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
635
+@pytest.mark.parametrize(
636
+ "test_llm_kwargs",
637
+ [
638
+ {
639
+ "speculative_model": "JackFram/llama-68m",
640
+ "num_speculative_tokens": k,
641
+ "spec_decoding_acceptance_method": "typical_acceptance_sampler"
642
+ }
643
+ # Try a range of common k.
644
+ for k in [1, 2, 3]
645
+ ])
646
+@pytest.mark.parametrize("batch_size", [1, 32])
647
+@pytest.mark.parametrize(
648
+ "output_len",
649
+ [
650
+ # Use smaller output len for fast test.
651
+ 32,
652
+ ])
653
+@pytest.mark.parametrize("seed", [1])
654
+def test_typical_acceptance_sampling(baseline_llm_generator,
655
+ test_llm_generator, batch_size: int,
656
+ output_len: int):
657
+ """Verify that speculative decoding produces exact equality to without spec
658
+ decode with TypicalAcceptanceSampler as the draft token acceptance
659
+ sampling method.
660
+ """
661
+ run_greedy_equality_correctness_test(baseline_llm_generator,
662
+ test_llm_generator,
663
+ batch_size,
664
+ max_output_len=output_len,
665
+ force_output_len=True)
tests/spec_decode/test_dynamic_spec_decode.py
+7
-5
@@ -3,33 +3,35 @@ from unittest.mock import MagicMock, patch
3
import pytest
4
import torch
5
6
-from vllm.model_executor.layers.rejection_sampler import RejectionSampler
6
from vllm.sequence import ExecuteModelRequest
7
from vllm.spec_decode.metrics import AsyncMetricsCollector
8
from vllm.spec_decode.multi_step_worker import MultiStepWorker
9
from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
10
from vllm.spec_decode.top1_proposer import Top1Proposer
11
12
+from .test_utils import mock_spec_decode_sampler
13
from .utils import create_batch, mock_worker
14
15
16
@pytest.mark.parametrize('queue_size', [4])
17
@pytest.mark.parametrize('batch_size', [1])
18
@pytest.mark.parametrize('k', [1])
19
+@pytest.mark.parametrize("acceptance_sampler_method",
20
+ ["rejection_sampler", "typical_acceptance_sampler"])
21
@torch.inference_mode()
20
-def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int):
22
+def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
23
+ acceptance_sampler_method: str):
24
"""Verify that speculative tokens are disabled when the batch size
25
exceeds the threshold.
26
"""
27
disable_by_batch_size = 3
25
-
28
draft_worker = mock_worker(cls=MultiStepWorker)
29
target_worker = mock_worker()
28
- rejection_sampler = MagicMock(spec=RejectionSampler)
30
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
31
worker = SpecDecodeWorker(proposer_worker=draft_worker,
32
scorer_worker=target_worker,
32
- rejection_sampler=rejection_sampler,
33
+ spec_decode_sampler=mock_spec_decode_sampler(
34
+ acceptance_sampler_method),
35
metrics_collector=metrics_collector,
36
disable_by_batch_size=disable_by_batch_size)
37
tests/spec_decode/test_metrics.py
+47
-47
@@ -10,16 +10,16 @@ from vllm.spec_decode.metrics import AsyncMetricsCollector
10
def test_initial_call_returns_none():
11
"""Expect first call to get metrics to return None.
12
"""
13
- rej_sampler = MagicMock()
14
- rej_sampler.num_accepted_tokens = torch.tensor(0,
15
- dtype=torch.long,
16
- device='cuda')
17
- rej_sampler.num_emitted_tokens = torch.tensor(0,
18
- dtype=torch.long,
19
- device='cuda')
20
- rej_sampler.num_draft_tokens = 0
21
-
22
- collector = AsyncMetricsCollector(rej_sampler)
13
+ spec_decode_sampler = MagicMock()
14
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
15
+ dtype=torch.long,
16
+ device='cuda')
17
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
18
+ dtype=torch.long,
19
+ device='cuda')
20
+ spec_decode_sampler.num_draft_tokens = 0
21
+
22
+ collector = AsyncMetricsCollector(spec_decode_sampler)
23
collector.init_gpu_tensors(rank=0)
24
maybe_metrics = collector.maybe_collect_rejsample_metrics(k=5)
25
assert maybe_metrics is None
@@ -28,14 +28,14 @@ def test_initial_call_returns_none():
28
def test_second_call_returns_metrics():
29
"""Expect second call to not return None.
30
"""
31
- rej_sampler = MagicMock()
32
- rej_sampler.num_accepted_tokens = torch.tensor(0,
33
- dtype=torch.long,
34
- device='cuda')
35
- rej_sampler.num_emitted_tokens = torch.tensor(0,
36
- dtype=torch.long,
37
- device='cuda')
38
- rej_sampler.num_draft_tokens = 0
31
+ spec_decode_sampler = MagicMock()
32
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
33
+ dtype=torch.long,
34
+ device='cuda')
35
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
36
+ dtype=torch.long,
37
+ device='cuda')
38
+ spec_decode_sampler.num_draft_tokens = 0
39
40
collect_interval_s = 5.0
41
timer = MagicMock()
@@ -43,7 +43,7 @@ def test_second_call_returns_metrics():
43
0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
44
]
45
46
- collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
46
+ collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
47
timer=timer,
48
collect_interval_s=collect_interval_s)
49
collector.init_gpu_tensors(rank=0)
@@ -56,16 +56,16 @@ def test_second_call_returns_metrics():
56
def test_nonzero_rank_noop(rank):
57
"""Verify nonzero ranks don't collect metrics.
58
"""
59
- rej_sampler = MagicMock()
60
- rej_sampler.num_accepted_tokens = torch.tensor(0,
61
- dtype=torch.long,
62
- device='cuda')
63
- rej_sampler.num_emitted_tokens = torch.tensor(0,
64
- dtype=torch.long,
65
- device='cuda')
66
- rej_sampler.num_draft_tokens = 0
67
-
68
- collector = AsyncMetricsCollector(rej_sampler)
59
+ spec_decode_sampler = MagicMock()
60
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
61
+ dtype=torch.long,
62
+ device='cuda')
63
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
64
+ dtype=torch.long,
65
+ device='cuda')
66
+ spec_decode_sampler.num_draft_tokens = 0
67
+
68
+ collector = AsyncMetricsCollector(spec_decode_sampler)
69
collector.init_gpu_tensors(rank=rank)
70
_ = collector.maybe_collect_rejsample_metrics(k=5)
71
metrics = collector.maybe_collect_rejsample_metrics(k=5)
@@ -75,14 +75,14 @@ def test_nonzero_rank_noop(rank):
75
def test_noop_until_time():
76
"""Verify metrics aren't collected until enough time passes.
77
"""
78
- rej_sampler = MagicMock()
79
- rej_sampler.num_accepted_tokens = torch.tensor(0,
80
- dtype=torch.long,
81
- device='cuda')
82
- rej_sampler.num_emitted_tokens = torch.tensor(0,
83
- dtype=torch.long,
84
- device='cuda')
85
- rej_sampler.num_draft_tokens = 0
78
+ spec_decode_sampler = MagicMock()
79
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(0,
80
+ dtype=torch.long,
81
+ device='cuda')
82
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(0,
83
+ dtype=torch.long,
84
+ device='cuda')
85
+ spec_decode_sampler.num_draft_tokens = 0
86
87
collect_interval_s = 5.0
88
timer = MagicMock()
@@ -91,7 +91,7 @@ def test_noop_until_time():
91
collect_interval_s + 0.1, collect_interval_s + 0.1
92
]
93
94
- collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
94
+ collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
95
timer=timer,
96
collect_interval_s=collect_interval_s)
97
collector.init_gpu_tensors(rank=0)
@@ -122,14 +122,14 @@ def test_initial_metrics_has_correct_values(has_data: bool):
122
max_num_emitted_tokens = AsyncMetricsCollector.get_max_num_emitted_tokens(
123
num_draft_tokens, k)
124
125
- rej_sampler = MagicMock()
126
- rej_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127
- dtype=torch.long,
128
- device='cuda')
129
- rej_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130
- dtype=torch.long,
131
- device='cuda')
132
- rej_sampler.num_draft_tokens = num_draft_tokens
125
+ spec_decode_sampler = MagicMock()
126
+ spec_decode_sampler.num_accepted_tokens = torch.tensor(num_accepted_tokens,
127
+ dtype=torch.long,
128
+ device='cuda')
129
+ spec_decode_sampler.num_emitted_tokens = torch.tensor(num_emitted_tokens,
130
+ dtype=torch.long,
131
+ device='cuda')
132
+ spec_decode_sampler.num_draft_tokens = num_draft_tokens
133
134
collect_interval_s = 5.0
135
timer = MagicMock()
@@ -137,7 +137,7 @@ def test_initial_metrics_has_correct_values(has_data: bool):
137
0.0, collect_interval_s + 0.1, collect_interval_s + 0.2
138
]
139
140
- collector = AsyncMetricsCollector(rejection_sampler=rej_sampler,
140
+ collector = AsyncMetricsCollector(spec_decode_sampler=spec_decode_sampler,
141
timer=timer,
142
collect_interval_s=collect_interval_s)
143
collector.init_gpu_tensors(rank=0)
tests/spec_decode/test_spec_decode_worker.py
+85
-69
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock
6
import pytest
7
import torch
8
9
-from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9
from vllm.model_executor.utils import set_random_seed
10
from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
11
from vllm.spec_decode.interfaces import SpeculativeProposals
@@ -16,23 +15,26 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
15
from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
16
split_num_cache_blocks_evenly)
17
18
+from .test_utils import mock_spec_decode_sampler
19
from .utils import create_batch, create_sampler_output_list, mock_worker
20
21
22
@pytest.mark.parametrize('k', [1, 2, 6])
23
@pytest.mark.parametrize('batch_size', [1, 2, 32])
24
+@pytest.mark.parametrize("acceptance_sampler_method",
25
+ ["rejection_sampler", "typical_acceptance_sampler"])
26
@torch.inference_mode()
25
-def test_correctly_calls_draft_model(k: int, batch_size: int):
27
+def test_correctly_calls_draft_model(k: int, batch_size: int,
28
+ acceptance_sampler_method: str):
29
"""Verify SpecDecodeWorker calls the draft worker with correct
30
inputs. Everything else is mocked out.
31
"""
32
draft_worker = mock_worker(cls=MultiStepWorker)
33
target_worker = mock_worker()
31
- rejection_sampler = MagicMock(spec=RejectionSampler)
34
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
33
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
34
- metrics_collector)
35
-
35
+ worker = SpecDecodeWorker(
36
+ draft_worker, target_worker,
37
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
38
exception_secret = 'artificial stop'
39
draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
40
@@ -53,15 +55,16 @@ def test_correctly_calls_draft_model(k: int, batch_size: int):
55
56
@pytest.mark.parametrize('k', [1, 2, 6])
57
@pytest.mark.parametrize('batch_size', [1, 2, 32])
58
+@pytest.mark.parametrize("acceptance_sampler_method",
59
+ ["rejection_sampler", "typical_acceptance_sampler"])
60
@torch.inference_mode()
57
-def test_correctly_calls_target_model(k: int, batch_size: int):
61
+def test_correctly_calls_target_model(k: int, batch_size: int,
62
+ acceptance_sampler_method: str):
63
"""Verify SpecDecodeWorker calls the target model with correct
64
inputs. Everything else is mocked out.
65
"""
66
draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
67
target_worker = mock_worker(use_spec=False)
63
- rejection_sampler = MagicMock(spec=RejectionSampler)
64
- rejection_sampler.token_id_dtype = torch.int64
68
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
69
70
draft_worker.device = 'cuda'
@@ -69,8 +72,9 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
72
73
set_random_seed(1)
74
72
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
73
- metrics_collector)
75
+ worker = SpecDecodeWorker(
76
+ draft_worker, target_worker,
77
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
78
worker.init_device()
79
80
vocab_size = 32_000
@@ -133,8 +137,11 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
137
138
@pytest.mark.parametrize('k', [1, 2, 6])
139
@pytest.mark.parametrize('batch_size', [1, 2, 32])
140
+@pytest.mark.parametrize("acceptance_sampler_method",
141
+ ["rejection_sampler", "typical_acceptance_sampler"])
142
@torch.inference_mode()
137
-def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
143
+def test_correctly_calls_spec_decode_sampler(k: int, batch_size: int,
144
+ acceptance_sampler_method: str):
145
"""Verify SpecDecodeWorker calls the rejection sampler with
146
correct inputs. Everything else is mocked out.
147
"""
@@ -144,15 +151,14 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
151
vocab_size=vocab_size,
152
use_spec=False)
153
target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
147
- rejection_sampler = MagicMock(spec=RejectionSampler)
148
- rejection_sampler.token_id_dtype = torch.int64
154
+ spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
155
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
156
draft_worker.device = 'cuda'
157
target_worker.device = 'cuda'
158
159
set_random_seed(1)
160
155
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
161
+ worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
162
metrics_collector)
163
worker.init_device()
164
@@ -199,15 +205,16 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
205
target_worker.execute_model.return_value = [target_output[0]]
206
207
exception_secret = 'artificial stop'
202
- rejection_sampler.side_effect = ValueError(exception_secret)
208
+
209
+ spec_decode_sampler.side_effect = ValueError(exception_secret)
210
211
with pytest.raises(ValueError, match=exception_secret):
212
worker.execute_model(execute_model_req=ExecuteModelRequest(
213
seq_group_metadata_list=seq_group_metadata_list,
214
num_lookahead_slots=k))
215
209
- assert len(rejection_sampler.call_args_list) == 1
210
- _, kwargs = rejection_sampler.call_args_list[0]
216
+ assert len(spec_decode_sampler.call_args_list) == 1
217
+ _, kwargs = spec_decode_sampler.call_args_list[0]
218
actual = SimpleNamespace(**kwargs)
219
220
assert torch.equal(actual.bonus_token_ids,
@@ -221,8 +228,11 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
228
229
@pytest.mark.parametrize('k', [1, 2, 6])
230
@pytest.mark.parametrize('batch_size', [1, 2, 32])
231
+@pytest.mark.parametrize("acceptance_sampler_method",
232
+ ["rejection_sampler", "typical_acceptance_sampler"])
233
@torch.inference_mode()
225
-def test_correctly_formats_output(k: int, batch_size: int):
234
+def test_correctly_formats_output(k: int, batch_size: int,
235
+ acceptance_sampler_method: str):
236
"""Verify SpecDecodeWorker formats sampler output correctly.
237
Everything else is mocked out.
238
"""
@@ -232,15 +242,13 @@ def test_correctly_formats_output(k: int, batch_size: int):
242
vocab_size=vocab_size,
243
use_spec=False)
244
target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
235
- rejection_sampler = MagicMock(spec=RejectionSampler)
236
- rejection_sampler.token_id_dtype = torch.int64
245
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
246
draft_worker.device = 'cuda'
247
target_worker.device = 'cuda'
248
249
set_random_seed(1)
242
-
243
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
250
+ spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
251
+ worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
252
metrics_collector)
253
worker.init_device()
254
@@ -286,24 +294,23 @@ def test_correctly_formats_output(k: int, batch_size: int):
294
295
target_worker.execute_model.return_value = [target_output[0]]
296
289
- rejection_sampler_output = torch.randint(low=0,
290
- high=vocab_size,
291
- size=(batch_size, k + 1),
292
- dtype=torch.int64,
293
- device='cuda')
297
+ spec_decode_sampler_output = torch.randint(low=0,
298
+ high=vocab_size,
299
+ size=(batch_size, k + 1),
300
+ dtype=torch.int64,
301
+ device='cuda')
302
for i in range(batch_size):
303
minimum_accepted_tokens = 1
296
- rejection_sampler_output[i][
304
+ spec_decode_sampler_output[i][
305
-random.randint(minimum_accepted_tokens, k + 1):] = -1
306
299
- rejection_sampler.return_value = rejection_sampler_output
300
-
307
+ spec_decode_sampler.return_value = spec_decode_sampler_output
308
output = worker.execute_model(execute_model_req=ExecuteModelRequest(
309
seq_group_metadata_list=seq_group_metadata_list,
310
num_lookahead_slots=k))
311
312
expected_output = create_sampler_output_list(
306
- token_ids=rejection_sampler_output.transpose(0, 1),
313
+ token_ids=spec_decode_sampler_output.transpose(0, 1),
314
probs=[None for _ in range(k + 1)],
315
logprobs=[None for _ in range(k + 1)])
316
@@ -350,8 +357,11 @@ def test_correctly_formats_output(k: int, batch_size: int):
357
@pytest.mark.parametrize('k', [1, 2])
358
@pytest.mark.parametrize('batch_size', [1])
359
@pytest.mark.parametrize('returns_metrics', [True, False])
360
+@pytest.mark.parametrize("acceptance_sampler_method",
361
+ ["rejection_sampler", "typical_acceptance_sampler"])
362
@torch.inference_mode()
354
-def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
363
+def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
364
+ acceptance_sampler_method: str):
365
"""Verify SpecDecodeWorker collects metrics.
366
"""
367
vocab_size = 32_000
@@ -360,15 +370,14 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
370
vocab_size=vocab_size,
371
use_spec=False)
372
target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
363
- rejection_sampler = MagicMock(spec=RejectionSampler)
364
- rejection_sampler.token_id_dtype = torch.int64
373
+ spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
374
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
375
draft_worker.device = 'cuda'
376
target_worker.device = 'cuda'
377
378
set_random_seed(1)
379
371
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
380
+ worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
381
metrics_collector)
382
worker.init_device()
383
@@ -414,17 +423,16 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
423
424
target_worker.execute_model.return_value = [target_output[0]]
425
417
- rejection_sampler_output = torch.randint(low=0,
418
- high=vocab_size,
419
- size=(batch_size, k + 1),
420
- dtype=torch.int64,
421
- device='cuda')
426
+ spec_decode_sampler_output = torch.randint(low=0,
427
+ high=vocab_size,
428
+ size=(batch_size, k + 1),
429
+ dtype=torch.int64,
430
+ device='cuda')
431
for i in range(batch_size):
432
minimum_accepted_tokens = 1
424
- rejection_sampler_output[i][
433
+ spec_decode_sampler_output[i][
434
-random.randint(minimum_accepted_tokens, k + 1):] = -1
426
-
427
- rejection_sampler.return_value = rejection_sampler_output
435
+ spec_decode_sampler.return_value = spec_decode_sampler_output
436
437
mock_rejsample_metrics = MagicMock(
438
spec=SpecDecodeWorkerMetrics) if returns_metrics else None
@@ -445,15 +453,16 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
453
454
@pytest.mark.parametrize('k', [0])
455
@pytest.mark.parametrize('batch_size', [1, 2, 32])
456
+@pytest.mark.parametrize("acceptance_sampler_method",
457
+ ["rejection_sampler", "typical_acceptance_sampler"])
458
@torch.inference_mode()
449
-def test_k_equals_zero(k: int, batch_size: int):
459
+def test_k_equals_zero(k: int, batch_size: int,
460
+ acceptance_sampler_method: str):
461
"""Verify that the SpecDecodeWorker calls the draft and target workers
462
when k is zero. This happens during prefill.
463
"""
464
draft_worker = mock_worker(cls=MultiStepWorker)
465
target_worker = mock_worker()
455
- rejection_sampler = MagicMock(spec=RejectionSampler)
456
- rejection_sampler.token_id_dtype = torch.int64
466
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
467
468
sampler_output = MagicMock(spec=SamplerOutput)
@@ -465,8 +474,9 @@ def test_k_equals_zero(k: int, batch_size: int):
474
475
set_random_seed(1)
476
468
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
469
- metrics_collector)
477
+ worker = SpecDecodeWorker(
478
+ draft_worker, target_worker,
479
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
480
481
seq_group_metadata_list, _, _ = create_batch(batch_size,
482
k,
@@ -487,16 +497,17 @@ def test_k_equals_zero(k: int, batch_size: int):
497
498
@pytest.mark.parametrize('k', [0, 5])
499
@pytest.mark.parametrize('batch_size', [0])
500
+@pytest.mark.parametrize("acceptance_sampler_method",
501
+ ["rejection_sampler", "typical_acceptance_sampler"])
502
@torch.inference_mode()
491
-def test_empty_input_batch(k: int, batch_size: int):
503
+def test_empty_input_batch(k: int, batch_size: int,
504
+ acceptance_sampler_method: str):
505
"""Verify that the SpecDecodeWorker calls the draft and target workers
506
when the input batch is empty. This can happen if the engine communicates
507
to the workers information without scheduling a batch.
508
"""
509
draft_worker = mock_worker(cls=MultiStepWorker)
510
target_worker = mock_worker()
498
- rejection_sampler = MagicMock(spec=RejectionSampler)
499
- rejection_sampler.token_id_dtype = torch.int64
511
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
512
513
sampler_output = MagicMock(spec=SamplerOutput)
@@ -508,8 +519,9 @@ def test_empty_input_batch(k: int, batch_size: int):
519
520
set_random_seed(1)
521
511
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
512
- metrics_collector)
522
+ worker = SpecDecodeWorker(
523
+ draft_worker, target_worker,
524
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
525
526
seq_group_metadata_list, _, _ = create_batch(batch_size,
527
k,
@@ -528,18 +540,19 @@ def test_empty_input_batch(k: int, batch_size: int):
540
target_worker.execute_model.assert_called_once_with(execute_model_req)
541
542
543
+@pytest.mark.parametrize("acceptance_sampler_method",
544
+ ["rejection_sampler", "typical_acceptance_sampler"])
545
@pytest.mark.skip_global_cleanup
532
-def test_init_device():
546
+def test_init_device(acceptance_sampler_method: str):
547
"""Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
548
well as other GPU initialization.
549
"""
550
draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
551
target_worker = mock_worker(use_spec=False)
538
- rejection_sampler = MagicMock(spec=RejectionSampler)
539
- rejection_sampler.token_id_dtype = torch.int64
552
+ spec_decode_sampler = mock_spec_decode_sampler(acceptance_sampler_method)
553
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
554
542
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
555
+ worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
556
metrics_collector)
557
558
worker.init_device()
@@ -549,22 +562,23 @@ def test_init_device():
562
target_worker.init_device.assert_called_once()
563
564
metrics_collector.init_gpu_tensors.assert_called_once()
552
- rejection_sampler.init_gpu_tensors.assert_called_once()
565
+ spec_decode_sampler.init_gpu_tensors.assert_called_once()
566
567
568
+@pytest.mark.parametrize("acceptance_sampler_method",
569
+ ["rejection_sampler", "typical_acceptance_sampler"])
570
@torch.inference_mode()
556
-def test_initialize_cache():
571
+def test_initialize_cache(acceptance_sampler_method):
572
"""Verify SpecDecodeWorker invokes initialize_cache on proposer/scorer
573
workers.
574
"""
575
draft_worker = mock_worker(cls=MultiStepWorker)
576
target_worker = mock_worker()
562
- rejection_sampler = MagicMock(spec=RejectionSampler)
563
- rejection_sampler.token_id_dtype = torch.int64
577
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
578
566
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
567
- metrics_collector)
579
+ worker = SpecDecodeWorker(
580
+ draft_worker, target_worker,
581
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
582
583
kwargs = {"num_gpu_blocks": 1024, "num_cpu_blocks": 1023}
584
worker.initialize_cache(**kwargs)
@@ -577,19 +591,20 @@ def test_initialize_cache():
591
@pytest.mark.parametrize('available_cpu_blocks', [500])
592
@pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
593
@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
594
+@pytest.mark.parametrize("acceptance_sampler_method",
595
+ ["rejection_sampler", "typical_acceptance_sampler"])
596
@pytest.mark.skip_global_cleanup
597
def test_determine_num_available_blocks(available_gpu_blocks: int,
598
available_cpu_blocks: int,
599
target_cache_block_size_bytes: int,
584
- draft_kv_size_bytes: int):
600
+ draft_kv_size_bytes: int,
601
+ acceptance_sampler_method: str):
602
"""Verify SpecDecodeWorker correctly profiles num available GPU blocks.
603
Specifically, it should run profiling in the scorer worker, and then evenly
604
split the blocks between proposer and scorer worker.
605
"""
606
draft_worker = mock_worker(cls=MultiStepWorker)
607
target_worker = mock_worker()
591
- rejection_sampler = MagicMock(spec=RejectionSampler)
592
- rejection_sampler.token_id_dtype = torch.int64
608
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
609
610
target_worker.determine_num_available_blocks.return_value = (
@@ -598,8 +613,9 @@ def test_determine_num_available_blocks(available_gpu_blocks: int,
613
target_cache_block_size_bytes)
614
draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
615
601
- worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
602
- metrics_collector)
616
+ worker = SpecDecodeWorker(
617
+ draft_worker, target_worker,
618
+ mock_spec_decode_sampler(acceptance_sampler_method), metrics_collector)
619
620
num_gpu_blocks, num_cpu_blocks = worker.determine_num_available_blocks()
621
tests/spec_decode/test_utils.py
+22
@@ -1,7 +1,11 @@
1
from unittest.mock import MagicMock
2
3
import pytest
4
+import torch
5
6
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7
+from vllm.model_executor.layers.typical_acceptance_sampler import (
8
+ TypicalAcceptanceSampler)
9
from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
10
from vllm.spec_decode.util import split_batch_by_proposal_len
11
@@ -109,3 +113,21 @@ def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
113
114
assert filtered_groups == []
115
assert indices == []
116
+
117
+
118
+def mock_spec_decode_sampler(acceptance_sampler_method):
119
+ """
120
+ Returns either a RejectionSampler or TypicalAcceptanceSampler
121
+ object depending on whether acceptance_sampler_method is
122
+ 'rejection_sampler' or 'typical_acceptance_sampler' respectively.
123
+ """
124
+ if acceptance_sampler_method == "rejection_sampler":
125
+ sampler = MagicMock(spec=RejectionSampler)
126
+ sampler.token_id_dtype = torch.int64
127
+ return sampler
128
+ elif acceptance_sampler_method == "typical_acceptance_sampler":
129
+ sampler = MagicMock(spec=TypicalAcceptanceSampler)
130
+ sampler.token_id_dtype = torch.int64
131
+ return sampler
132
+ else:
133
+ raise ValueError(f"Invalid sampler name {acceptance_sampler_method}")
vllm/config.py
+73
-2
@@ -753,7 +753,6 @@ class SchedulerConfig:
753
self.chunked_prefill_enabled = enable_chunked_prefill
754
self.embedding_mode = embedding_mode
755
self.preemption_mode = preemption_mode
756
-
756
self._verify_args()
757
758
def _verify_args(self) -> None:
@@ -834,6 +833,9 @@ class SpeculativeConfig:
833
speculative_disable_by_batch_size: Optional[int],
834
ngram_prompt_lookup_max: Optional[int],
835
ngram_prompt_lookup_min: Optional[int],
836
+ draft_token_acceptance_method: str,
837
+ typical_acceptance_sampler_posterior_threshold: Optional[float],
838
+ typical_acceptance_sampler_posterior_alpha: Optional[float],
839
) -> Optional["SpeculativeConfig"]:
840
"""Create a SpeculativeConfig if possible, else return None.
841
@@ -870,7 +872,20 @@ class SpeculativeConfig:
872
window, if provided.
873
ngram_prompt_lookup_min (Optional[int]): Min size of ngram token
874
window, if provided.
873
-
875
+ draft_token_acceptance_method (str): The method to use for
876
+ accepting draft tokens. This can take two possible
877
+ values 'rejection_sampler' and 'typical_acceptance_sampler'
878
+ for RejectionSampler and TypicalAcceptanceSampler
879
+ respectively.
880
+ typical_acceptance_sampler_posterior_threshold (Optional[float]):
881
+ A threshold value that sets a lower bound on the posterior
882
+ probability of a token in the target model for it to be
883
+ accepted. This threshold is used only when we use the
884
+ TypicalAcceptanceSampler for token acceptance.
885
+ typical_acceptance_sampler_posterior_alpha (Optional[float]):
886
+ A scaling factor for the entropy-based threshold in the
887
+ TypicalAcceptanceSampler.
888
+
889
Returns:
890
Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
891
the necessary conditions are met, else None.
@@ -984,6 +999,11 @@ class SpeculativeConfig:
999
"speculative_model unless the draft model config contains an "
1000
"n_predict parameter.")
1001
1002
+ if typical_acceptance_sampler_posterior_threshold is None:
1003
+ typical_acceptance_sampler_posterior_threshold = 0.09
1004
+ if typical_acceptance_sampler_posterior_alpha is None:
1005
+ typical_acceptance_sampler_posterior_alpha = 0.3
1006
+
1007
return SpeculativeConfig(
1008
draft_model_config,
1009
draft_parallel_config,
@@ -991,6 +1011,11 @@ class SpeculativeConfig:
1011
speculative_disable_by_batch_size,
1012
ngram_prompt_lookup_max,
1013
ngram_prompt_lookup_min,
1014
+ draft_token_acceptance_method=draft_token_acceptance_method,
1015
+ typical_acceptance_sampler_posterior_threshold=\
1016
+ typical_acceptance_sampler_posterior_threshold,
1017
+ typical_acceptance_sampler_posterior_alpha=\
1018
+ typical_acceptance_sampler_posterior_alpha,
1019
)
1020
1021
@staticmethod
@@ -1072,6 +1097,9 @@ class SpeculativeConfig:
1097
speculative_disable_by_batch_size: Optional[int],
1098
ngram_prompt_lookup_max: Optional[int],
1099
ngram_prompt_lookup_min: Optional[int],
1100
+ draft_token_acceptance_method: str,
1101
+ typical_acceptance_sampler_posterior_threshold: float,
1102
+ typical_acceptance_sampler_posterior_alpha: float,
1103
):
1104
"""Create a SpeculativeConfig object.
1105
@@ -1085,6 +1113,19 @@ class SpeculativeConfig:
1113
enqueue requests is larger than this value.
1114
ngram_prompt_lookup_max: Max size of ngram token window.
1115
ngram_prompt_lookup_min: Min size of ngram token window.
1116
+ draft_token_acceptance_method (str): The method to use for
1117
+ accepting draft tokens. This can take two possible
1118
+ values 'rejection_sampler' and 'typical_acceptance_sampler'
1119
+ for RejectionSampler and TypicalAcceptanceSampler
1120
+ respectively.
1121
+ typical_acceptance_sampler_posterior_threshold (Optional[float]):
1122
+ A threshold value that sets a lower bound on the posterior
1123
+ probability of a token in the target model for it to be
1124
+ accepted. This threshold is used only when we use the
1125
+ TypicalAcceptanceSampler for token acceptance.
1126
+ typical_acceptance_sampler_posterior_alpha (Optional[float]):
1127
+ A scaling factor for the entropy-based threshold in the
1128
+ TypicalAcceptanceSampler.
1129
"""
1130
self.draft_model_config = draft_model_config
1131
self.draft_parallel_config = draft_parallel_config
@@ -1093,6 +1134,11 @@ class SpeculativeConfig:
1134
speculative_disable_by_batch_size
1135
self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0
1136
self.ngram_prompt_lookup_min = ngram_prompt_lookup_min or 0
1137
+ self.draft_token_acceptance_method = draft_token_acceptance_method
1138
+ self.typical_acceptance_sampler_posterior_threshold = \
1139
+ typical_acceptance_sampler_posterior_threshold
1140
+ self.typical_acceptance_sampler_posterior_alpha = \
1141
+ typical_acceptance_sampler_posterior_alpha
1142
1143
self._verify_args()
1144
@@ -1104,6 +1150,31 @@ class SpeculativeConfig:
1150
if self.draft_model_config:
1151
self.draft_model_config.verify_with_parallel_config(
1152
self.draft_parallel_config)
1153
+ # Validate and set draft token acceptance related settings.
1154
+
1155
+ if (self.draft_token_acceptance_method is None):
1156
+ raise ValueError("draft_token_acceptance_method is not set. "
1157
+ "Expected values are rejection_sampler or "
1158
+ "typical_acceptance_sampler.")
1159
+
1160
+ if (self.draft_token_acceptance_method != 'rejection_sampler'
1161
+ and self.draft_token_acceptance_method !=
1162
+ 'typical_acceptance_sampler'):
1163
+ raise ValueError(
1164
+ "Expected draft_token_acceptance_method to be either "
1165
+ "rejection_sampler or typical_acceptance_sampler. Instead it "
1166
+ f"is {self.draft_token_acceptance_method}")
1167
+
1168
+ if (self.typical_acceptance_sampler_posterior_threshold < 0
1169
+ or self.typical_acceptance_sampler_posterior_alpha < 0):
1170
+ raise ValueError(
1171
+ "Expected typical_acceptance_sampler_posterior_threshold "
1172
+ "and typical_acceptance_sampler_posterior_alpha to be > 0. "
1173
+ "Instead found "
1174
+ f"typical_acceptance_sampler_posterior_threshold = "
1175
+ f"{self.typical_acceptance_sampler_posterior_threshold} and "
1176
+ f"typical_acceptance_sampler_posterior_alpha = "
1177
+ f"{self.typical_acceptance_sampler_posterior_alpha}")
1178
1179
@property
1180
def num_lookahead_slots(self) -> int:
vllm/engine/arg_utils.py
+41
-1
@@ -100,7 +100,9 @@ class EngineArgs:
100
speculative_disable_by_batch_size: Optional[int] = None
101
ngram_prompt_lookup_max: Optional[int] = None
102
ngram_prompt_lookup_min: Optional[int] = None
103
-
103
+ spec_decoding_acceptance_method: str = 'rejection_sampler'
104
+ typical_acceptance_sampler_posterior_threshold: Optional[float] = None
105
+ typical_acceptance_sampler_posterior_alpha: Optional[float] = None
106
qlora_adapter_name_or_path: Optional[str] = None
107
108
otlp_traces_endpoint: Optional[str] = None
@@ -577,6 +579,38 @@ class EngineArgs:
579
help='Min size of window for ngram prompt lookup in speculative '
580
'decoding.')
581
582
+ parser.add_argument(
583
+ '--spec-decoding-acceptance-method',
584
+ type=str,
585
+ default=EngineArgs.spec_decoding_acceptance_method,
586
+ choices=['rejection_sampler', 'typical_acceptance_sampler'],
587
+ help='Specify the acceptance method to use during draft token '
588
+ 'verification in speculative decoding. Two types of acceptance '
589
+ 'routines are supported: '
590
+ '1) RejectionSampler which does not allow changing the '
591
+ 'acceptance rate of draft tokens, '
592
+ '2) TypicalAcceptanceSampler which is configurable, allowing for '
593
+ 'a higher acceptance rate at the cost of lower quality, '
594
+ 'and vice versa.')
595
+
596
+ parser.add_argument(
597
+ '--typical-acceptance-sampler-posterior-threshold',
598
+ type=float,
599
+ default=EngineArgs.typical_acceptance_sampler_posterior_threshold,
600
+ help='Set the lower bound threshold for the posterior '
601
+ 'probability of a token to be accepted. This threshold is '
602
+ 'used by the TypicalAcceptanceSampler to make sampling decisions '
603
+ 'during speculative decoding. Defaults to 0.09')
604
+
605
+ parser.add_argument(
606
+ '--typical-acceptance-sampler-posterior-alpha',
607
+ type=float,
608
+ default=EngineArgs.typical_acceptance_sampler_posterior_alpha,
609
+ help='A scaling factor for the entropy-based threshold for token '
610
+ 'acceptance in the TypicalAcceptanceSampler. Typically defaults '
611
+ 'to sqrt of --typical-acceptance-sampler-posterior-threshold '
612
+ 'i.e. 0.3')
613
+
614
parser.add_argument('--model-loader-extra-config',
615
type=nullable_str,
616
default=EngineArgs.model_loader_extra_config,
@@ -737,6 +771,12 @@ class EngineArgs:
771
use_v2_block_manager=self.use_v2_block_manager,
772
ngram_prompt_lookup_max=self.ngram_prompt_lookup_max,
773
ngram_prompt_lookup_min=self.ngram_prompt_lookup_min,
774
+ draft_token_acceptance_method=\
775
+ self.spec_decoding_acceptance_method,
776
+ typical_acceptance_sampler_posterior_threshold=self.
777
+ typical_acceptance_sampler_posterior_threshold,
778
+ typical_acceptance_sampler_posterior_alpha=self.
779
+ typical_acceptance_sampler_posterior_alpha,
780
)
781
782
scheduler_config = SchedulerConfig(
vllm/engine/metrics.py
+1
-1
@@ -457,4 +457,4 @@ class PrometheusStatLogger(StatLoggerBase):
457
458
class RayPrometheusStatLogger(PrometheusStatLogger):
459
"""RayPrometheusStatLogger uses Ray metrics instead."""
460
- _metrics_cls = RayMetrics
460
+ _metrics_cls = RayMetrics
\ No newline at end of file
vllm/model_executor/layers/rejection_sampler.py
+9
-9
@@ -3,13 +3,12 @@ from typing import Tuple
3
4
import torch
5
import torch.jit
6
-import torch.nn as nn
6
7
from vllm.model_executor.layers.spec_decode_base_sampler import (
8
SpecDecodeBaseSampler)
9
10
12
-class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
11
+class RejectionSampler(SpecDecodeBaseSampler):
12
"""Apply modified rejection sampling as described in "Accelerating Large
13
Language Model Decoding with Speculative Sampling"
14
https://arxiv.org/pdf/2302.01318.pdf.
@@ -28,8 +27,8 @@ class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
27
during sampling. This catches correctness issues but adds
28
nontrivial latency.
29
"""
31
- SpecDecodeBaseSampler.__init__(self, disable_bonus_tokens, strict_mode)
32
- nn.Module.__init__(self)
30
+ super().__init__(disable_bonus_tokens=disable_bonus_tokens,
31
+ strict_mode=strict_mode)
32
33
def forward(
34
self,
@@ -78,11 +77,12 @@ class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
77
self._raise_if_incorrect_input(target_probs, bonus_token_ids,
78
draft_probs, draft_token_ids)
79
81
- accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
82
- target_probs,
83
- draft_probs,
84
- draft_token_ids,
85
- )
80
+ accepted, recovered_token_ids = (
81
+ self._batch_modified_rejection_sampling(
82
+ target_probs,
83
+ draft_probs,
84
+ draft_token_ids,
85
+ ))
86
87
output_token_ids = self._create_output(
88
accepted,
vllm/model_executor/layers/spec_decode_base_sampler.py
+14
-1
@@ -1,9 +1,12 @@
1
+from abc import abstractmethod
2
from typing import Optional
3
4
import torch
5
+import torch.jit
6
+import torch.nn as nn
7
8
6
-class SpecDecodeBaseSampler():
9
+class SpecDecodeBaseSampler(nn.Module):
10
"""Base class for samplers used for Speculative Decoding verification
11
step.
12
"""
@@ -51,6 +54,16 @@ class SpecDecodeBaseSampler():
54
def token_id_dtype(self):
55
return torch.int64
56
57
+ @abstractmethod
58
+ def forward(
59
+ self,
60
+ target_probs: torch.Tensor,
61
+ bonus_token_ids: torch.Tensor,
62
+ draft_probs: torch.Tensor,
63
+ draft_token_ids: torch.Tensor,
64
+ ) -> torch.Tensor:
65
+ raise NotImplementedError
66
+
67
def _create_output(
68
self,
69
accepted: torch.Tensor, # [batch_size, k]
vllm/model_executor/layers/typical_acceptance_sampler.py
+10
-12
@@ -1,12 +1,11 @@
1
import torch
2
import torch.jit
3
-import torch.nn as nn
3
4
from vllm.model_executor.layers.spec_decode_base_sampler import (
5
SpecDecodeBaseSampler)
6
7
9
-class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
8
+class TypicalAcceptanceSampler(SpecDecodeBaseSampler):
9
"""Apply typical acceptance sampling as described in section 3.3.1 in
10
"MEDUSA: Simple LLM Inference Acceleration Framework with
11
Multiple Decoding Heads"
@@ -15,10 +14,10 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
14
15
def __init__(
16
self,
17
+ posterior_threshold: float,
18
+ posterior_alpha: float,
19
disable_bonus_tokens: bool = False,
20
strict_mode: bool = False,
20
- posterior_threshold: float = 0.09,
21
- posterior_alpha: float = 0.3,
21
):
22
"""Create a Typical Acceptance Sampler.
23
@@ -31,23 +30,20 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
30
nontrivial latency.
31
posterior_threshold : A threshold value that sets a lower bound
32
on the posterior probability of a token in target model for it
34
- to be accepted. Default is 0.09
33
+ to be accepted.
34
posterior_alpha : A scaling factor for the entropy-based
36
- threshold in typical acceptance sampling. Typically defaults to
37
- sqrt of posterior_threshold and is set to 0.3.
35
+ threshold in typical acceptance sampling.
36
"""
39
- SpecDecodeBaseSampler.__init__(
40
- self,
41
- disable_bonus_tokens=disable_bonus_tokens,
42
- strict_mode=strict_mode)
43
- nn.Module.__init__(self)
37
self._posterior_threshold = posterior_threshold
38
self._posterior_alpha = posterior_alpha
39
+ super().__init__(disable_bonus_tokens=disable_bonus_tokens,
40
+ strict_mode=strict_mode)
41
42
def forward(
43
self,
44
target_probs: torch.Tensor,
45
bonus_token_ids: torch.Tensor,
46
+ draft_probs: torch.Tensor,
47
draft_token_ids: torch.Tensor,
48
) -> torch.Tensor:
49
"""Sample token ids using typical acceptance sampling. This accepts
@@ -69,6 +65,8 @@ class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
65
speculative tokens in a sequence are accepted.
66
shape = [batch_size, num_bonus_tokens]
67
68
+ draft_probs: This parameter is unused by the acceptance sampler.
69
+
70
draft_token_ids: The token ids that were sampled from the draft
71
probabilities.
72
shape = [batch_size, num_speculative_tokens]
vllm/spec_decode/metrics.py
+13
-11
@@ -4,7 +4,8 @@ from typing import Callable, Optional
4
5
import torch
6
7
-from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7
+from vllm.model_executor.layers.spec_decode_base_sampler import (
8
+ SpecDecodeBaseSampler)
9
from vllm.utils import is_pin_memory_available
10
11
@@ -46,15 +47,15 @@ Timer = Callable[[], float]
47
48
49
class AsyncMetricsCollector:
49
- """Class which copies rejection sampler metrics from the device to CPU on a
50
- non-default Torch stream.
50
+ """Class which copies rejection/typical-acceptance sampler metrics
51
+ from the device to CPU on a non-default Torch stream.
52
"""
53
54
def __init__(self,
54
- rejection_sampler: RejectionSampler,
55
+ spec_decode_sampler: SpecDecodeBaseSampler,
56
timer: Optional[Timer] = None,
57
collect_interval_s: float = 5.0):
57
- self._rejection_sampler = rejection_sampler
58
+ self.spec_decode_sampler = spec_decode_sampler
59
self._timer = time.time if timer is None else timer
60
61
self._rank: Optional[int] = None
@@ -95,7 +96,7 @@ class AsyncMetricsCollector:
96
return None
97
98
def _should_collect_rejsample_metrics(self, now: float) -> bool:
98
- """Return whether or not this iteration should print rejection sampling
99
+ """Return whether or not this iteration should print sampling
100
metrics.
101
"""
102
if self._rank != 0:
@@ -107,8 +108,8 @@ class AsyncMetricsCollector:
108
return True
109
110
def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
110
- """Copy rejection sampling metrics (number of accepted tokens, etc) to
111
- CPU asynchronously.
111
+ """Copy rejection/typical-acceptance sampling metrics
112
+ (number of accepted tokens, etc) to CPU asynchronously.
113
114
Returns a CUDA event recording when the copy is complete.
115
"""
@@ -117,13 +118,14 @@ class AsyncMetricsCollector:
118
119
with torch.cuda.stream(self._copy_stream):
120
self._aggregate_num_accepted_tokens.copy_(
120
- self._rejection_sampler.num_accepted_tokens, non_blocking=True)
121
+ self.spec_decode_sampler.num_accepted_tokens,
122
+ non_blocking=True)
123
self._aggregate_num_emitted_tokens.copy_(
122
- self._rejection_sampler.num_emitted_tokens, non_blocking=True)
124
+ self.spec_decode_sampler.num_emitted_tokens, non_blocking=True)
125
# Number of draft tokens is calculated on CPU, so no copy is
126
# required.
127
self._aggregate_num_draft_tokens = (
126
- self._rejection_sampler.num_draft_tokens)
128
+ self.spec_decode_sampler.num_draft_tokens)
129
130
aggregate_metrics_ready = torch.cuda.Event()
131
aggregate_metrics_ready.record(self._copy_stream)
vllm/spec_decode/spec_decode_worker.py
+43
-19
@@ -7,6 +7,10 @@ from vllm.config import ParallelConfig, SpeculativeConfig
7
from vllm.distributed.communication_op import broadcast_tensor_dict
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10
+from vllm.model_executor.layers.spec_decode_base_sampler import (
11
+ SpecDecodeBaseSampler)
12
+from vllm.model_executor.layers.typical_acceptance_sampler import (
13
+ TypicalAcceptanceSampler)
14
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
15
HiddenStates, SamplerOutput, SequenceGroupMetadata,
16
get_all_seq_ids)
@@ -56,7 +60,12 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
60
draft_worker_kwargs=draft_worker_kwargs,
61
disable_by_batch_size=speculative_config.
62
speculative_disable_by_batch_size,
59
- )
63
+ draft_token_acceptance_method=speculative_config.
64
+ draft_token_acceptance_method,
65
+ typical_acceptance_sampler_posterior_threshold=speculative_config.
66
+ typical_acceptance_sampler_posterior_threshold,
67
+ typical_acceptance_sampler_posterior_alpha=speculative_config.
68
+ typical_acceptance_sampler_posterior_alpha)
69
70
return spec_decode_worker
71
@@ -78,8 +87,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
87
welcome!).
88
* Only top-1 proposal and scoring are implemented. Tree-attention is left as
89
future work.
81
- * Only lossless rejection sampling is supported. Contributions adding lossy
82
- verification routines are welcome (e.g. Medusa's typical acceptance).
90
* All sequences in a batch must have the same proposal length, or zero. This
91
can be improved by having per-sequence speculation in the future.
92
* The scoring forward pass is done without an MQA kernel, which is
@@ -95,6 +102,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
102
scorer_worker: Worker,
103
draft_worker_kwargs: Dict[str, Any],
104
disable_by_batch_size: Optional[int],
105
+ draft_token_acceptance_method: str,
106
+ typical_acceptance_sampler_posterior_threshold: float,
107
+ typical_acceptance_sampler_posterior_alpha: float,
108
) -> "SpecDecodeWorker":
109
110
ngram_prompt_lookup_max = (
@@ -127,17 +137,30 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
137
logger.info("Configuring SpecDecodeWorker with proposer=%s",
138
type(proposer_worker))
139
140
+ spec_decode_sampler: SpecDecodeBaseSampler = None
141
+ if draft_token_acceptance_method == "rejection_sampler":
142
+ spec_decode_sampler = RejectionSampler(
143
+ disable_bonus_tokens=disable_bonus_tokens, )
144
+ elif draft_token_acceptance_method == "typical_acceptance_sampler":
145
+ spec_decode_sampler = TypicalAcceptanceSampler(
146
+ disable_bonus_tokens=disable_bonus_tokens,
147
+ posterior_threshold=\
148
+ typical_acceptance_sampler_posterior_threshold,
149
+ posterior_alpha=typical_acceptance_sampler_posterior_alpha,
150
+ )
151
+ logger.info("Configuring SpecDecodeWorker with sampler=%s",
152
+ type(spec_decode_sampler))
153
+
154
return SpecDecodeWorker(proposer_worker,
155
scorer_worker,
156
disable_by_batch_size=disable_by_batch_size,
133
- rejection_sampler=RejectionSampler(
134
- disable_bonus_tokens=disable_bonus_tokens))
157
+ spec_decode_sampler=spec_decode_sampler)
158
159
def __init__(
160
self,
161
proposer_worker: ProposerWorkerBase,
162
scorer_worker: WorkerBase,
140
- rejection_sampler: RejectionSampler,
163
+ spec_decode_sampler: SpecDecodeBaseSampler,
164
metrics_collector: Optional[AsyncMetricsCollector] = None,
165
disable_by_batch_size: Optional[int] = None,
166
):
@@ -150,8 +173,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
173
scorer_worker: A worker that produces probabilities of speculative
174
tokens according to some base model. Typically a vanilla vLLM
175
Worker.
153
- rejection_sampler: A Torch module used to perform modified rejection
154
- sampling for speculative decoding.
176
+ spec_decode_sampler: A Torch module used to perform acceptance
177
+ sampling of the draft tokens in the verification step of
178
+ speculative decoding. Currently we support two different
179
+ types of sampler namely RejectionSampler and
180
+ TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
181
+ instance of RejectionSampler or TypicalAcceptanceSampler.
182
disable_by_batch_size: If the batch size is larger than this,
183
disable speculative decoding for new incoming requests.
184
metrics_collector: Helper class for collecting metrics; can be set
@@ -160,15 +187,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
187
self.proposer_worker = proposer_worker
188
self.scorer_worker = scorer_worker
189
self.disable_by_batch_size = disable_by_batch_size or float("inf")
163
- self.rejection_sampler = rejection_sampler
164
-
190
+ self.spec_decode_sampler = spec_decode_sampler
191
self._metrics = AsyncMetricsCollector(
166
- rejection_sampler
192
+ self.spec_decode_sampler
193
) if metrics_collector is None else metrics_collector
168
-
169
- self.probs_dtype = self.rejection_sampler.probs_dtype
170
- self.token_id_dtype = self.rejection_sampler.token_id_dtype
171
-
194
+ self.probs_dtype = self.spec_decode_sampler.probs_dtype
195
+ self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
196
# Lazy initiazliation.
197
self.scorer: SpeculativeScorer
198
@@ -189,7 +213,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
213
self.proposer_worker.load_model()
214
215
self._metrics.init_gpu_tensors(self.rank)
192
- self.rejection_sampler.init_gpu_tensors(self.rank)
216
+ self.spec_decode_sampler.init_gpu_tensors(self.rank)
217
+
218
self.scorer = BatchExpansionTop1Scorer(
219
scorer_worker=self.scorer_worker,
220
device=self.device,
@@ -203,7 +228,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
228
def _configure_model_sampler_for_spec_decode(self):
229
"""Configure model sampler to emit GPU tensors. This allows spec decode
230
to keep data on device without transferring to CPU and serializing,
206
- which significantly reduces overhead of rejection sampling.
231
+ which significantly reduces overhead of sampling during verification.
232
233
NOTE(cade): This breaks abstraction boundaries pretty badly. The better
234
design is to have the "move to CPU and serialize" sampling decision be
@@ -481,7 +506,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
506
# Get proposed tokens.
507
proposal_token_ids = proposals.proposal_token_ids[spec_indices]
508
484
- accepted_token_ids = self.rejection_sampler(
509
+ accepted_token_ids = self.spec_decode_sampler(
510
target_probs=proposal_verifier_probs,
511
bonus_token_ids=bonus_token_ids,
512
draft_probs=proposal_probs,
@@ -496,7 +521,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
521
accepted_token_ids = torch.cat(
522
[accepted_token_ids, non_spec_token_ids])
523
logprobs = proposal_scores.logprobs
499
-
524
# Rearrange so that results are in the order of the original seq group
525
# metadata.
526
accepted_token_ids[original_indices] = accepted_token_ids.clone()