A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[SpecDecode][Kernel] Flashinfer Rejection Sampling (#7244)
Lily Liu committed
Sep 1, 2024 at 21:23 UTC
e6a26ed0376f39c0ae99ee1af1e390087fc81f8a
9 files changed
+306
-109
Dockerfile
+1
-1
@@ -162,7 +162,7 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
162
163
RUN --mount=type=cache,target=/root/.cache/pip \
164
. /etc/environment && \
165
- python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
165
+ python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.6/flashinfer-0.1.6+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
166
#################### vLLM installation IMAGE ####################
167
168
tests/samplers/test_rejection_sampler.py
+97
-19
@@ -44,12 +44,16 @@ def mock_causal_accepted_tensor(
44
["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
45
@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
46
@pytest.mark.parametrize("device", CUDA_DEVICES)
47
+@pytest.mark.parametrize("use_flashinfer", [True, False])
48
@torch.inference_mode()
48
-def test_correct_output_format(which_tokens_accepted: str,
49
- disable_bonus_tokens: bool, seed: int,
50
- device: str):
49
+def test_correct_output_format(which_tokens_accepted: str, seed: int,
50
+ disable_bonus_tokens: bool, device: str,
51
+ use_flashinfer: bool):
52
"""Verify the output has correct format given predetermined accepted matrix.
53
"""
54
+ if use_flashinfer and disable_bonus_tokens:
55
+ pytest.skip("Flashinfer rejection sampler must enable bonus token.")
56
+
57
set_random_seed(seed)
58
torch.set_default_device(device)
59
@@ -85,7 +89,8 @@ def test_correct_output_format(which_tokens_accepted: str,
89
dtype=torch.int64)
90
91
rejection_sampler = RejectionSampler(
88
- disable_bonus_tokens=disable_bonus_tokens)
92
+ disable_bonus_tokens=disable_bonus_tokens,
93
+ use_flashinfer=use_flashinfer)
94
rejection_sampler.init_gpu_tensors(device=device)
95
output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
96
accepted,
@@ -133,15 +138,20 @@ def test_correct_output_format(which_tokens_accepted: str,
138
@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
139
@pytest.mark.parametrize("batch_size", list(range(1, 32)))
140
@pytest.mark.parametrize("device", CUDA_DEVICES)
141
+@pytest.mark.parametrize("use_flashinfer", [True, False])
142
@torch.inference_mode()
143
def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
138
- device: str):
144
+ device: str, use_flashinfer: bool):
145
torch.set_default_device(device)
140
- rejection_sampler = RejectionSampler()
146
+ rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
147
+ use_flashinfer=use_flashinfer)
148
rejection_sampler.init_gpu_tensors(device=device)
149
150
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
144
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
151
+ target_probs = torch.rand(batch_size,
152
+ k + 1,
153
+ vocab_size,
154
+ dtype=torch.float32)
155
bonus_token_ids = torch.randint(low=0,
156
high=vocab_size,
157
size=(batch_size, 1),
@@ -161,16 +171,21 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
171
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
172
@pytest.mark.parametrize("n_rep", [100])
173
@pytest.mark.parametrize("device", CUDA_DEVICES)
174
+@pytest.mark.parametrize("use_flashinfer", [True, False])
175
@torch.inference_mode()
176
def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
166
- frac_seeded: float, n_rep: int,
167
- device: str):
177
+ frac_seeded: float, n_rep: int, device: str,
178
+ use_flashinfer: bool):
179
torch.set_default_device(device)
169
- rejection_sampler = RejectionSampler()
180
+ rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
181
+ use_flashinfer=use_flashinfer)
182
rejection_sampler.init_gpu_tensors(device=device)
183
184
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
173
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
185
+ target_probs = torch.rand(batch_size,
186
+ k + 1,
187
+ vocab_size,
188
+ dtype=torch.float32)
189
bonus_token_ids = torch.randint(low=0,
190
high=vocab_size,
191
size=(batch_size, 1),
@@ -198,23 +213,85 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
213
assert torch.equal(results[j][i], results[0][i])
214
215
216
+@pytest.mark.parametrize("k", [1, 3, 6])
217
+@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
218
+@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
219
+@pytest.mark.parametrize("device", CUDA_DEVICES)
220
+@torch.inference_mode()
221
+def test_compare_nonflashinfer_backend(k: int, vocab_size: int,
222
+ batch_size: int, device: str):
223
+ """
224
+ Test the flashinfer and nonflashinfer backend generate
225
+ the same output metrics.
226
+ """
227
+ torch.set_default_device(device)
228
+ torch.manual_seed(0)
229
+ draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
230
+ target_probs = torch.rand(batch_size,
231
+ k + 1,
232
+ vocab_size,
233
+ dtype=torch.float32)
234
+ bonus_token_ids = torch.randint(low=0,
235
+ high=vocab_size,
236
+ size=(batch_size, 1),
237
+ dtype=torch.int64)
238
+ draft_token_ids = torch.randint(low=0,
239
+ high=vocab_size,
240
+ size=(batch_size, k),
241
+ dtype=torch.int64)
242
+
243
+ num_accepted_tokens = []
244
+ num_emitted_tokens = []
245
+ num_draft_tokens = []
246
+
247
+ def get_seeded_seqs():
248
+ return {
249
+ i: torch.Generator(device=device).manual_seed(i)
250
+ for i in range(batch_size)
251
+ }
252
+
253
+ for use_flashinfer in [True, False]:
254
+ rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
255
+ use_flashinfer=use_flashinfer)
256
+ rejection_sampler.init_gpu_tensors(device=device)
257
+ # We use seeded sequences to ensure the same tokens are accepted
258
+ # for both flashinfer and nonflashinfer backends.
259
+ seeded_seqs = get_seeded_seqs()
260
+ rejection_sampler(target_probs, bonus_token_ids, draft_probs,
261
+ draft_token_ids, seeded_seqs)
262
+ num_accepted_tokens.append(rejection_sampler.num_accepted_tokens)
263
+ num_emitted_tokens.append(rejection_sampler.num_emitted_tokens)
264
+ num_draft_tokens.append(rejection_sampler.num_draft_tokens)
265
+
266
+ assert num_accepted_tokens[0] == num_accepted_tokens[1]
267
+ assert num_emitted_tokens[0] == num_emitted_tokens[1]
268
+ assert num_draft_tokens[0] == num_draft_tokens[1]
269
+
270
+
271
@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
272
@pytest.mark.parametrize("which_token_ids",
273
["bonus_token_ids", "draft_token_ids"])
274
@pytest.mark.parametrize("device", CUDA_DEVICES)
275
+@pytest.mark.parametrize("use_flashinfer", [True, False])
276
@torch.inference_mode()
277
def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
207
- which_token_ids: str, device: str):
278
+ which_token_ids: str, device: str,
279
+ use_flashinfer: bool):
280
k = 3
281
batch_size = 5
282
vocab_size = 30_000
283
torch.set_default_device(device)
284
213
- rejection_sampler = RejectionSampler(strict_mode=True)
285
+ rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
286
+ use_flashinfer=use_flashinfer,
287
+ strict_mode=True)
288
rejection_sampler.init_gpu_tensors(device=device)
289
290
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
217
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
291
+ target_probs = torch.rand(batch_size,
292
+ k + 1,
293
+ vocab_size,
294
+ dtype=torch.float32)
295
bonus_token_ids = torch.randint(low=0,
296
high=vocab_size,
297
size=(batch_size, 1),
@@ -248,9 +325,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
325
326
@pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
327
@pytest.mark.parametrize("seed", list(range(5)))
328
+@pytest.mark.parametrize("use_flashinfer", [True, False])
329
@torch.inference_mode()
330
def test_rejection_sampling_approximates_target_distribution(
253
- seed: int, draft_and_target_probs_equal: bool):
331
+ seed: int, draft_and_target_probs_equal: bool, use_flashinfer: bool):
332
"""Verify rejection sampling approximates target distribution,
333
despite sampling from a potentially distinct draft distribution.
334
@@ -279,10 +357,10 @@ def test_rejection_sampling_approximates_target_distribution(
357
"""
358
torch.set_default_device("cpu")
359
set_random_seed(seed)
282
-
360
helper = _CorrectnessTestHelper(
361
vocab_size=10,
285
- rejection_sampler=RejectionSampler(),
362
+ rejection_sampler=RejectionSampler(disable_bonus_tokens=False,
363
+ use_flashinfer=use_flashinfer),
364
)
365
366
draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
@@ -398,10 +476,10 @@ class _CorrectnessTestHelper:
476
draft_probs = draft_probs.reshape(1, self.k, self.vocab_size).repeat(
477
num_samples, 1, 1)
478
401
- # Repeat target probs num_samples * k times.
479
+ # Repeat target probs num_samples * (k + 1) times.
480
# Rejection sampler requires bonus token probs, but they aren't used.
481
target_probs = target_probs.reshape(1, 1, self.vocab_size).repeat(
404
- num_samples, self.k, 1)
482
+ num_samples, self.k + 1, 1)
483
484
# Randomly sample draft token ids from draft probs.
485
draft_token_ids = torch.multinomial(draft_probs[:, 0, :],
tests/samplers/test_typical_acceptance_sampler.py
+32
-18
@@ -79,7 +79,10 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
79
torch.set_default_device(device)
80
typical_acceptance_sampler = get_acceptance_sampler()
81
typical_acceptance_sampler.init_gpu_tensors(device=device)
82
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
82
+ target_with_bonus_probs = torch.rand(batch_size,
83
+ k + 1,
84
+ vocab_size,
85
+ dtype=torch.float32)
86
bonus_token_ids = torch.randint(low=0,
87
high=vocab_size,
88
size=(batch_size, 1),
@@ -89,7 +92,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
92
size=(batch_size, k),
93
dtype=torch.int64)
94
# Verify that sampling succeeds for all cases.
92
- typical_acceptance_sampler(target_probs,
95
+ typical_acceptance_sampler(target_with_bonus_probs,
96
bonus_token_ids,
97
draft_probs=None,
98
draft_token_ids=draft_token_ids)
@@ -112,7 +115,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
115
torch.set_default_device(device)
116
typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
117
typical_acceptance_sampler.init_gpu_tensors(device=device)
115
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
118
+ target_with_bonus_probs = torch.rand(batch_size,
119
+ k + 1,
120
+ vocab_size,
121
+ dtype=torch.float32)
122
bonus_token_ids = torch.randint(low=0,
123
high=vocab_size,
124
size=(batch_size, 1),
@@ -141,7 +147,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
147
oob_token_ids[0][0] = rogue_token_id
148
149
with pytest.raises(AssertionError):
144
- typical_acceptance_sampler(target_probs,
150
+ typical_acceptance_sampler(target_with_bonus_probs,
151
bonus_token_ids,
152
draft_probs=None,
153
draft_token_ids=draft_token_ids)
@@ -172,7 +178,10 @@ def test_uniform_target_distribution_accepts_all_tokens(
178
typical_acceptance_sampler = get_acceptance_sampler(
179
strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
180
typical_acceptance_sampler.init_gpu_tensors(device=device)
175
- target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
181
+ target_with_bonus_probs = torch.rand(batch_size,
182
+ k + 1,
183
+ vocab_size,
184
+ dtype=torch.float32)
185
draft_token_ids = torch.randint(low=0,
186
high=vocab_size,
187
size=(batch_size, k),
@@ -182,7 +191,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
191
size=(batch_size, 1),
192
dtype=torch.int64)
193
output_token_ids = typical_acceptance_sampler(
185
- target_probs,
194
+ target_with_bonus_probs,
195
bonus_token_ids,
196
draft_probs=None,
197
draft_token_ids=draft_token_ids)
@@ -229,8 +238,9 @@ def test_temperature_zero_target_distribution(seed: int,
238
# Simulate temperature 0 probability distribution for target probabilities
239
# and create target probabilities such that only 1 token id has
240
# probability 1.0
232
- target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
233
- batch_size, k, vocab_size)
241
+ target_with_bonus_probs, zero_temperature_token_ids = \
242
+ get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
243
+ zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
244
# Populate draft_token_ids such that they exclude the token_ids
245
# with probability = 1.0
246
draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
@@ -245,7 +255,7 @@ def test_temperature_zero_target_distribution(seed: int,
255
# fallback to the greedy sampling for selecting 1 token for each sequence.
256
# Verify the same.
257
output_token_ids = typical_acceptance_sampler(
248
- target_probs,
258
+ target_with_bonus_probs,
259
bonus_token_ids,
260
draft_probs=None,
261
draft_token_ids=draft_token_ids)
@@ -289,8 +299,10 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
299
# For sequences 0 and 2 set the distribution to a temperature
300
# zero distribution. For sequences 1 and 3 set it to a uniform
301
# distribution.
292
- target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
293
- batch_size, k, vocab_size))
302
+ target_with_bonus_probs, zero_temperature_token_ids = \
303
+ get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
304
+ zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
305
+ target_probs = target_with_bonus_probs[:, :-1]
306
draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
307
zero_temperature_token_ids)
308
uniform_probs = torch.rand(2, k, vocab_size, dtype=torch.float32)
@@ -300,7 +312,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
312
size=(batch_size, 1),
313
dtype=torch.int64)
314
output_token_ids = typical_acceptance_sampler(
303
- target_probs,
315
+ target_with_bonus_probs,
316
bonus_token_ids,
317
draft_probs=None,
318
draft_token_ids=draft_token_ids)
@@ -356,15 +368,16 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
368
# Create a temperature zero target probability distribution and ensure
369
# all draft token ids correspond to the tokens with 1.0 probability.
370
# Verify that all of them are accepted.
359
- target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
360
- batch_size, k, vocab_size))
371
+ target_with_bonus_probs, zero_temperature_token_ids = \
372
+ get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
373
+ zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
374
draft_token_ids = zero_temperature_token_ids
375
bonus_token_ids = torch.randint(low=0,
376
high=vocab_size,
377
size=(batch_size, 1),
378
dtype=torch.int64)
379
output_token_ids = typical_acceptance_sampler(
367
- target_probs,
380
+ target_with_bonus_probs,
381
bonus_token_ids,
382
draft_probs=None,
383
draft_token_ids=draft_token_ids)
@@ -384,7 +397,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
397
draft_token_ids = torch.cat(
398
(draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
399
output_token_ids = typical_acceptance_sampler(
387
- target_probs,
400
+ target_with_bonus_probs,
401
bonus_token_ids,
402
draft_probs=None,
403
draft_token_ids=draft_token_ids)
@@ -421,8 +434,9 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
434
# 0.00001. Populate draft_token_ids such that they exclude the token_ids
435
# with probability = 1.0. Without any changes to the posterior thresholds
436
# none of the draft tokens are accepted.
424
- target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
425
- batch_size, k, vocab_size))
437
+ target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
438
+ batch_size, k + 1, vocab_size)
439
+ zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
440
target_probs[target_probs == 0] = 0.00001
441
draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
442
zero_temperature_token_ids)
tests/spec_decode/test_spec_decode_worker.py
+2
-3
@@ -230,9 +230,8 @@ def test_correctly_calls_spec_decode_sampler(k: int, batch_size: int,
230
231
assert torch.equal(actual.bonus_token_ids,
232
target_token_ids.reshape(batch_size, k + 1)[:, -1:])
233
- assert torch.equal(
234
- actual.target_probs,
235
- target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
233
+ assert torch.equal(actual.target_with_bonus_probs,
234
+ target_token_probs.reshape(batch_size, k + 1, -1))
235
assert torch.equal(actual.draft_token_ids, proposal_token_ids)
236
assert torch.equal(actual.draft_probs, proposal_probs)
237
vllm/envs.py
+1
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
31
VLLM_TRACE_FUNCTION: int = 0
32
VLLM_ATTENTION_BACKEND: Optional[str] = None
33
VLLM_USE_FLASHINFER_SAMPLER: bool = False
34
+ VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35
VLLM_PP_LAYER_PARTITION: Optional[str] = None
36
VLLM_CPU_KVCACHE_SPACE: int = 0
37
VLLM_CPU_OMP_THREADS_BIND: str = ""
vllm/model_executor/layers/rejection_sampler.py
+141
-43
@@ -1,12 +1,28 @@
1
from functools import cached_property
2
+from importlib.util import find_spec
3
from typing import Dict, List, Optional, Tuple
4
5
import torch
6
import torch.jit
7
8
+import vllm.envs as envs
9
+from vllm.logger import init_logger
10
from vllm.model_executor.layers.spec_decode_base_sampler import (
11
SpecDecodeStochasticBaseSampler)
12
13
+logger = init_logger(__name__)
14
+
15
+if find_spec("flashinfer"):
16
+ """
17
+ Consider utilizing the FlashInfer rejection sampling kernel initially,
18
+ as it employs a dedicated kernel rather than relying on
19
+ Torch tensor operations. This design choice helps to fuse operations,
20
+ reduce memory I/O, and consequently enhances performance.
21
+ """
22
+ from flashinfer.sampling import chain_speculative_sampling
23
+else:
24
+ chain_speculative_sampling = None
25
+
26
27
class RejectionSampler(SpecDecodeStochasticBaseSampler):
28
"""Apply modified rejection sampling as described in "Accelerating Large
@@ -16,7 +32,8 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
32
33
def __init__(self,
34
disable_bonus_tokens: bool = True,
19
- strict_mode: bool = False):
35
+ strict_mode: bool = False,
36
+ use_flashinfer: Optional[bool] = None):
37
"""Create a rejection sampler.
38
39
Args:
@@ -26,13 +43,29 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
43
strict_mode: Whether or not to perform shape/device/dtype checks
44
during sampling. This catches correctness issues but adds
45
nontrivial latency.
46
+ use_falshinfer: We will use this parameter to determine whether
47
+ to use the FlashInfer rejection sampling kernel or not. If it's
48
+ None, we will use the default value from the environment variable.
49
+ This parameter is only used for testing purposes.
50
"""
51
super().__init__(disable_bonus_tokens=disable_bonus_tokens,
52
strict_mode=strict_mode)
53
+ if use_flashinfer is None:
54
+ self.use_flashinfer = envs.VLLM_USE_FLASHINFER_SAMPLER and (
55
+ chain_speculative_sampling is not None)
56
+ else:
57
+ self.use_flashinfer = use_flashinfer
58
+
59
+ if self.use_flashinfer:
60
+ assert not disable_bonus_tokens, \
61
+ "flashinfer will enable bonus token by default"
62
+ logger.info("Use flashinfer for rejection sampling.")
63
+ else:
64
+ logger.info("Use pytorch for rejection sampling.")
65
66
def forward(
67
self,
35
- target_probs: torch.Tensor,
68
+ target_with_bonus_probs: torch.Tensor,
69
bonus_token_ids: torch.Tensor,
70
draft_probs: torch.Tensor,
71
draft_token_ids: torch.Tensor,
@@ -50,9 +83,9 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
83
sequence.
84
85
Args:
53
- target_probs: The probability distribution over token ids given
54
- context according to the target model.
55
- shape = [batch_size, num_speculative_tokens, vocab_size]
86
+ target_with_bonus_probs: The probability distribution
87
+ over token ids given context according to the target model.
88
+ shape = [batch_size, num_speculative_tokens + 1, vocab_size]
89
90
bonus_token_ids: The "bonus" token ids that are accepted iff all
91
speculative tokens in a sequence are accepted.
@@ -78,23 +111,52 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
111
# Only perform shape/dtype/device checking in strict mode, as it adds
112
# overhead.
113
if self._strict_mode:
81
- self._raise_if_incorrect_input(target_probs, draft_token_ids,
82
- bonus_token_ids, draft_probs)
114
+ self._raise_if_incorrect_input(target_with_bonus_probs,
115
+ draft_token_ids, bonus_token_ids,
116
+ draft_probs)
117
84
- accepted, recovered_token_ids = (
85
- self._batch_modified_rejection_sampling(
86
- target_probs,
87
- draft_probs,
88
- draft_token_ids,
89
- seeded_seqs,
90
- ))
118
+ batch_size, k, _ = draft_probs.shape
119
92
- output_token_ids = self._create_output(
93
- accepted,
94
- recovered_token_ids,
95
- draft_token_ids,
96
- bonus_token_ids,
97
- )
120
+ # batch_size = 0 when all requests in the batch are
121
+ # non_spec requests. In this case, output_token_ids is
122
+ # just an empty tensor.
123
+ if batch_size == 0:
124
+ return torch.empty(0, k + 1, device=draft_probs.device, dtype=int)
125
+
126
+ # If use Flashinfer chain_speculative_sampling kernel
127
+ # for rejection sampling
128
+ if self.use_flashinfer:
129
+ batch_size, k, _ = draft_probs.shape
130
+ uniform_samples = self._create_uniform_samples(
131
+ seeded_seqs, batch_size, k, draft_probs.device)
132
+ output_token_ids, accepted_token_num, emitted_token_num \
133
+ = chain_speculative_sampling(
134
+ draft_probs, draft_token_ids, uniform_samples,
135
+ target_with_bonus_probs)
136
+
137
+ # num_emitted_tokens returned by flashinfer
138
+ # does not include the bonus token
139
+ # Flashinfer stops at the first token that violates
140
+ # the condition p >= q and does not include recovery/bonus token.
141
+ # Therefore, we need to add batch_size here.
142
+ self.num_accepted_tokens += accepted_token_num.sum()
143
+ self.num_emitted_tokens += emitted_token_num.sum() + batch_size
144
+ self.num_draft_tokens += batch_size * k
145
+ else:
146
+ accepted, recovered_token_ids = (
147
+ self._batch_modified_rejection_sampling(
148
+ target_with_bonus_probs[:, :-1],
149
+ draft_probs,
150
+ draft_token_ids,
151
+ seeded_seqs,
152
+ ))
153
+
154
+ output_token_ids = self._create_output(
155
+ accepted,
156
+ recovered_token_ids,
157
+ draft_token_ids,
158
+ bonus_token_ids,
159
+ )
160
161
return output_token_ids
162
@@ -135,6 +197,63 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
197
198
return accepted, recovered_token_ids
199
200
+ def _create_uniform_samples(self,
201
+ seeded_seqs: Optional[Dict[int,
202
+ torch.Generator]],
203
+ batch_size: int, k: int,
204
+ device: torch.device) -> torch.Tensor:
205
+ """
206
+ Generates a batch of uniform random samples, with optional seeding
207
+ for specific sequences.
208
+
209
+ This method creates a tensor of shape `(batch_size, k + 1)` filled
210
+ with uniform random values in the range [0, 1). If `seeded_seqs`
211
+ is provided, the sequences corresponding to specific indices
212
+ will be generated using the provided `torch.Generator` for
213
+ reproducibility. The other sequences will be generated without
214
+ a seed.
215
+
216
+ Args:
217
+ seeded_seqs : Optional[Dict[int, torch.Generator]]
218
+ A dictionary mapping indices in the batch to
219
+ `torch.Generator` objects. If `None`, all samples are
220
+ generated without a seed.
221
+ batch_size : int
222
+ The number of sequences to generate.
223
+ k : int
224
+ The number of random samples per sequence.
225
+ device : torch.device
226
+ The device on which to allocate the tensor.
227
+
228
+ Returns:
229
+ uniform_rand : torch.Tensor
230
+ A tensor of shape `(batch_size, k + 1)` containing uniform
231
+ random values in the range [0, 1).
232
+ """
233
+ if not seeded_seqs:
234
+ return torch.rand(batch_size, k + 1, device=device)
235
+
236
+ uniform_rand = torch.empty(batch_size, k + 1, device=device)
237
+
238
+ non_seeded_indices = []
239
+ for idx in range(batch_size):
240
+ generator = seeded_seqs.get(idx)
241
+ if generator is None:
242
+ non_seeded_indices.append(idx)
243
+ else:
244
+ uniform_rand[idx, :] = torch.rand(1,
245
+ k + 1,
246
+ dtype=self.probs_dtype,
247
+ device=device,
248
+ generator=generator)
249
+ if non_seeded_indices:
250
+ uniform_rand[non_seeded_indices, :] = torch.rand(
251
+ len(non_seeded_indices),
252
+ k + 1,
253
+ dtype=self.probs_dtype,
254
+ device=device)
255
+ return uniform_rand
256
+
257
def _get_accepted(
258
self,
259
target_probs: torch.Tensor, # [batch_size, k, vocab_size]
@@ -175,29 +294,8 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
294
selected_target_probs = target_probs[batch_indices, probs_indicies,
295
draft_token_ids]
296
178
- if not seeded_seqs:
179
- uniform_rand = torch.rand_like(selected_target_probs)
180
- else:
181
- uniform_rand = torch.empty_like(selected_target_probs)
182
-
183
- non_seeded_indices = []
184
- for idx in range(batch_size):
185
- generator = seeded_seqs.get(idx)
186
- if generator is None:
187
- non_seeded_indices.append(idx)
188
- else:
189
- uniform_rand[idx, :] = torch.rand(
190
- 1,
191
- k,
192
- dtype=self.probs_dtype,
193
- device=target_probs.device,
194
- generator=generator)
195
- if non_seeded_indices:
196
- uniform_rand[non_seeded_indices, :] = torch.rand(
197
- len(non_seeded_indices),
198
- k,
199
- dtype=self.probs_dtype,
200
- device=target_probs.device)
297
+ uniform_rand = self._create_uniform_samples(seeded_seqs, batch_size,
298
+ k - 1, target_probs.device)
299
300
capped_ratio = torch.minimum(
301
selected_target_probs / selected_draft_probs,
vllm/model_executor/layers/spec_decode_base_sampler.py
+25
-18
@@ -130,29 +130,35 @@ class SpecDecodeBaseSampler(nn.Module):
130
131
def _raise_if_incorrect_input(
132
self,
133
- target_probs: torch.Tensor,
133
+ target_with_bonus_probs: torch.Tensor,
134
draft_token_ids: torch.Tensor,
135
bonus_token_ids: torch.Tensor,
136
draft_probs: Optional[torch.Tensor] = None,
137
) -> None:
138
- self._raise_if_incorrect_shape(target_probs, draft_token_ids,
139
- bonus_token_ids, draft_probs)
140
- self._raise_if_incorrect_dtype(target_probs, draft_token_ids,
141
- bonus_token_ids, draft_probs)
142
- self._raise_if_inconsistent_device(target_probs, draft_token_ids,
143
- bonus_token_ids, draft_probs)
144
- self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
138
+ self._raise_if_incorrect_shape(target_with_bonus_probs,
139
+ draft_token_ids, bonus_token_ids,
140
+ draft_probs)
141
+ self._raise_if_incorrect_dtype(target_with_bonus_probs,
142
+ draft_token_ids, bonus_token_ids,
143
+ draft_probs)
144
+ self._raise_if_inconsistent_device(target_with_bonus_probs,
145
+ draft_token_ids, bonus_token_ids,
146
+ draft_probs)
147
+ self._raise_if_out_of_bounds_vocab(target_with_bonus_probs.shape[-1],
148
draft_token_ids, bonus_token_ids)
149
150
def _raise_if_incorrect_shape(
151
self,
149
- target_probs: torch.Tensor,
152
+ target_with_bonus_probs: torch.Tensor,
153
draft_token_ids: torch.Tensor,
154
bonus_token_ids: torch.Tensor,
155
draft_probs: Optional[torch.Tensor] = None,
156
) -> None:
157
(target_batch_size, num_target_probs,
155
- target_vocab_size) = target_probs.shape
158
+ target_vocab_size) = target_with_bonus_probs.shape
159
+
160
+ # Does not count the extra token
161
+ num_target_probs -= 1
162
163
# validate the shape of draft token ids.
164
draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
@@ -175,12 +181,12 @@ class SpecDecodeBaseSampler(nn.Module):
181
182
def _raise_if_incorrect_dtype(
183
self,
178
- target_probs: torch.Tensor,
184
+ target_with_bonus_probs: torch.Tensor,
185
draft_token_ids: torch.Tensor,
186
bonus_token_ids: torch.Tensor,
187
draft_probs: Optional[torch.Tensor] = None,
188
) -> None:
183
- assert target_probs.dtype == self.probs_dtype
189
+ assert target_with_bonus_probs.dtype == self.probs_dtype
190
assert draft_token_ids.dtype == self.token_id_dtype
191
assert bonus_token_ids.dtype == self.token_id_dtype
192
if draft_probs is not None:
@@ -188,15 +194,16 @@ class SpecDecodeBaseSampler(nn.Module):
194
195
def _raise_if_inconsistent_device(
196
self,
191
- target_probs: torch.Tensor,
197
+ target_with_bonus_probs: torch.Tensor,
198
draft_token_ids: torch.Tensor,
199
bonus_token_ids: torch.Tensor,
200
draft_probs: Optional[torch.Tensor] = None,
201
) -> None:
202
devices = [
197
- t.device for t in
198
- [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
199
- if t is not None
203
+ t.device for t in [
204
+ target_with_bonus_probs, bonus_token_ids, draft_probs,
205
+ draft_token_ids
206
+ ] if t is not None
207
]
208
assert all([devices[0] == device for device in devices])
209
@@ -220,7 +227,7 @@ class SpecDecodeDeterministicBaseSampler(SpecDecodeBaseSampler):
227
@abstractmethod
228
def forward(
229
self,
223
- target_probs: torch.Tensor,
230
+ target_with_bonus_probs: torch.Tensor,
231
bonus_token_ids: torch.Tensor,
232
draft_probs: torch.Tensor,
233
draft_token_ids: torch.Tensor,
@@ -236,7 +243,7 @@ class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
243
@abstractmethod
244
def forward(
245
self,
239
- target_probs: torch.Tensor,
246
+ target_with_bonus_probs: torch.Tensor,
247
bonus_token_ids: torch.Tensor,
248
draft_probs: torch.Tensor,
249
draft_token_ids: torch.Tensor,
vllm/model_executor/layers/typical_acceptance_sampler.py
+4
-3
@@ -41,7 +41,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
41
42
def forward(
43
self,
44
- target_probs: torch.Tensor,
44
+ target_with_bonus_probs: torch.Tensor,
45
bonus_token_ids: torch.Tensor,
46
draft_probs: torch.Tensor,
47
draft_token_ids: torch.Tensor,
@@ -80,8 +80,9 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
80
# Only perform shape/dtype/device checking in strict mode, as it adds
81
# overhead.
82
if self._strict_mode:
83
- self._raise_if_incorrect_input(target_probs, draft_token_ids,
84
- bonus_token_ids)
83
+ self._raise_if_incorrect_input(target_with_bonus_probs,
84
+ draft_token_ids, bonus_token_ids)
85
+ target_probs = target_with_bonus_probs[:, :-1]
86
accepted = self._evaluate_accepted_tokens(target_probs,
87
draft_token_ids)
88
recovered_token_ids = self._replacement_token_ids(target_probs)
vllm/spec_decode/spec_decode_worker.py
+3
-4
@@ -625,8 +625,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
625
seq_group_metadata_list, proposal_lens_list)
626
original_indices = spec_indices + non_spec_indices
627
628
- # Get probabilities of target model, excluding bonus token.
629
- proposal_verifier_probs = proposal_scores.probs[spec_indices, :-1]
628
+ # Get probabilities of target model, including bonus tokens.
629
+ proposal_verifier_probs = proposal_scores.probs[spec_indices]
630
631
# Get non-speculative sampled tokens from target model.
632
non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
@@ -651,13 +651,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
651
}
652
653
accepted_token_ids = self.spec_decode_sampler(
654
- target_probs=proposal_verifier_probs,
654
+ target_with_bonus_probs=proposal_verifier_probs,
655
bonus_token_ids=bonus_token_ids,
656
draft_probs=proposal_probs,
657
draft_token_ids=proposal_token_ids,
658
**sampler_extra_kwargs,
659
)
660
-
660
# Append output tokens from non-speculative sequences to
661
# the accepted token ids tensor.
662
non_spec_token_ids = non_spec_token_ids.expand(-1, max_proposal_len +