@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e6a26ed037
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[SpecDecode][Kernel] Flashinfer Rejection Sampling (#7244)

Lily Liu committed Sep 1, 2024 at 21:23 UTC e6a26ed0376f39c0ae99ee1af1e390087fc81f8a
9 files changed +306 -109
Dockerfile
+1 -1
@@ -162,7 +162,7 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
162
163 RUN --mount=type=cache,target=/root/.cache/pip \
164 . /etc/environment && \
165 - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
165 + python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.6/flashinfer-0.1.6+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
166 #################### vLLM installation IMAGE ####################
167
168
tests/samplers/test_rejection_sampler.py
+97 -19
@@ -44,12 +44,16 @@ def mock_causal_accepted_tensor(
44 ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
45 @pytest.mark.parametrize("disable_bonus_tokens", [True, False])
46 @pytest.mark.parametrize("device", CUDA_DEVICES)
47 +@pytest.mark.parametrize("use_flashinfer", [True, False])
48 @torch.inference_mode()
48 -def test_correct_output_format(which_tokens_accepted: str,
49 - disable_bonus_tokens: bool, seed: int,
50 - device: str):
49 +def test_correct_output_format(which_tokens_accepted: str, seed: int,
50 + disable_bonus_tokens: bool, device: str,
51 + use_flashinfer: bool):
52 """Verify the output has correct format given predetermined accepted matrix.
53 """
54 + if use_flashinfer and disable_bonus_tokens:
55 + pytest.skip("Flashinfer rejection sampler must enable bonus token.")
56 +
57 set_random_seed(seed)
58 torch.set_default_device(device)
59
@@ -85,7 +89,8 @@ def test_correct_output_format(which_tokens_accepted: str,
89 dtype=torch.int64)
90
91 rejection_sampler = RejectionSampler(
88 - disable_bonus_tokens=disable_bonus_tokens)
92 + disable_bonus_tokens=disable_bonus_tokens,
93 + use_flashinfer=use_flashinfer)
94 rejection_sampler.init_gpu_tensors(device=device)
95 output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
96 accepted,
@@ -133,15 +138,20 @@ def test_correct_output_format(which_tokens_accepted: str,
138 @pytest.mark.parametrize("vocab_size", [30_000, 50_000])
139 @pytest.mark.parametrize("batch_size", list(range(1, 32)))
140 @pytest.mark.parametrize("device", CUDA_DEVICES)
141 +@pytest.mark.parametrize("use_flashinfer", [True, False])
142 @torch.inference_mode()
143 def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
138 - device: str):
144 + device: str, use_flashinfer: bool):
145 torch.set_default_device(device)
140 - rejection_sampler = RejectionSampler()
146 + rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
147 + use_flashinfer=use_flashinfer)
148 rejection_sampler.init_gpu_tensors(device=device)
149
150 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
144 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
151 + target_probs = torch.rand(batch_size,
152 + k + 1,
153 + vocab_size,
154 + dtype=torch.float32)
155 bonus_token_ids = torch.randint(low=0,
156 high=vocab_size,
157 size=(batch_size, 1),
@@ -161,16 +171,21 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
171 @pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
172 @pytest.mark.parametrize("n_rep", [100])
173 @pytest.mark.parametrize("device", CUDA_DEVICES)
174 +@pytest.mark.parametrize("use_flashinfer", [True, False])
175 @torch.inference_mode()
176 def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
166 - frac_seeded: float, n_rep: int,
167 - device: str):
177 + frac_seeded: float, n_rep: int, device: str,
178 + use_flashinfer: bool):
179 torch.set_default_device(device)
169 - rejection_sampler = RejectionSampler()
180 + rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
181 + use_flashinfer=use_flashinfer)
182 rejection_sampler.init_gpu_tensors(device=device)
183
184 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
173 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
185 + target_probs = torch.rand(batch_size,
186 + k + 1,
187 + vocab_size,
188 + dtype=torch.float32)
189 bonus_token_ids = torch.randint(low=0,
190 high=vocab_size,
191 size=(batch_size, 1),
@@ -198,23 +213,85 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
213 assert torch.equal(results[j][i], results[0][i])
214
215
216 +@pytest.mark.parametrize("k", [1, 3, 6])
217 +@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
218 +@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
219 +@pytest.mark.parametrize("device", CUDA_DEVICES)
220 +@torch.inference_mode()
221 +def test_compare_nonflashinfer_backend(k: int, vocab_size: int,
222 + batch_size: int, device: str):
223 + """
224 + Test the flashinfer and nonflashinfer backend generate
225 + the same output metrics.
226 + """
227 + torch.set_default_device(device)
228 + torch.manual_seed(0)
229 + draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
230 + target_probs = torch.rand(batch_size,
231 + k + 1,
232 + vocab_size,
233 + dtype=torch.float32)
234 + bonus_token_ids = torch.randint(low=0,
235 + high=vocab_size,
236 + size=(batch_size, 1),
237 + dtype=torch.int64)
238 + draft_token_ids = torch.randint(low=0,
239 + high=vocab_size,
240 + size=(batch_size, k),
241 + dtype=torch.int64)
242 +
243 + num_accepted_tokens = []
244 + num_emitted_tokens = []
245 + num_draft_tokens = []
246 +
247 + def get_seeded_seqs():
248 + return {
249 + i: torch.Generator(device=device).manual_seed(i)
250 + for i in range(batch_size)
251 + }
252 +
253 + for use_flashinfer in [True, False]:
254 + rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
255 + use_flashinfer=use_flashinfer)
256 + rejection_sampler.init_gpu_tensors(device=device)
257 + # We use seeded sequences to ensure the same tokens are accepted
258 + # for both flashinfer and nonflashinfer backends.
259 + seeded_seqs = get_seeded_seqs()
260 + rejection_sampler(target_probs, bonus_token_ids, draft_probs,
261 + draft_token_ids, seeded_seqs)
262 + num_accepted_tokens.append(rejection_sampler.num_accepted_tokens)
263 + num_emitted_tokens.append(rejection_sampler.num_emitted_tokens)
264 + num_draft_tokens.append(rejection_sampler.num_draft_tokens)
265 +
266 + assert num_accepted_tokens[0] == num_accepted_tokens[1]
267 + assert num_emitted_tokens[0] == num_emitted_tokens[1]
268 + assert num_draft_tokens[0] == num_draft_tokens[1]
269 +
270 +
271 @pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
272 @pytest.mark.parametrize("which_token_ids",
273 ["bonus_token_ids", "draft_token_ids"])
274 @pytest.mark.parametrize("device", CUDA_DEVICES)
275 +@pytest.mark.parametrize("use_flashinfer", [True, False])
276 @torch.inference_mode()
277 def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
207 - which_token_ids: str, device: str):
278 + which_token_ids: str, device: str,
279 + use_flashinfer: bool):
280 k = 3
281 batch_size = 5
282 vocab_size = 30_000
283 torch.set_default_device(device)
284
213 - rejection_sampler = RejectionSampler(strict_mode=True)
285 + rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
286 + use_flashinfer=use_flashinfer,
287 + strict_mode=True)
288 rejection_sampler.init_gpu_tensors(device=device)
289
290 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
217 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
291 + target_probs = torch.rand(batch_size,
292 + k + 1,
293 + vocab_size,
294 + dtype=torch.float32)
295 bonus_token_ids = torch.randint(low=0,
296 high=vocab_size,
297 size=(batch_size, 1),
@@ -248,9 +325,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
325
326 @pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
327 @pytest.mark.parametrize("seed", list(range(5)))
328 +@pytest.mark.parametrize("use_flashinfer", [True, False])
329 @torch.inference_mode()
330 def test_rejection_sampling_approximates_target_distribution(
253 - seed: int, draft_and_target_probs_equal: bool):
331 + seed: int, draft_and_target_probs_equal: bool, use_flashinfer: bool):
332 """Verify rejection sampling approximates target distribution,
333 despite sampling from a potentially distinct draft distribution.
334
@@ -279,10 +357,10 @@ def test_rejection_sampling_approximates_target_distribution(
357 """
358 torch.set_default_device("cpu")
359 set_random_seed(seed)
282 -
360 helper = _CorrectnessTestHelper(
361 vocab_size=10,
285 - rejection_sampler=RejectionSampler(),
362 + rejection_sampler=RejectionSampler(disable_bonus_tokens=False,
363 + use_flashinfer=use_flashinfer),
364 )
365
366 draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
@@ -398,10 +476,10 @@ class _CorrectnessTestHelper:
476 draft_probs = draft_probs.reshape(1, self.k, self.vocab_size).repeat(
477 num_samples, 1, 1)
478
401 - # Repeat target probs num_samples * k times.
479 + # Repeat target probs num_samples * (k + 1) times.
480 # Rejection sampler requires bonus token probs, but they aren't used.
481 target_probs = target_probs.reshape(1, 1, self.vocab_size).repeat(
404 - num_samples, self.k, 1)
482 + num_samples, self.k + 1, 1)
483
484 # Randomly sample draft token ids from draft probs.
485 draft_token_ids = torch.multinomial(draft_probs[:, 0, :],
tests/samplers/test_typical_acceptance_sampler.py
+32 -18
@@ -79,7 +79,10 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
79 torch.set_default_device(device)
80 typical_acceptance_sampler = get_acceptance_sampler()
81 typical_acceptance_sampler.init_gpu_tensors(device=device)
82 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
82 + target_with_bonus_probs = torch.rand(batch_size,
83 + k + 1,
84 + vocab_size,
85 + dtype=torch.float32)
86 bonus_token_ids = torch.randint(low=0,
87 high=vocab_size,
88 size=(batch_size, 1),
@@ -89,7 +92,7 @@ def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
92 size=(batch_size, k),
93 dtype=torch.int64)
94 # Verify that sampling succeeds for all cases.
92 - typical_acceptance_sampler(target_probs,
95 + typical_acceptance_sampler(target_with_bonus_probs,
96 bonus_token_ids,
97 draft_probs=None,
98 draft_token_ids=draft_token_ids)
@@ -112,7 +115,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
115 torch.set_default_device(device)
116 typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
117 typical_acceptance_sampler.init_gpu_tensors(device=device)
115 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
118 + target_with_bonus_probs = torch.rand(batch_size,
119 + k + 1,
120 + vocab_size,
121 + dtype=torch.float32)
122 bonus_token_ids = torch.randint(low=0,
123 high=vocab_size,
124 size=(batch_size, 1),
@@ -141,7 +147,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
147 oob_token_ids[0][0] = rogue_token_id
148
149 with pytest.raises(AssertionError):
144 - typical_acceptance_sampler(target_probs,
150 + typical_acceptance_sampler(target_with_bonus_probs,
151 bonus_token_ids,
152 draft_probs=None,
153 draft_token_ids=draft_token_ids)
@@ -172,7 +178,10 @@ def test_uniform_target_distribution_accepts_all_tokens(
178 typical_acceptance_sampler = get_acceptance_sampler(
179 strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
180 typical_acceptance_sampler.init_gpu_tensors(device=device)
175 - target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
181 + target_with_bonus_probs = torch.rand(batch_size,
182 + k + 1,
183 + vocab_size,
184 + dtype=torch.float32)
185 draft_token_ids = torch.randint(low=0,
186 high=vocab_size,
187 size=(batch_size, k),
@@ -182,7 +191,7 @@ def test_uniform_target_distribution_accepts_all_tokens(
191 size=(batch_size, 1),
192 dtype=torch.int64)
193 output_token_ids = typical_acceptance_sampler(
185 - target_probs,
194 + target_with_bonus_probs,
195 bonus_token_ids,
196 draft_probs=None,
197 draft_token_ids=draft_token_ids)
@@ -229,8 +238,9 @@ def test_temperature_zero_target_distribution(seed: int,
238 # Simulate temperature 0 probability distribution for target probabilities
239 # and create target probabilities such that only 1 token id has
240 # probability 1.0
232 - target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
233 - batch_size, k, vocab_size)
241 + target_with_bonus_probs, zero_temperature_token_ids = \
242 + get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
243 + zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
244 # Populate draft_token_ids such that they exclude the token_ids
245 # with probability = 1.0
246 draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
@@ -245,7 +255,7 @@ def test_temperature_zero_target_distribution(seed: int,
255 # fallback to the greedy sampling for selecting 1 token for each sequence.
256 # Verify the same.
257 output_token_ids = typical_acceptance_sampler(
248 - target_probs,
258 + target_with_bonus_probs,
259 bonus_token_ids,
260 draft_probs=None,
261 draft_token_ids=draft_token_ids)
@@ -289,8 +299,10 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
299 # For sequences 0 and 2 set the distribution to a temperature
300 # zero distribution. For sequences 1 and 3 set it to a uniform
301 # distribution.
292 - target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
293 - batch_size, k, vocab_size))
302 + target_with_bonus_probs, zero_temperature_token_ids = \
303 + get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
304 + zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
305 + target_probs = target_with_bonus_probs[:, :-1]
306 draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
307 zero_temperature_token_ids)
308 uniform_probs = torch.rand(2, k, vocab_size, dtype=torch.float32)
@@ -300,7 +312,7 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
312 size=(batch_size, 1),
313 dtype=torch.int64)
314 output_token_ids = typical_acceptance_sampler(
303 - target_probs,
315 + target_with_bonus_probs,
316 bonus_token_ids,
317 draft_probs=None,
318 draft_token_ids=draft_token_ids)
@@ -356,15 +368,16 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
368 # Create a temperature zero target probability distribution and ensure
369 # all draft token ids correspond to the tokens with 1.0 probability.
370 # Verify that all of them are accepted.
359 - target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
360 - batch_size, k, vocab_size))
371 + target_with_bonus_probs, zero_temperature_token_ids = \
372 + get_zero_temperature_prob_dist(batch_size, k + 1, vocab_size)
373 + zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
374 draft_token_ids = zero_temperature_token_ids
375 bonus_token_ids = torch.randint(low=0,
376 high=vocab_size,
377 size=(batch_size, 1),
378 dtype=torch.int64)
379 output_token_ids = typical_acceptance_sampler(
367 - target_probs,
380 + target_with_bonus_probs,
381 bonus_token_ids,
382 draft_probs=None,
383 draft_token_ids=draft_token_ids)
@@ -384,7 +397,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
397 draft_token_ids = torch.cat(
398 (draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
399 output_token_ids = typical_acceptance_sampler(
387 - target_probs,
400 + target_with_bonus_probs,
401 bonus_token_ids,
402 draft_probs=None,
403 draft_token_ids=draft_token_ids)
@@ -421,8 +434,9 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
434 # 0.00001. Populate draft_token_ids such that they exclude the token_ids
435 # with probability = 1.0. Without any changes to the posterior thresholds
436 # none of the draft tokens are accepted.
424 - target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
425 - batch_size, k, vocab_size))
437 + target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
438 + batch_size, k + 1, vocab_size)
439 + zero_temperature_token_ids = zero_temperature_token_ids[:, :-1]
440 target_probs[target_probs == 0] = 0.00001
441 draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
442 zero_temperature_token_ids)
tests/spec_decode/test_spec_decode_worker.py
+2 -3
@@ -230,9 +230,8 @@ def test_correctly_calls_spec_decode_sampler(k: int, batch_size: int,
230
231 assert torch.equal(actual.bonus_token_ids,
232 target_token_ids.reshape(batch_size, k + 1)[:, -1:])
233 - assert torch.equal(
234 - actual.target_probs,
235 - target_token_probs.reshape(batch_size, k + 1, -1)[:, :-1])
233 + assert torch.equal(actual.target_with_bonus_probs,
234 + target_token_probs.reshape(batch_size, k + 1, -1))
235 assert torch.equal(actual.draft_token_ids, proposal_token_ids)
236 assert torch.equal(actual.draft_probs, proposal_probs)
237
vllm/envs.py
+1
@@ -31,6 +31,7 @@ if TYPE_CHECKING:
31 VLLM_TRACE_FUNCTION: int = 0
32 VLLM_ATTENTION_BACKEND: Optional[str] = None
33 VLLM_USE_FLASHINFER_SAMPLER: bool = False
34 + VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35 VLLM_PP_LAYER_PARTITION: Optional[str] = None
36 VLLM_CPU_KVCACHE_SPACE: int = 0
37 VLLM_CPU_OMP_THREADS_BIND: str = ""
vllm/model_executor/layers/rejection_sampler.py
+141 -43
@@ -1,12 +1,28 @@
1 from functools import cached_property
2 +from importlib.util import find_spec
3 from typing import Dict, List, Optional, Tuple
4
5 import torch
6 import torch.jit
7
8 +import vllm.envs as envs
9 +from vllm.logger import init_logger
10 from vllm.model_executor.layers.spec_decode_base_sampler import (
11 SpecDecodeStochasticBaseSampler)
12
13 +logger = init_logger(__name__)
14 +
15 +if find_spec("flashinfer"):
16 + """
17 + Consider utilizing the FlashInfer rejection sampling kernel initially,
18 + as it employs a dedicated kernel rather than relying on
19 + Torch tensor operations. This design choice helps to fuse operations,
20 + reduce memory I/O, and consequently enhances performance.
21 + """
22 + from flashinfer.sampling import chain_speculative_sampling
23 +else:
24 + chain_speculative_sampling = None
25 +
26
27 class RejectionSampler(SpecDecodeStochasticBaseSampler):
28 """Apply modified rejection sampling as described in "Accelerating Large
@@ -16,7 +32,8 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
32
33 def __init__(self,
34 disable_bonus_tokens: bool = True,
19 - strict_mode: bool = False):
35 + strict_mode: bool = False,
36 + use_flashinfer: Optional[bool] = None):
37 """Create a rejection sampler.
38
39 Args:
@@ -26,13 +43,29 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
43 strict_mode: Whether or not to perform shape/device/dtype checks
44 during sampling. This catches correctness issues but adds
45 nontrivial latency.
46 + use_falshinfer: We will use this parameter to determine whether
47 + to use the FlashInfer rejection sampling kernel or not. If it's
48 + None, we will use the default value from the environment variable.
49 + This parameter is only used for testing purposes.
50 """
51 super().__init__(disable_bonus_tokens=disable_bonus_tokens,
52 strict_mode=strict_mode)
53 + if use_flashinfer is None:
54 + self.use_flashinfer = envs.VLLM_USE_FLASHINFER_SAMPLER and (
55 + chain_speculative_sampling is not None)
56 + else:
57 + self.use_flashinfer = use_flashinfer
58 +
59 + if self.use_flashinfer:
60 + assert not disable_bonus_tokens, \
61 + "flashinfer will enable bonus token by default"
62 + logger.info("Use flashinfer for rejection sampling.")
63 + else:
64 + logger.info("Use pytorch for rejection sampling.")
65
66 def forward(
67 self,
35 - target_probs: torch.Tensor,
68 + target_with_bonus_probs: torch.Tensor,
69 bonus_token_ids: torch.Tensor,
70 draft_probs: torch.Tensor,
71 draft_token_ids: torch.Tensor,
@@ -50,9 +83,9 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
83 sequence.
84
85 Args:
53 - target_probs: The probability distribution over token ids given
54 - context according to the target model.
55 - shape = [batch_size, num_speculative_tokens, vocab_size]
86 + target_with_bonus_probs: The probability distribution
87 + over token ids given context according to the target model.
88 + shape = [batch_size, num_speculative_tokens + 1, vocab_size]
89
90 bonus_token_ids: The "bonus" token ids that are accepted iff all
91 speculative tokens in a sequence are accepted.
@@ -78,23 +111,52 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
111 # Only perform shape/dtype/device checking in strict mode, as it adds
112 # overhead.
113 if self._strict_mode:
81 - self._raise_if_incorrect_input(target_probs, draft_token_ids,
82 - bonus_token_ids, draft_probs)
114 + self._raise_if_incorrect_input(target_with_bonus_probs,
115 + draft_token_ids, bonus_token_ids,
116 + draft_probs)
117
84 - accepted, recovered_token_ids = (
85 - self._batch_modified_rejection_sampling(
86 - target_probs,
87 - draft_probs,
88 - draft_token_ids,
89 - seeded_seqs,
90 - ))
118 + batch_size, k, _ = draft_probs.shape
119
92 - output_token_ids = self._create_output(
93 - accepted,
94 - recovered_token_ids,
95 - draft_token_ids,
96 - bonus_token_ids,
97 - )
120 + # batch_size = 0 when all requests in the batch are
121 + # non_spec requests. In this case, output_token_ids is
122 + # just an empty tensor.
123 + if batch_size == 0:
124 + return torch.empty(0, k + 1, device=draft_probs.device, dtype=int)
125 +
126 + # If use Flashinfer chain_speculative_sampling kernel
127 + # for rejection sampling
128 + if self.use_flashinfer:
129 + batch_size, k, _ = draft_probs.shape
130 + uniform_samples = self._create_uniform_samples(
131 + seeded_seqs, batch_size, k, draft_probs.device)
132 + output_token_ids, accepted_token_num, emitted_token_num \
133 + = chain_speculative_sampling(
134 + draft_probs, draft_token_ids, uniform_samples,
135 + target_with_bonus_probs)
136 +
137 + # num_emitted_tokens returned by flashinfer
138 + # does not include the bonus token
139 + # Flashinfer stops at the first token that violates
140 + # the condition p >= q and does not include recovery/bonus token.
141 + # Therefore, we need to add batch_size here.
142 + self.num_accepted_tokens += accepted_token_num.sum()
143 + self.num_emitted_tokens += emitted_token_num.sum() + batch_size
144 + self.num_draft_tokens += batch_size * k
145 + else:
146 + accepted, recovered_token_ids = (
147 + self._batch_modified_rejection_sampling(
148 + target_with_bonus_probs[:, :-1],
149 + draft_probs,
150 + draft_token_ids,
151 + seeded_seqs,
152 + ))
153 +
154 + output_token_ids = self._create_output(
155 + accepted,
156 + recovered_token_ids,
157 + draft_token_ids,
158 + bonus_token_ids,
159 + )
160
161 return output_token_ids
162
@@ -135,6 +197,63 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
197
198 return accepted, recovered_token_ids
199
200 + def _create_uniform_samples(self,
201 + seeded_seqs: Optional[Dict[int,
202 + torch.Generator]],
203 + batch_size: int, k: int,
204 + device: torch.device) -> torch.Tensor:
205 + """
206 + Generates a batch of uniform random samples, with optional seeding
207 + for specific sequences.
208 +
209 + This method creates a tensor of shape `(batch_size, k + 1)` filled
210 + with uniform random values in the range [0, 1). If `seeded_seqs`
211 + is provided, the sequences corresponding to specific indices
212 + will be generated using the provided `torch.Generator` for
213 + reproducibility. The other sequences will be generated without
214 + a seed.
215 +
216 + Args:
217 + seeded_seqs : Optional[Dict[int, torch.Generator]]
218 + A dictionary mapping indices in the batch to
219 + `torch.Generator` objects. If `None`, all samples are
220 + generated without a seed.
221 + batch_size : int
222 + The number of sequences to generate.
223 + k : int
224 + The number of random samples per sequence.
225 + device : torch.device
226 + The device on which to allocate the tensor.
227 +
228 + Returns:
229 + uniform_rand : torch.Tensor
230 + A tensor of shape `(batch_size, k + 1)` containing uniform
231 + random values in the range [0, 1).
232 + """
233 + if not seeded_seqs:
234 + return torch.rand(batch_size, k + 1, device=device)
235 +
236 + uniform_rand = torch.empty(batch_size, k + 1, device=device)
237 +
238 + non_seeded_indices = []
239 + for idx in range(batch_size):
240 + generator = seeded_seqs.get(idx)
241 + if generator is None:
242 + non_seeded_indices.append(idx)
243 + else:
244 + uniform_rand[idx, :] = torch.rand(1,
245 + k + 1,
246 + dtype=self.probs_dtype,
247 + device=device,
248 + generator=generator)
249 + if non_seeded_indices:
250 + uniform_rand[non_seeded_indices, :] = torch.rand(
251 + len(non_seeded_indices),
252 + k + 1,
253 + dtype=self.probs_dtype,
254 + device=device)
255 + return uniform_rand
256 +
257 def _get_accepted(
258 self,
259 target_probs: torch.Tensor, # [batch_size, k, vocab_size]
@@ -175,29 +294,8 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
294 selected_target_probs = target_probs[batch_indices, probs_indicies,
295 draft_token_ids]
296
178 - if not seeded_seqs:
179 - uniform_rand = torch.rand_like(selected_target_probs)
180 - else:
181 - uniform_rand = torch.empty_like(selected_target_probs)
182 -
183 - non_seeded_indices = []
184 - for idx in range(batch_size):
185 - generator = seeded_seqs.get(idx)
186 - if generator is None:
187 - non_seeded_indices.append(idx)
188 - else:
189 - uniform_rand[idx, :] = torch.rand(
190 - 1,
191 - k,
192 - dtype=self.probs_dtype,
193 - device=target_probs.device,
194 - generator=generator)
195 - if non_seeded_indices:
196 - uniform_rand[non_seeded_indices, :] = torch.rand(
197 - len(non_seeded_indices),
198 - k,
199 - dtype=self.probs_dtype,
200 - device=target_probs.device)
297 + uniform_rand = self._create_uniform_samples(seeded_seqs, batch_size,
298 + k - 1, target_probs.device)
299
300 capped_ratio = torch.minimum(
301 selected_target_probs / selected_draft_probs,
vllm/model_executor/layers/spec_decode_base_sampler.py
+25 -18
@@ -130,29 +130,35 @@ class SpecDecodeBaseSampler(nn.Module):
130
131 def _raise_if_incorrect_input(
132 self,
133 - target_probs: torch.Tensor,
133 + target_with_bonus_probs: torch.Tensor,
134 draft_token_ids: torch.Tensor,
135 bonus_token_ids: torch.Tensor,
136 draft_probs: Optional[torch.Tensor] = None,
137 ) -> None:
138 - self._raise_if_incorrect_shape(target_probs, draft_token_ids,
139 - bonus_token_ids, draft_probs)
140 - self._raise_if_incorrect_dtype(target_probs, draft_token_ids,
141 - bonus_token_ids, draft_probs)
142 - self._raise_if_inconsistent_device(target_probs, draft_token_ids,
143 - bonus_token_ids, draft_probs)
144 - self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
138 + self._raise_if_incorrect_shape(target_with_bonus_probs,
139 + draft_token_ids, bonus_token_ids,
140 + draft_probs)
141 + self._raise_if_incorrect_dtype(target_with_bonus_probs,
142 + draft_token_ids, bonus_token_ids,
143 + draft_probs)
144 + self._raise_if_inconsistent_device(target_with_bonus_probs,
145 + draft_token_ids, bonus_token_ids,
146 + draft_probs)
147 + self._raise_if_out_of_bounds_vocab(target_with_bonus_probs.shape[-1],
148 draft_token_ids, bonus_token_ids)
149
150 def _raise_if_incorrect_shape(
151 self,
149 - target_probs: torch.Tensor,
152 + target_with_bonus_probs: torch.Tensor,
153 draft_token_ids: torch.Tensor,
154 bonus_token_ids: torch.Tensor,
155 draft_probs: Optional[torch.Tensor] = None,
156 ) -> None:
157 (target_batch_size, num_target_probs,
155 - target_vocab_size) = target_probs.shape
158 + target_vocab_size) = target_with_bonus_probs.shape
159 +
160 + # Does not count the extra token
161 + num_target_probs -= 1
162
163 # validate the shape of draft token ids.
164 draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
@@ -175,12 +181,12 @@ class SpecDecodeBaseSampler(nn.Module):
181
182 def _raise_if_incorrect_dtype(
183 self,
178 - target_probs: torch.Tensor,
184 + target_with_bonus_probs: torch.Tensor,
185 draft_token_ids: torch.Tensor,
186 bonus_token_ids: torch.Tensor,
187 draft_probs: Optional[torch.Tensor] = None,
188 ) -> None:
183 - assert target_probs.dtype == self.probs_dtype
189 + assert target_with_bonus_probs.dtype == self.probs_dtype
190 assert draft_token_ids.dtype == self.token_id_dtype
191 assert bonus_token_ids.dtype == self.token_id_dtype
192 if draft_probs is not None:
@@ -188,15 +194,16 @@ class SpecDecodeBaseSampler(nn.Module):
194
195 def _raise_if_inconsistent_device(
196 self,
191 - target_probs: torch.Tensor,
197 + target_with_bonus_probs: torch.Tensor,
198 draft_token_ids: torch.Tensor,
199 bonus_token_ids: torch.Tensor,
200 draft_probs: Optional[torch.Tensor] = None,
201 ) -> None:
202 devices = [
197 - t.device for t in
198 - [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
199 - if t is not None
203 + t.device for t in [
204 + target_with_bonus_probs, bonus_token_ids, draft_probs,
205 + draft_token_ids
206 + ] if t is not None
207 ]
208 assert all([devices[0] == device for device in devices])
209
@@ -220,7 +227,7 @@ class SpecDecodeDeterministicBaseSampler(SpecDecodeBaseSampler):
227 @abstractmethod
228 def forward(
229 self,
223 - target_probs: torch.Tensor,
230 + target_with_bonus_probs: torch.Tensor,
231 bonus_token_ids: torch.Tensor,
232 draft_probs: torch.Tensor,
233 draft_token_ids: torch.Tensor,
@@ -236,7 +243,7 @@ class SpecDecodeStochasticBaseSampler(SpecDecodeBaseSampler):
243 @abstractmethod
244 def forward(
245 self,
239 - target_probs: torch.Tensor,
246 + target_with_bonus_probs: torch.Tensor,
247 bonus_token_ids: torch.Tensor,
248 draft_probs: torch.Tensor,
249 draft_token_ids: torch.Tensor,
vllm/model_executor/layers/typical_acceptance_sampler.py
+4 -3
@@ -41,7 +41,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
41
42 def forward(
43 self,
44 - target_probs: torch.Tensor,
44 + target_with_bonus_probs: torch.Tensor,
45 bonus_token_ids: torch.Tensor,
46 draft_probs: torch.Tensor,
47 draft_token_ids: torch.Tensor,
@@ -80,8 +80,9 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
80 # Only perform shape/dtype/device checking in strict mode, as it adds
81 # overhead.
82 if self._strict_mode:
83 - self._raise_if_incorrect_input(target_probs, draft_token_ids,
84 - bonus_token_ids)
83 + self._raise_if_incorrect_input(target_with_bonus_probs,
84 + draft_token_ids, bonus_token_ids)
85 + target_probs = target_with_bonus_probs[:, :-1]
86 accepted = self._evaluate_accepted_tokens(target_probs,
87 draft_token_ids)
88 recovered_token_ids = self._replacement_token_ids(target_probs)
vllm/spec_decode/spec_decode_worker.py
+3 -4
@@ -625,8 +625,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
625 seq_group_metadata_list, proposal_lens_list)
626 original_indices = spec_indices + non_spec_indices
627
628 - # Get probabilities of target model, excluding bonus token.
629 - proposal_verifier_probs = proposal_scores.probs[spec_indices, :-1]
628 + # Get probabilities of target model, including bonus tokens.
629 + proposal_verifier_probs = proposal_scores.probs[spec_indices]
630
631 # Get non-speculative sampled tokens from target model.
632 non_spec_token_ids = proposal_scores.token_ids[non_spec_indices]
@@ -651,13 +651,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
651 }
652
653 accepted_token_ids = self.spec_decode_sampler(
654 - target_probs=proposal_verifier_probs,
654 + target_with_bonus_probs=proposal_verifier_probs,
655 bonus_token_ids=bonus_token_ids,
656 draft_probs=proposal_probs,
657 draft_token_ids=proposal_token_ids,
658 **sampler_extra_kwargs,
659 )
660 -
660 # Append output tokens from non-speculative sequences to
661 # the accepted token ids tensor.
662 non_spec_token_ids = non_spec_token_ids.expand(-1, max_proposal_len +