@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa9e385229
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Speculative Decoding 1/2 ] Add typical acceptance sampling as one of the sampling techniques in the verifier (#5131)

sroy745 committed Jun 17, 2024 at 19:29 UTC fa9e3852290ecb6eaae45befbd629bb060f57fb7
4 files changed +866 -164
tests/samplers/test_typical_acceptance_sampler.py new
+464
@@ -0,0 +1,464 @@
1 +"""Tests for rejection sampling."""
2 +
3 +import pytest
4 +import torch
5 +
6 +from vllm.model_executor.layers.typical_acceptance_sampler import (
7 + TypicalAcceptanceSampler)
8 +from vllm.model_executor.utils import set_random_seed
9 +
10 +CUDA_DEVICES = [f"cuda:{i}" for i in range(1)]
11 +
12 +
13 +def get_zero_temperature_prob_dist(batch_size, k, vocab_size):
14 + """
15 + Generates a fake temperature zero probability distribution.
16 + Returns:
17 + 1. A fake temperature zero probability distribution of shape
18 + [batch_size, k, vocab_size]
19 + 2. Tensor of shape [batch_size, k] containing the token ids
20 + of the probability 1.0 tokens at each position.
21 + """
22 + # Simulate temperature 0 probability distribution for target probabilities
23 + # and create target probabilities such that only 1 token id has
24 + # probability 1.0
25 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
26 + probs = torch.rand(batch_size, k, vocab_size)
27 + _, zero_temperature_token_ids = torch.max(probs, dim=-1)
28 + # set the probability of the tokens with ids in zero_temperature_token_ids
29 + # to 1 and the rest to 0.
30 + target_probs = torch.zeros_like(probs).scatter_(
31 + -1, zero_temperature_token_ids.unsqueeze(-1), 1.0)
32 + return target_probs, zero_temperature_token_ids
33 +
34 +
35 +def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
36 + token_ids_to_exclude: torch.Tensor):
37 + """
38 + Returns a tensor of shape [batch_size, k] of fake draft token ids
39 + drawn randomly from a vocab of size vocab_size. We however ensure
40 + that token_ids from token_ids_to_exclude are excluded at the
41 + corresponding positions.
42 + """
43 + draft_token_ids = torch.empty(batch_size, k, dtype=torch.long)
44 + for i in range(batch_size):
45 + for j in range(k):
46 + # Generate a random token ID excluding token_ids_to_exclude[i, j]
47 + while True:
48 + token_id = torch.randint(0, vocab_size, (1, )).item()
49 + if token_id != token_ids_to_exclude[i, j]:
50 + draft_token_ids[i, j] = token_id
51 + break
52 + return draft_token_ids
53 +
54 +
55 +@pytest.mark.parametrize("k", list(range(1, 6)))
56 +@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
57 +@pytest.mark.parametrize("batch_size", list(range(1, 32)))
58 +@pytest.mark.parametrize("device", CUDA_DEVICES)
59 +@torch.inference_mode()
60 +def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
61 + device: str):
62 + """
63 + Tests that the TypicalAcceptancSampler forward succeeds for
64 + different combinations of k, vocab_size, batch_size and num devices.
65 + """
66 + torch.set_default_device(device)
67 + typical_acceptance_sampler = TypicalAcceptanceSampler()
68 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
69 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
70 + bonus_token_ids = torch.randint(low=0,
71 + high=vocab_size,
72 + size=(batch_size, 1),
73 + dtype=torch.int64)
74 + draft_token_ids = torch.randint(low=0,
75 + high=vocab_size,
76 + size=(batch_size, k),
77 + dtype=torch.int64)
78 + # Verify that sampling succeeds for all cases.
79 + typical_acceptance_sampler(target_probs, bonus_token_ids, draft_token_ids)
80 +
81 +
82 +@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
83 +@pytest.mark.parametrize("which_token_ids",
84 + ["bonus_token_ids", "draft_token_ids"])
85 +@pytest.mark.parametrize("device", CUDA_DEVICES)
86 +@torch.inference_mode()
87 +def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
88 + which_token_ids: str, device: str):
89 + """
90 + Tests that we throw an exception of the token ids fall outside
91 + the bound of the provided vocabulary.
92 + """
93 + k = 3
94 + batch_size = 5
95 + vocab_size = 30_000
96 + torch.set_default_device(device)
97 + typical_acceptance_sampler = TypicalAcceptanceSampler(strict_mode=True)
98 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
99 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
100 + bonus_token_ids = torch.randint(low=0,
101 + high=vocab_size,
102 + size=(batch_size, 1),
103 + dtype=torch.int64)
104 + draft_token_ids = torch.randint(low=0,
105 + high=vocab_size,
106 + size=(batch_size, k),
107 + dtype=torch.int64)
108 + # Verify that appropriate exceptions are thrown for out
109 + # of bound vocabs.
110 + oob_token_ids = None
111 + if which_token_ids == "bonus_token_ids":
112 + oob_token_ids = bonus_token_ids
113 + elif which_token_ids == "draft_token_ids":
114 + oob_token_ids = draft_token_ids
115 + else:
116 + raise AssertionError()
117 +
118 + if above_or_below_vocab_range == "above":
119 + rogue_token_id = vocab_size + 1
120 + elif above_or_below_vocab_range == "below":
121 + rogue_token_id = -1
122 + else:
123 + raise AssertionError()
124 +
125 + oob_token_ids[0][0] = rogue_token_id
126 +
127 + with pytest.raises(AssertionError):
128 + typical_acceptance_sampler(target_probs, bonus_token_ids,
129 + draft_token_ids)
130 +
131 +
132 +@pytest.mark.parametrize("seed", list(range(10)))
133 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
134 +@pytest.mark.parametrize("device", CUDA_DEVICES)
135 +@torch.inference_mode()
136 +def test_uniform_target_distribution_accepts_all_tokens(
137 + seed: int, disable_bonus_tokens: bool, device: str):
138 + """
139 + Test the TypicalAcceptanceSampler with a uniform target probability
140 + distribution.
141 +
142 + This test verifies that when provided with a uniform target probability
143 + distribution, the TypicalAcceptanceSampler accepts all draft tokens. The
144 + entropy of the uniform target distribution being high should lead to all
145 + draft tokens being accepted. The test also ensures that the behavior
146 + regarding bonus tokens is consistent with the `disable_bonus_tokens`
147 + flag.
148 + """
149 + set_random_seed(seed)
150 + k = 3
151 + batch_size = 5
152 + vocab_size = 30_000
153 + torch.set_default_device(device)
154 + typical_acceptance_sampler = TypicalAcceptanceSampler(
155 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
156 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
157 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
158 + draft_token_ids = torch.randint(low=0,
159 + high=vocab_size,
160 + size=(batch_size, k),
161 + dtype=torch.int64)
162 + bonus_token_ids = torch.randint(low=0,
163 + high=vocab_size,
164 + size=(batch_size, 1),
165 + dtype=torch.int64)
166 + output_token_ids = typical_acceptance_sampler(target_probs,
167 + bonus_token_ids,
168 + draft_token_ids)
169 + # We are using a uniform target probability distribution.
170 + # For a uniform distribution the entropy is very high and it
171 + # should lead to all draft tokens being accepted. Verify that.
172 + assert output_token_ids.shape[0] == batch_size
173 + assert output_token_ids.shape[1] == (k + 1)
174 + if disable_bonus_tokens:
175 + assert torch.all(output_token_ids[:, -1] == -1)
176 + else:
177 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
178 +
179 + assert torch.all(output_token_ids[:, :k] == draft_token_ids)
180 +
181 +
182 +@pytest.mark.parametrize("seed", list(range(10)))
183 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
184 +@pytest.mark.parametrize("device", CUDA_DEVICES)
185 +@torch.inference_mode()
186 +def test_temperature_zero_target_distribution(seed: int,
187 + disable_bonus_tokens: bool,
188 + device: str):
189 + """
190 + Test the TypicalAcceptanceSampler with a zero-temperature target
191 + probability distribution.
192 +
193 + This test verifies that when using a zero-temperature target probability
194 + distribution, where only one token has a probability of 1.0, the
195 + TypicalAcceptanceSampler correctly rejects all draft tokens that do not
196 + match this probability. Additionally, it ensures that when all draft
197 + tokens are rejected, the sampler falls back to greedy sampling to select a
198 + single token from the target distribution.
199 + """
200 + set_random_seed(seed)
201 + k = 3
202 + batch_size = 5
203 + vocab_size = 30_000
204 + torch.set_default_device(device)
205 +
206 + typical_acceptance_sampler = TypicalAcceptanceSampler(
207 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
208 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
209 + # Simulate temperature 0 probability distribution for target probabilities
210 + # and create target probabilities such that only 1 token id has
211 + # probability 1.0
212 + target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
213 + batch_size, k, vocab_size)
214 + # Populate draft_token_ids such that they exclude the token_ids
215 + # with probability = 1.0
216 + draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
217 + zero_temperature_token_ids)
218 + bonus_token_ids = torch.randint(low=0,
219 + high=vocab_size,
220 + size=(batch_size, 1),
221 + dtype=torch.int64)
222 + # The target probaility distribution is a temperature zero distribution
223 + # with zero entroy. Since our draft token ids don't match the probability
224 + # 1.0 tokens in the target distribution we will reject all of them and
225 + # fallback to the greedy sampling for selecting 1 token for each sequence.
226 + # Verify the same.
227 + output_token_ids = typical_acceptance_sampler(target_probs,
228 + bonus_token_ids,
229 + draft_token_ids)
230 + assert output_token_ids.shape[0] == batch_size
231 + assert output_token_ids.shape[1] == (k + 1)
232 + assert torch.all(output_token_ids[:, -1] == -1)
233 + assert torch.all(output_token_ids[:, 0] == zero_temperature_token_ids[:,
234 + 0])
235 +
236 +
237 +@pytest.mark.parametrize("seed", list(range(10)))
238 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
239 +@pytest.mark.parametrize("device", CUDA_DEVICES)
240 +@torch.inference_mode()
241 +def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
242 + device: str):
243 + """
244 + Test the TypicalAcceptanceSampler with a mixed target probability
245 + distribution.
246 +
247 + This test ensures that the TypicalAcceptanceSampler handles a mixed
248 + target probability distribution correctly. Specifically, it uses a
249 + zero-temperature distribution for some sequences and a uniform
250 + distribution for others. The test verifies that:
251 +
252 + - For sequences with a zero-temperature distribution, only the token
253 + with a probability of 1.0 is accepted, and all other tokens are rejected.
254 + - For sequences with a uniform distribution, all draft tokens are
255 + accepted.
256 + - When `disable_bonus_tokens` is False, the bonus tokens are also accepted
257 + for sequences with a uniform distribution.
258 + """
259 + set_random_seed(seed)
260 + k = 3
261 + batch_size = 4
262 + vocab_size = 30_000
263 + torch.set_default_device(device)
264 + typical_acceptance_sampler = TypicalAcceptanceSampler(
265 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
266 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
267 + # For sequences 0 and 2 set the distribution to a temperature
268 + # zero distribution. For sequences 1 and 3 set it to a uniform
269 + # distribution.
270 + target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
271 + batch_size, k, vocab_size))
272 + draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
273 + zero_temperature_token_ids)
274 + uniform_probs = torch.rand(2, k, vocab_size, dtype=torch.float32)
275 + target_probs[[1, 3]] = uniform_probs
276 + bonus_token_ids = torch.randint(low=0,
277 + high=vocab_size,
278 + size=(batch_size, 1),
279 + dtype=torch.int64)
280 + output_token_ids = typical_acceptance_sampler(target_probs,
281 + bonus_token_ids,
282 + draft_token_ids)
283 + # verify the shape of output_token_ids
284 + assert output_token_ids.shape[0] == batch_size
285 + assert output_token_ids.shape[1] == (k + 1)
286 + # For sequences 0 and 2 verify that only 1 token is accepted
287 + # which is the token with probability 1.0 in the target distribution
288 + # at position 0.
289 + assert torch.all(output_token_ids[[0, 2], 1:] == -1)
290 + assert (torch.all(output_token_ids[[0, 2],
291 + 0] == zero_temperature_token_ids[[0, 2],
292 + 0]))
293 + # For sequences 1 and 3 verify that all tokens are accepted since the
294 + # target probability distribution is uniform. In addition verify that
295 + # if disable_bonus_tokens is false then we also accept the bonus tokens.
296 + assert torch.all(
297 + output_token_ids[[1, 3], :-1] == draft_token_ids[[1, 3], :])
298 + if disable_bonus_tokens:
299 + assert torch.all(output_token_ids[[1, 3], -1] == -1)
300 + else:
301 + assert torch.all(output_token_ids[[1, 3], -1] != -1)
302 +
303 +
304 +@pytest.mark.parametrize("seed", list(range(10)))
305 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
306 +@pytest.mark.parametrize("device", CUDA_DEVICES)
307 +@torch.inference_mode()
308 +def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
309 + device: str):
310 + """
311 + Test the TypicalAcceptanceSampler's behavior when only a subset of draft
312 + tokens should be accepted.
313 +
314 + This test verifies that the TypicalAcceptanceSampler correctly accepts or
315 + rejects draft tokens based on a zero-temperature target probability
316 + distribution. Specifically, it ensures that:
317 +
318 + - When all draft tokens match tokens with a probability of 1.0 in the
319 + target distribution, all draft tokens are accepted.
320 + - When only some draft tokens match tokens with a probability of 1.0 in
321 + the target distribution, only those matching tokens are accepted, and the
322 + rest are rejected.
323 + """
324 + set_random_seed(seed)
325 + k = 5
326 + batch_size = 1
327 + vocab_size = 30_000
328 + torch.set_default_device(device)
329 + typical_acceptance_sampler = TypicalAcceptanceSampler(
330 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
331 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
332 + # Create a temperature zero target probability distribution and ensure
333 + # all draft token ids correspond to the tokens with 1.0 probability.
334 + # Verify that all of them are accepted.
335 + target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
336 + batch_size, k, vocab_size))
337 + draft_token_ids = zero_temperature_token_ids
338 + bonus_token_ids = torch.randint(low=0,
339 + high=vocab_size,
340 + size=(batch_size, 1),
341 + dtype=torch.int64)
342 + output_token_ids = typical_acceptance_sampler(target_probs,
343 + bonus_token_ids,
344 + draft_token_ids)
345 + assert output_token_ids.shape[0] == batch_size
346 + assert output_token_ids.shape[1] == (k + 1)
347 + assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
348 + if disable_bonus_tokens:
349 + assert torch.all(output_token_ids[:, -1] == -1)
350 + else:
351 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
352 + # Next only keep the first 2 draft tokens same as the zero temperature
353 + # tokens. For the remaining 3 choose some other tokens. In the
354 + # response we will expect the first 2 tokens to be the same as the
355 + # draft tokens and the rest as -1
356 + draft_token_ids_to_replace = get_draft_token_ids(
357 + batch_size, k, vocab_size, zero_temperature_token_ids)
358 + draft_token_ids = torch.cat(
359 + (draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
360 + output_token_ids = typical_acceptance_sampler(target_probs,
361 + bonus_token_ids,
362 + draft_token_ids)
363 + assert output_token_ids.shape[0] == batch_size
364 + assert output_token_ids.shape[1] == (k + 1)
365 + assert torch.all(output_token_ids[:, :2] == draft_token_ids[:, :2])
366 + assert torch.all(output_token_ids[:, -3:] == -1)
367 +
368 +
369 +@pytest.mark.parametrize("seed", list(range(1)))
370 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
371 +@pytest.mark.parametrize("device", CUDA_DEVICES)
372 +@torch.inference_mode()
373 +def test_accept_tokens_set_non_default_posteriors(seed: int,
374 + disable_bonus_tokens: bool,
375 + device: str):
376 + """
377 + Test the TypicalAcceptanceSampler with custom posterior thresholds and
378 + alpha values. This test verifies that by modifying the posterior
379 + thresholds and alpha values we can change the acceptance behavior of the
380 + sampler.
381 + """
382 + set_random_seed(seed)
383 + k = 5
384 + batch_size = 1
385 + vocab_size = 30_000
386 + torch.set_default_device(device)
387 + typical_acceptance_sampler = TypicalAcceptanceSampler(
388 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
389 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
390 + # Simulate temperature 0 probability distribution for target
391 + # probabilities and create target probabilities such that only 1 token
392 + # id has probability 1.0 and others have a very low probability of
393 + # 0.00001. Populate draft_token_ids such that they exclude the token_ids
394 + # with probability = 1.0. Without any changes to the posterior thresholds
395 + # none of the draft tokens are accepted.
396 + target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
397 + batch_size, k, vocab_size))
398 + target_probs[target_probs == 0] = 0.00001
399 + draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
400 + zero_temperature_token_ids)
401 + bonus_token_ids = torch.randint(low=0,
402 + high=vocab_size,
403 + size=(batch_size, 1),
404 + dtype=torch.int64)
405 + output_token_ids = typical_acceptance_sampler(target_probs,
406 + bonus_token_ids,
407 + draft_token_ids)
408 + assert output_token_ids.shape[0] == batch_size
409 + assert output_token_ids.shape[1] == (k + 1)
410 + assert torch.all(output_token_ids[:, 1:-1] == -1)
411 +
412 + # Change the posterior threshold values to 0.0 so that we will
413 + # now accept even draft tokens with very low probability in the
414 + # target distribution. Simulate and verify the same.
415 + typical_acceptance_sampler = TypicalAcceptanceSampler(
416 + strict_mode=True,
417 + disable_bonus_tokens=disable_bonus_tokens,
418 + posterior_threshold=0.0,
419 + posterior_alpha=0.0)
420 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
421 + output_token_ids = typical_acceptance_sampler(target_probs,
422 + bonus_token_ids,
423 + draft_token_ids)
424 + assert output_token_ids.shape[0] == batch_size
425 + assert output_token_ids.shape[1] == (k + 1)
426 + assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
427 + if disable_bonus_tokens:
428 + assert torch.all(output_token_ids[:, -1] == -1)
429 + else:
430 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
431 +
432 +
433 +@pytest.mark.parametrize("seed", list(range(10)))
434 +@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
435 +@pytest.mark.parametrize("device", CUDA_DEVICES)
436 +@torch.inference_mode()
437 +def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
438 + device: str):
439 + """
440 + Test the TypicalAcceptanceSampler's method for generating
441 + replacement token IDs.
442 +
443 + This test verifies that the `_replacement_token_ids` method of the
444 + TypicalAcceptanceSampler correctly identifies the token IDs to be used
445 + as replacements based on the target probability distribution.
446 + Specifically, it ensures that the method correctly identifies the
447 + tokens with the highest probability for each sequence in the batch.
448 + """
449 + set_random_seed(seed)
450 + k = 10
451 + batch_size = 5
452 + vocab_size = 30_000
453 + torch.set_default_device(device)
454 + typical_acceptance_sampler = TypicalAcceptanceSampler(
455 + strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
456 + typical_acceptance_sampler.init_gpu_tensors(rank=0)
457 + target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
458 + expected_replacement_tokens = -torch.ones(
459 + (batch_size, k), dtype=torch.long)
460 + expected_replacement_tokens[:, 0] = torch.argmax(target_probs[:, 0, :],
461 + dim=1)
462 + actual_replacement_tokens = (
463 + typical_acceptance_sampler._replacement_token_ids(target_probs))
464 + assert torch.all(expected_replacement_tokens == actual_replacement_tokens)
vllm/model_executor/layers/rejection_sampler.py
+10 -164
@@ -1,12 +1,15 @@
1 from functools import cached_property
2 -from typing import Optional, Tuple
2 +from typing import Tuple
3
4 import torch
5 import torch.jit
6 import torch.nn as nn
7
8 +from vllm.model_executor.layers.spec_decode_base_sampler import (
9 + SpecDecodeBaseSampler)
10
9 -class RejectionSampler(nn.Module):
11 +
12 +class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
13 """Apply modified rejection sampling as described in "Accelerating Large
14 Language Model Decoding with Speculative Sampling"
15 https://arxiv.org/pdf/2302.01318.pdf.
@@ -22,39 +25,11 @@ class RejectionSampler(nn.Module):
25 Require when bonus tokens will cause corrupt KV cache for
26 proposal methods that require KV cache.
27 strict_mode: Whether or not to perform shape/device/dtype checks
25 - during sampling. This catches correctness issues but adds
26 - nontrivial latency.
28 + during sampling. This catches correctness issues but adds
29 + nontrivial latency.
30 """
28 - super().__init__()
29 - self._disable_bonus_tokens = disable_bonus_tokens
30 - self._strict_mode = strict_mode
31 -
32 - # NOTE: A "bonus token" is accepted iff all proposal tokens are
33 - # accepted. There is always only one possible bonus token. We store this
34 - # value in a variable for readability.
35 - self._num_bonus_tokens = 1
36 -
37 - self.num_accepted_tokens: Optional[torch.Tensor] = None
38 - self.num_emitted_tokens: Optional[torch.Tensor] = None
39 - self.num_draft_tokens: int = 0
40 -
41 - def init_gpu_tensors(self, rank: int) -> None:
42 - assert self.num_accepted_tokens is None
43 - device = f"cuda:{rank}"
44 - self.num_accepted_tokens = torch.tensor(0,
45 - dtype=torch.long,
46 - device=device)
47 - self.num_emitted_tokens = torch.tensor(0,
48 - dtype=torch.long,
49 - device=device)
50 -
51 - @property
52 - def probs_dtype(self):
53 - return torch.float32
54 -
55 - @property
56 - def token_id_dtype(self):
57 - return torch.int64
31 + SpecDecodeBaseSampler.__init__(self, disable_bonus_tokens, strict_mode)
32 + nn.Module.__init__(self)
33
34 def forward(
35 self,
@@ -100,15 +75,8 @@ class RejectionSampler(nn.Module):
75 # Only perform shape/dtype/device checking in strict mode, as it adds
76 # overhead.
77 if self._strict_mode:
103 - self._raise_if_incorrect_shape(target_probs, bonus_token_ids,
104 - draft_probs, draft_token_ids)
105 - self._raise_if_incorrect_dtype(target_probs, bonus_token_ids,
78 + self._raise_if_incorrect_input(target_probs, bonus_token_ids,
79 draft_probs, draft_token_ids)
107 - self._raise_if_inconsistent_device(target_probs, bonus_token_ids,
108 - draft_probs, draft_token_ids)
109 - self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
110 - bonus_token_ids,
111 - draft_token_ids)
80
81 accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
82 target_probs,
@@ -272,128 +240,6 @@ class RejectionSampler(nn.Module):
240 """
241 return torch.finfo(self.probs_dtype).tiny
242
275 - def _create_output(
276 - self,
277 - accepted: torch.Tensor, # [batch_size, k]
278 - recovered_token_ids: torch.Tensor, # [batch_size, k]
279 - draft_token_ids: torch.Tensor, # [batch_size, k]
280 - bonus_token_ids: torch.Tensor, # [batch_size]
281 - ) -> torch.Tensor:
282 - """Format output. Returns a matrix of token ids. When
283 - a token is rejected via rejection sampling, all subsequent
284 - token ids are set to -1 for the sequence.
285 -
286 - shape = [batch_size, k + num_bonus_tokens]
287 - """
288 - bonus_token_ids = bonus_token_ids.squeeze()
289 - batch_size, k = recovered_token_ids.shape
290 -
291 - # Determine the index of the first False value for each row.
292 - limits = (accepted == 0).max(1).indices
293 - limits[~(accepted == 0).any(1)] = k
294 -
295 - # Create masks using the indices.
296 - indices = torch.arange(k, device=accepted.device).unsqueeze(0)
297 - accepted_mask = indices < limits.unsqueeze(1)
298 - after_false_mask = indices == limits.unsqueeze(1)
299 -
300 - # Create an extended output tensor
301 - output_with_bonus_tokens = -torch.ones(
302 - (batch_size, k + self._num_bonus_tokens),
303 - dtype=self.token_id_dtype,
304 - device=accepted.device)
305 - output = output_with_bonus_tokens[:, :k]
306 -
307 - # Fill in the first k columns of the output tensor using masks and data
308 - # tensors.
309 - torch.where(accepted_mask,
310 - draft_token_ids,
311 - -torch.ones_like(draft_token_ids),
312 - out=output)
313 -
314 - # Fill the last column.
315 - # We check output directly as accepted may have True values inconsistent
316 - # with causal acceptance.
317 - output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
318 - bonus_token_ids, -1)
319 -
320 - # We disable bonus tokens because it causes corrupt KV cache for
321 - # proposal methods that require KV cache. We can fix it by "prefilling"
322 - # the bonus token in the proposer. The following issue tracks the fix.
323 - # https://github.com/vllm-project/vllm/issues/4212
324 - if self._disable_bonus_tokens:
325 - output_with_bonus_tokens[:, -1] = -1
326 -
327 - # Fill the recovered token ids.
328 - output.mul_(~after_false_mask).add_(
329 - recovered_token_ids.mul(after_false_mask))
330 -
331 - self.num_accepted_tokens += accepted.sum()
332 - self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
333 - self.num_draft_tokens += batch_size * k
334 -
335 - return output_with_bonus_tokens
336 -
337 - def _raise_if_incorrect_shape(
338 - self,
339 - target_probs: torch.Tensor,
340 - bonus_token_ids: torch.Tensor,
341 - draft_probs: torch.Tensor,
342 - draft_token_ids: torch.Tensor,
343 - ) -> None:
344 - (target_batch_size, num_target_probs,
345 - target_vocab_size) = target_probs.shape
346 - bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
347 - draft_batch_size, num_draft_probs, draft_vocab_size = draft_probs.shape
348 - draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
349 -
350 - assert draft_batch_size == target_batch_size
351 - assert num_draft_probs == num_target_probs
352 - assert (draft_vocab_size == target_vocab_size
353 - ), f"{draft_vocab_size=} {target_vocab_size=}"
354 -
355 - assert draft_token_ids_batch_size == draft_batch_size
356 - assert num_draft_token_ids == num_draft_probs
357 -
358 - assert bonus_batch_size == target_batch_size
359 - assert num_bonus_tokens == self._num_bonus_tokens
360 -
361 - def _raise_if_incorrect_dtype(
362 - self,
363 - target_probs: torch.Tensor,
364 - bonus_token_ids: torch.Tensor,
365 - draft_probs: torch.Tensor,
366 - draft_token_ids: torch.Tensor,
367 - ) -> None:
368 - assert all(probs.dtype == self.probs_dtype
369 - for probs in [target_probs, draft_probs])
370 - assert all(token_ids.dtype == self.token_id_dtype
371 - for token_ids in [bonus_token_ids, draft_token_ids])
372 -
373 - def _raise_if_inconsistent_device(
374 - self,
375 - target_probs: torch.Tensor,
376 - bonus_token_ids: torch.Tensor,
377 - draft_probs: torch.Tensor,
378 - draft_token_ids: torch.Tensor,
379 - ) -> None:
380 - devices = [
381 - t.device for t in
382 - [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
383 - ]
384 - assert all([devices[0] == device for device in devices])
385 -
386 - def _raise_if_out_of_bounds_vocab(
387 - self,
388 - vocab_size: int,
389 - bonus_token_ids: torch.Tensor,
390 - draft_token_ids: torch.Tensor,
391 - ) -> None:
392 - assert torch.all(bonus_token_ids < vocab_size)
393 - assert torch.all(bonus_token_ids >= 0)
394 - assert torch.all(draft_token_ids < vocab_size)
395 - assert torch.all(draft_token_ids >= 0)
396 -
243
244 # torch.multinomial forces a GPU<->CPU sync.
245 # Therefore, we use an optimized implementation instead that skips the sync.
vllm/model_executor/layers/spec_decode_base_sampler.py new
+206
@@ -0,0 +1,206 @@
1 +from typing import Optional
2 +
3 +import torch
4 +
5 +
6 +class SpecDecodeBaseSampler():
7 + """Base class for samplers used for Speculative Decoding verification
8 + step.
9 + """
10 +
11 + def __init__(self,
12 + disable_bonus_tokens: bool = True,
13 + strict_mode: bool = False):
14 + """Base class constructor.
15 + Args:
16 + disable_bonus_tokens: Whether or not to disable the bonus token.
17 + Require when bonus tokens will cause corrupt KV cache for
18 + proposal methods that require KV cache.
19 + strict_mode: Whether or not to perform shape/device/dtype checks
20 + during sampling. This catches correctness issues but adds
21 + nontrivial latency.
22 + """
23 + super().__init__()
24 + self._disable_bonus_tokens = disable_bonus_tokens
25 + self._strict_mode = strict_mode
26 +
27 + # NOTE: A "bonus token" is accepted iff all proposal tokens are
28 + # accepted. There is always only one possible bonus token. We store this
29 + # value in a variable for readability.
30 + self._num_bonus_tokens = 1
31 +
32 + self.num_accepted_tokens: Optional[torch.Tensor] = None
33 + self.num_emitted_tokens: Optional[torch.Tensor] = None
34 + self.num_draft_tokens: int = 0
35 +
36 + def init_gpu_tensors(self, rank: int) -> None:
37 + assert self.num_accepted_tokens is None
38 + device = f"cuda:{rank}"
39 + self.num_accepted_tokens = torch.tensor(0,
40 + dtype=torch.long,
41 + device=device)
42 + self.num_emitted_tokens = torch.tensor(0,
43 + dtype=torch.long,
44 + device=device)
45 +
46 + @property
47 + def probs_dtype(self):
48 + return torch.float32
49 +
50 + @property
51 + def token_id_dtype(self):
52 + return torch.int64
53 +
54 + def _create_output(
55 + self,
56 + accepted: torch.Tensor, # [batch_size, k]
57 + substitute_token_ids: torch.Tensor, # [batch_size, k]
58 + draft_token_ids: torch.Tensor, # [batch_size, k]
59 + bonus_token_ids: torch.Tensor, # [batch_size]
60 + ) -> torch.Tensor:
61 + """Format output. Returns a matrix of token ids. When
62 + a token is rejected via sampling, all subsequent token ids are
63 + set to -1 for the sequence.
64 +
65 + Args:
66 + accepted: A boolean tensor indicating if the corresponding
67 + draft token in draft_token_ids should be accepted or not.
68 + substitute_token_ids: A tensor of token_ids that can be used
69 + as substitutes for the draft token ids if the proposed token
70 + is rejected.
71 + draft_token_ids: A tensor of token ids speculated by the
72 + draft model.
73 + bonus_token_ids: Token ids to use as the bonus token if
74 + all the draft tokens are accepted.
75 + Returns:
76 + A tensor containing the accepted token ids. The shape of the
77 + tensor is [batch_size, k + num_bonus_tokens]
78 + """
79 + batch_size, k = substitute_token_ids.shape
80 + bonus_token_ids = bonus_token_ids.squeeze()
81 + # Determine the index of the first False value for each row.
82 + limits = (accepted == 0).max(1).indices
83 + limits[~(accepted == 0).any(1)] = k
84 +
85 + # Create masks using the indices.
86 + indices = torch.arange(k, device=accepted.device).unsqueeze(0)
87 + accepted_mask = indices < limits.unsqueeze(1)
88 + after_false_mask = indices == limits.unsqueeze(1)
89 +
90 + # Create an extended output tensor
91 + output_with_bonus_tokens = -torch.ones(
92 + (batch_size, k + self._num_bonus_tokens),
93 + dtype=self.token_id_dtype,
94 + device=accepted.device)
95 + output = output_with_bonus_tokens[:, :k]
96 +
97 + # Fill in the first k columns of the output tensor using masks and data
98 + # tensors.
99 + output[:, :k] = torch.where(accepted_mask, draft_token_ids,
100 + -torch.ones_like(draft_token_ids))
101 +
102 + # Fill the last column.
103 + # We check output directly as accepted may have True values inconsistent
104 + # with causal acceptance.
105 + output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
106 + bonus_token_ids, -1)
107 +
108 + # We disable bonus tokens because it causes corrupt KV cache for
109 + # proposal methods that require KV cache. We can fix it by "prefilling"
110 + # the bonus token in the proposer. The following issue tracks the fix.
111 + # https://github.com/vllm-project/vllm/issues/4212
112 + if self._disable_bonus_tokens:
113 + output_with_bonus_tokens[:, -1] = -1
114 +
115 + # Fill the recovered token ids.
116 + output.mul_(~after_false_mask).add_(
117 + substitute_token_ids.mul(after_false_mask))
118 +
119 + self.num_accepted_tokens += accepted.sum()
120 + self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
121 + self.num_draft_tokens += batch_size * k
122 +
123 + return output_with_bonus_tokens
124 +
125 + def _raise_if_incorrect_input(
126 + self,
127 + target_probs: torch.Tensor,
128 + draft_token_ids: torch.Tensor,
129 + bonus_token_ids: torch.Tensor,
130 + draft_probs: Optional[torch.Tensor] = None,
131 + ) -> None:
132 + self._raise_if_incorrect_shape(target_probs, draft_token_ids,
133 + bonus_token_ids, draft_probs)
134 + self._raise_if_incorrect_dtype(target_probs, draft_token_ids,
135 + bonus_token_ids, draft_probs)
136 + self._raise_if_inconsistent_device(target_probs, draft_token_ids,
137 + bonus_token_ids, draft_probs)
138 + self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
139 + draft_token_ids, bonus_token_ids)
140 +
141 + def _raise_if_incorrect_shape(
142 + self,
143 + target_probs: torch.Tensor,
144 + draft_token_ids: torch.Tensor,
145 + bonus_token_ids: torch.Tensor,
146 + draft_probs: Optional[torch.Tensor] = None,
147 + ) -> None:
148 + (target_batch_size, num_target_probs,
149 + target_vocab_size) = target_probs.shape
150 +
151 + # validate the shape of draft token ids.
152 + draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
153 + assert draft_token_ids_batch_size == target_batch_size
154 + assert num_draft_token_ids == num_target_probs
155 +
156 + # validate the shape of bonus token ids
157 + bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
158 + assert bonus_batch_size == target_batch_size
159 + assert num_bonus_tokens == self._num_bonus_tokens
160 +
161 + # validate the shape of draft probs if it is set
162 + if draft_probs is not None:
163 + (draft_batch_size, num_draft_probs,
164 + draft_vocab_size) = draft_probs.shape
165 + assert draft_batch_size == target_batch_size
166 + assert num_draft_probs == num_target_probs
167 + assert (draft_vocab_size == target_vocab_size
168 + ), f"{draft_vocab_size=} {target_vocab_size=}"
169 +
170 + def _raise_if_incorrect_dtype(
171 + self,
172 + target_probs: torch.Tensor,
173 + draft_token_ids: torch.Tensor,
174 + bonus_token_ids: torch.Tensor,
175 + draft_probs: Optional[torch.Tensor] = None,
176 + ) -> None:
177 + assert target_probs.dtype == self.probs_dtype
178 + assert draft_token_ids.dtype == self.token_id_dtype
179 + assert bonus_token_ids.dtype == self.token_id_dtype
180 + if draft_probs is not None:
181 + assert draft_probs.dtype == self.probs_dtype
182 +
183 + def _raise_if_inconsistent_device(
184 + self,
185 + target_probs: torch.Tensor,
186 + draft_token_ids: torch.Tensor,
187 + bonus_token_ids: torch.Tensor,
188 + draft_probs: Optional[torch.Tensor] = None,
189 + ) -> None:
190 + devices = [
191 + t.device for t in
192 + [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
193 + if t is not None
194 + ]
195 + assert all([devices[0] == device for device in devices])
196 +
197 + def _raise_if_out_of_bounds_vocab(
198 + self,
199 + vocab_size: int,
200 + draft_token_ids: torch.Tensor,
201 + bonus_token_ids: torch.Tensor,
202 + ) -> None:
203 + assert torch.all(bonus_token_ids < vocab_size)
204 + assert torch.all(bonus_token_ids >= 0)
205 + assert torch.all(draft_token_ids < vocab_size)
206 + assert torch.all(draft_token_ids >= 0)
vllm/model_executor/layers/typical_acceptance_sampler.py new
+186
@@ -0,0 +1,186 @@
1 +import torch
2 +import torch.jit
3 +import torch.nn as nn
4 +
5 +from vllm.model_executor.layers.spec_decode_base_sampler import (
6 + SpecDecodeBaseSampler)
7 +
8 +
9 +class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
10 + """Apply typical acceptance sampling as described in section 3.3.1 in
11 + "MEDUSA: Simple LLM Inference Acceleration Framework with
12 + Multiple Decoding Heads"
13 + https://arxiv.org/pdf/2401.10774
14 + """
15 +
16 + def __init__(
17 + self,
18 + disable_bonus_tokens: bool = False,
19 + strict_mode: bool = False,
20 + posterior_threshold: float = 0.09,
21 + posterior_alpha: float = 0.3,
22 + ):
23 + """Create a Typical Acceptance Sampler.
24 +
25 + Args:
26 + disable_bonus_tokens: Whether or not to disable the bonus token.
27 + Require when bonus tokens will cause corrupt KV cache for
28 + proposal methods that require KV cache.
29 + strict_mode: Whether or not to perform shape/device/dtype checks
30 + during sampling. This catches correctness issues but adds
31 + nontrivial latency.
32 + posterior_threshold : A threshold value that sets a lower bound
33 + on the posterior probability of a token in target model for it
34 + to be accepted. Default is 0.09
35 + posterior_alpha : A scaling factor for the entropy-based
36 + threshold in typical acceptance sampling. Typically defaults to
37 + sqrt of posterior_threshold and is set to 0.3.
38 + """
39 + SpecDecodeBaseSampler.__init__(
40 + self,
41 + disable_bonus_tokens=disable_bonus_tokens,
42 + strict_mode=strict_mode)
43 + nn.Module.__init__(self)
44 + self._posterior_threshold = posterior_threshold
45 + self._posterior_alpha = posterior_alpha
46 +
47 + def forward(
48 + self,
49 + target_probs: torch.Tensor,
50 + bonus_token_ids: torch.Tensor,
51 + draft_token_ids: torch.Tensor,
52 + ) -> torch.Tensor:
53 + """Sample token ids using typical acceptance sampling. This accepts
54 + or rejects tokens proposed by the draft model using the probability
55 + of each token according to the draft and target models.
56 +
57 + In the worst case where all draft tokens are rejected, it is guaranteed
58 + one token will be emitted.
59 +
60 + In the case where all draft tokens are accepted, the bonus token will be
61 + accepted conditioned on self._disable_bonus_tokens being false.
62 +
63 + Args:
64 + target_probs: The probability distribution over token ids given
65 + context according to the target model.
66 + shape = [batch_size, num_speculative_tokens, vocab_size]
67 +
68 + bonus_token_ids: The "bonus" token ids that are accepted iff all
69 + speculative tokens in a sequence are accepted.
70 + shape = [batch_size, num_bonus_tokens]
71 +
72 + draft_token_ids: The token ids that were sampled from the draft
73 + probabilities.
74 + shape = [batch_size, num_speculative_tokens]
75 +
76 + Returns:
77 + output_token_ids: The token ids sampled via rejection sampling,
78 + or -1 if unable to sample a token because the previous token
79 + was rejected.
80 + shape = [batch_size, num_speculative_tokens + num_bonus_tokens]
81 + """
82 + # Only perform shape/dtype/device checking in strict mode, as it adds
83 + # overhead.
84 + if self._strict_mode:
85 + self._raise_if_incorrect_input(target_probs, draft_token_ids,
86 + bonus_token_ids)
87 + accepted = self._evaluate_accepted_tokens(target_probs,
88 + draft_token_ids)
89 + recovered_token_ids = self._replacement_token_ids(target_probs)
90 + output_token_ids = self._create_output(accepted, recovered_token_ids,
91 + draft_token_ids,
92 + bonus_token_ids)
93 + return output_token_ids
94 +
95 + def _evaluate_accepted_tokens(self, target_probs, draft_token_ids):
96 + r"""
97 + Evaluates and returns a mask of accepted tokens based on the
98 + posterior probabilities.
99 +
100 + Parameters:
101 + ----------
102 + target_probs : torch.Tensor
103 + A tensor of shape (batch_size, k, vocab_size) representing
104 + the probabilities of each token in the vocabulary for each
105 + position in the proposed sequence. This is the distribution
106 + generated by the target model.
107 + draft_token_ids : torch.Tensor
108 + A tensor of shape (batch_size, k) representing the proposed
109 + token ids.
110 +
111 + A draft token_id x_{n+k} is accepted if it satisfies the
112 + following condition
113 +
114 + .. math::
115 + p_{\text{original}}(x_{n+k} | x_1, x_2, \dots, x_{n+k-1}) >
116 + \min \left( \epsilon, \delta * \exp \left(
117 + -H(p_{\text{original}}(
118 + \cdot | x_1, x_2, \ldots, x_{n+k-1})) \right) \right)
119 +
120 + where :math:`p_{\text{original}}` corresponds to target_probs
121 + and :math:`\epsilon` and :math:`\delta` correspond to hyperparameters
122 + specified using self._posterior_threshold and self._posterior_alpha
123 +
124 + This method computes the posterior probabilities for the given
125 + draft token ids based on the provided target probabilities. It
126 + calculates the entropy of the posterior distribution and determines
127 + a dynamic threshold for each token position using the provided
128 + posterior_threshold and posterior_alpha values. The method then
129 + returns a boolean mask indicating which tokens can be accepted.
130 +
131 + Returns:
132 + -------
133 + torch.Tensor
134 + A boolean tensor of shape (batch_size, k) where each element
135 + indicates whether the corresponding draft token has been accepted
136 + or rejected. True indicates acceptance and false indicates
137 + rejection.
138 +
139 + """
140 + device = target_probs.device
141 + candidates_prob = torch.gather(
142 + target_probs, dim=-1,
143 + index=draft_token_ids.unsqueeze(-1)).squeeze(-1)
144 + # A small constant added to prevent computing the logarithm of zero,
145 + # which can lead to undefined values.
146 + epsilon = 1e-5
147 + posterior_entropy = -torch.sum(
148 + target_probs * torch.log(target_probs + epsilon), dim=-1)
149 + threshold = torch.minimum(
150 + torch.ones_like(posterior_entropy, device=device) *
151 + self._posterior_threshold,
152 + torch.exp(-posterior_entropy) * self._posterior_alpha,
153 + )
154 + accepted_mask = candidates_prob > threshold
155 + return accepted_mask
156 +
157 + def _replacement_token_ids(self, target_probs):
158 + """
159 + Generate one replacement token ID for each sequence based on target
160 + probabilities. The replacement token is used as the fallback option
161 + if typical acceptance sampling does not accept any draft tokens for
162 + that particular sequence.
163 +
164 + This method computes the token IDs to be replaced by selecting the
165 + token with the highest probability for each sequence in the first
166 + position. The rest of the output is filled with -1.
167 +
168 + Parameters
169 + ----------
170 + target_probs : torch.Tensor
171 + A tensor of shape (batch_size, k, vocab_size) containing
172 + the target probability distribution
173 +
174 + Returns
175 + -------
176 + torch.Tensor
177 + A tensor of shape (batch_size, k) with the replacement
178 + token IDs. Only the first column is set, and the rest of the
179 + columns are filled with -1.
180 + """
181 + max_indices = torch.argmax(target_probs[:, 0, :], dim=1)
182 + output = -torch.ones((target_probs.shape[0], target_probs.shape[1]),
183 + dtype=self.token_id_dtype,
184 + device=target_probs.device)
185 + output[:, 0] = max_indices
186 + return output