@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79d64c4954
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Speculative decoding 1/9] Optimized rejection sampler (#2336)

Cade Daniel committed Jan 9, 2024 at 15:38 UTC 79d64c495463665cf80937a05226806785cfa583
2 files changed +784
tests/samplers/test_rejection_sampler.py new
+392
@@ -0,0 +1,392 @@
1 +"""Tests for rejection sampling."""
2 +import pytest
3 +from typing import List, Tuple
4 +
5 +import torch
6 +import torch.nn.functional as F
7 +
8 +from vllm.model_executor.utils import set_random_seed
9 +
10 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11 +
12 +
13 +def mock_causal_accepted_tensor(
14 + k: int, last_accepted_indices: torch.Tensor) -> torch.Tensor:
15 + """Generate an "accepted" tensor which should yield causally-accepted tokens
16 + up to last accepted indices.
17 +
18 + Tokens after last_accepted_indices+1 may also be accepted, although they
19 + will not be causally accepted.
20 + """
21 + batch_size = last_accepted_indices.shape[0]
22 +
23 + accepted = (torch.arange(k).expand(batch_size, k) <=
24 + last_accepted_indices.unsqueeze(-1).broadcast_to(
25 + batch_size, k)).to(device="cuda")
26 +
27 + # Sprinkle accepted values after the contiguous initial accepted values.
28 + # This replicates the behavior of rejection sampling, which may "accept"
29 + # a token that cannot be accepted because of causality.
30 + sprinkle_candidates = (
31 + torch.arange(k).expand(batch_size, k) >
32 + last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) + 1)
33 + sprinkle = torch.rand(batch_size, k, device="cuda") > 0.5
34 + accepted[sprinkle_candidates] = sprinkle[sprinkle_candidates]
35 + return accepted
36 +
37 +
38 +@pytest.mark.parametrize("seed", list(range(10)))
39 +@pytest.mark.parametrize(
40 + "which_tokens_accepted",
41 + ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
42 +@torch.inference_mode()
43 +def test_correct_output_format(which_tokens_accepted: str, seed: int):
44 + """Verify the output has correct format given predetermined accepted matrix.
45 + """
46 + set_random_seed(seed)
47 +
48 + batch_size = 10
49 + k = 5
50 + vocab_size = 3000
51 +
52 + if which_tokens_accepted == "all_tokens_accepted":
53 + accepted = mock_causal_accepted_tensor(
54 + k, -1 + k * torch.ones((batch_size, ), dtype=torch.long))
55 + elif which_tokens_accepted == "no_tokens_accepted":
56 + accepted = mock_causal_accepted_tensor(
57 + k, -torch.ones((batch_size, ), dtype=torch.long))
58 + elif which_tokens_accepted == "some_tokens_accepted":
59 + last_accepted_indices = torch.randint(low=-1,
60 + high=k,
61 + size=(batch_size, ))
62 + accepted = mock_causal_accepted_tensor(k, last_accepted_indices)
63 + else:
64 + raise AssertionError()
65 +
66 + recovered_token_ids = torch.randint(low=0,
67 + high=vocab_size,
68 + size=(batch_size, k),
69 + dtype=torch.int64,
70 + device="cuda")
71 + draft_token_ids = torch.randint(low=0,
72 + high=vocab_size,
73 + size=(batch_size, k),
74 + dtype=torch.int64,
75 + device="cuda")
76 + bonus_token_ids = torch.randint(low=0,
77 + high=vocab_size,
78 + size=(batch_size, 1),
79 + dtype=torch.int64,
80 + device="cuda")
81 +
82 + rejection_sampler = RejectionSampler()
83 + rejection_sampler.init_gpu_tensors(rank=0)
84 + output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
85 + accepted,
86 + recovered_token_ids,
87 + draft_token_ids,
88 + bonus_token_ids,
89 + )
90 +
91 + if which_tokens_accepted == "all_tokens_accepted":
92 + # Expect all tokens to be equal to draft tokens.
93 + assert torch.equal(output_token_ids[:, :-1], draft_token_ids)
94 +
95 + # Expect all bonus tokens to be included.
96 + assert torch.equal(output_token_ids[:, -1:], bonus_token_ids)
97 + elif which_tokens_accepted == "no_tokens_accepted":
98 + # Expect first token to be equal to recovered tokens.
99 + assert torch.equal(output_token_ids[:, 0], recovered_token_ids[:, 0])
100 +
101 + # Expect everything else to be -1.
102 + assert torch.equal(output_token_ids[:, 1:],
103 + torch.ones_like(output_token_ids[:, 1:]) * -1)
104 + elif which_tokens_accepted == "some_tokens_accepted":
105 + recovered_plus_bonus = torch.cat(
106 + (recovered_token_ids, bonus_token_ids), dim=-1)
107 + # Assert first rejected token is a recovered token or bonus token.
108 + assert torch.equal(
109 + recovered_plus_bonus[torch.arange(0, batch_size),
110 + last_accepted_indices + 1],
111 + output_token_ids[torch.arange(0, batch_size),
112 + last_accepted_indices + 1])
113 +
114 + # Assert every subsequent token is -1.
115 + subsequent_mask = torch.arange(0, k + 1).expand(
116 + batch_size, k + 1) >= (last_accepted_indices + 2).unsqueeze(-1)
117 + assert torch.all(output_token_ids[subsequent_mask] == -1)
118 +
119 +
120 +@pytest.mark.parametrize("k", list(range(1, 6)))
121 +@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
122 +@pytest.mark.parametrize("batch_size", list(range(1, 32)))
123 +@torch.inference_mode()
124 +def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
125 + rejection_sampler = RejectionSampler()
126 + rejection_sampler.init_gpu_tensors(rank=0)
127 +
128 + draft_probs = torch.rand(batch_size,
129 + k,
130 + vocab_size,
131 + dtype=torch.float32,
132 + device="cuda")
133 + target_probs = torch.rand(batch_size,
134 + k,
135 + vocab_size,
136 + dtype=torch.float32,
137 + device="cuda")
138 + bonus_token_ids = torch.randint(low=0,
139 + high=vocab_size,
140 + size=(batch_size, 1),
141 + dtype=torch.int64,
142 + device="cuda")
143 + draft_token_ids = torch.randint(low=0,
144 + high=vocab_size,
145 + size=(batch_size, k),
146 + dtype=torch.int64,
147 + device="cuda")
148 +
149 + rejection_sampler(target_probs, bonus_token_ids, draft_probs,
150 + draft_token_ids)
151 +
152 +
153 +@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
154 +@pytest.mark.parametrize("which_token_ids",
155 + ["bonus_token_ids", "draft_token_ids"])
156 +@torch.inference_mode()
157 +def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
158 + which_token_ids: str):
159 + k = 3
160 + batch_size = 5
161 + vocab_size = 30_000
162 +
163 + rejection_sampler = RejectionSampler(strict_mode=True)
164 + rejection_sampler.init_gpu_tensors(rank=0)
165 +
166 + draft_probs = torch.rand(batch_size,
167 + k,
168 + vocab_size,
169 + dtype=torch.float32,
170 + device="cuda")
171 + target_probs = torch.rand(batch_size,
172 + k,
173 + vocab_size,
174 + dtype=torch.float32,
175 + device="cuda")
176 + bonus_token_ids = torch.randint(low=0,
177 + high=vocab_size,
178 + size=(batch_size, 1),
179 + dtype=torch.int64,
180 + device="cuda")
181 + draft_token_ids = torch.randint(low=0,
182 + high=vocab_size,
183 + size=(batch_size, k),
184 + dtype=torch.int64,
185 + device="cuda")
186 +
187 + oob_token_ids = None
188 + if which_token_ids == "bonus_token_ids":
189 + oob_token_ids = bonus_token_ids
190 + elif which_token_ids == "draft_token_ids":
191 + oob_token_ids = draft_token_ids
192 + else:
193 + raise AssertionError()
194 +
195 + if above_or_below_vocab_range == "above":
196 + rogue_token_id = vocab_size + 1
197 + elif above_or_below_vocab_range == "below":
198 + rogue_token_id = -1
199 + else:
200 + raise AssertionError()
201 +
202 + oob_token_ids[0][0] = rogue_token_id
203 +
204 + with pytest.raises(AssertionError):
205 + rejection_sampler(target_probs, bonus_token_ids, draft_probs,
206 + draft_token_ids)
207 +
208 +
209 +@pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
210 +@pytest.mark.parametrize("seed", list(range(5)))
211 +@torch.inference_mode()
212 +def test_rejection_sampling_approximates_target_distribution(
213 + seed: int, draft_and_target_probs_equal: bool):
214 + """Verify rejection sampling approximates target distribution,
215 + despite sampling from a potentially distinct draft distribution.
216 +
217 + This is done by first creating a random target probability
218 + distribution and a random draft probability distribution. We then
219 + sample token ids from the rejection sampler using these draft
220 + and target distributions. The samples are used to estimate
221 + the output probability distribution, which we expect to approximate
222 + the target distribution.
223 +
224 + A basic distance metric is used to determine similarity between
225 + distributions.
226 +
227 + We expect that as we increase the number of samples,
228 + the distance between the observed distribution and the target
229 + distribution decreases. To measure this, we compare the distance
230 + of the observed distribution against both the target distribution
231 + and a uniform random distribution. We expect the distance between
232 + the observed distribution and the target distribution to improve
233 + much more than the distance improvement between the observed
234 + distribution and the random distribution.
235 +
236 + When draft_and_target_probs_equal=True, the draft and target
237 + probabilities are exactly equal. Rejection sampling should
238 + still work without any NaNs or exceptions.
239 + """
240 + set_random_seed(seed)
241 +
242 + helper = _CorrectnessTestHelper(
243 + vocab_size=10,
244 + rejection_sampler=RejectionSampler(),
245 + )
246 +
247 + draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
248 + draft_and_target_probs_equal)
249 +
250 + sample_sizes = [10, 100, 1_000, 10_000, 100_000]
251 + distance_wrt_reference = []
252 + distance_wrt_target = []
253 +
254 + for num_samples in sample_sizes:
255 + (reference_vs_rejsample_dist,
256 + target_vs_rejsample_dist) = helper.run_and_compare_distributions(
257 + draft_probs,
258 + target_probs,
259 + reference_probs,
260 + num_samples,
261 + )
262 +
263 + distance_wrt_reference.append(reference_vs_rejsample_dist)
264 + distance_wrt_target.append(target_vs_rejsample_dist)
265 +
266 + relative_change_in_distance_wrt_target = get_ratio_first_to_last(
267 + distance_wrt_target)
268 + relative_change_in_distance_wrt_reference = get_ratio_first_to_last(
269 + distance_wrt_reference)
270 +
271 + print(f"{num_samples=} {target_vs_rejsample_dist=:.05f} "
272 + f"{reference_vs_rejsample_dist=:.05f}")
273 + print(f"{num_samples=} {relative_change_in_distance_wrt_target=:.02f} "
274 + f"{relative_change_in_distance_wrt_reference=:.02f}")
275 +
276 + relative_change_in_distance_wrt_target = get_ratio_first_to_last(
277 + distance_wrt_target)
278 + relative_change_in_distance_wrt_reference = get_ratio_first_to_last(
279 + distance_wrt_reference)
280 +
281 + expected_improvement_multiplier = 20
282 + assert (relative_change_in_distance_wrt_target >
283 + relative_change_in_distance_wrt_reference *
284 + expected_improvement_multiplier)
285 +
286 +
287 +def get_ratio_first_to_last(elements: List[float]) -> float:
288 + return elements[0] / elements[-1]
289 +
290 +
291 +class _CorrectnessTestHelper:
292 + """Class that packages together logic required for the unit-level
293 + rejection sampling correctness test.
294 + """
295 +
296 + def __init__(self, vocab_size: int, rejection_sampler: RejectionSampler):
297 + self.rejection_sampler = rejection_sampler
298 + self.vocab_size = vocab_size
299 + self.vocab_range = (0, vocab_size)
300 +
301 + self.rejection_sampler.init_gpu_tensors(rank=0)
302 +
303 + # Keep test simple, use k=1
304 + self.k = 1
305 +
306 + # Bonus tokens not used, but rejection sampler requires
307 + # correct shape.
308 + self.num_bonus_tokens = 1
309 +
310 + def generate_probs_for_test(
311 + self, draft_and_target_probs_equal: bool
312 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
313 + draft_probs, target_probs = [
314 + F.softmax(
315 + torch.rand(self.vocab_size, dtype=torch.float32),
316 + dim=-1,
317 + ) for _ in range(2)
318 + ]
319 +
320 + num_reference_probs = 100
321 + reference_probs = F.softmax(
322 + torch.rand(num_reference_probs,
323 + self.vocab_size,
324 + dtype=torch.float32),
325 + dim=-1,
326 + )
327 +
328 + if draft_and_target_probs_equal:
329 + target_probs = draft_probs.clone()
330 +
331 + return draft_probs, target_probs, reference_probs
332 +
333 + def run_and_compare_distributions(self, draft_probs: torch.Tensor,
334 + target_probs: torch.Tensor,
335 + reference_probs: torch.Tensor,
336 + num_samples: int) -> Tuple[float, float]:
337 + # Sample using rejection sampling.
338 + rej_sample_probs = self._estimate_rejection_sampling_pdf(
339 + draft_probs, target_probs, num_samples)
340 +
341 + # Average distance from reference probs.
342 + reference_vs_rejsample_dist = torch.dist(
343 + reference_probs,
344 + rej_sample_probs).item() / reference_probs.shape[0]
345 + target_vs_rejsample_dist = torch.dist(target_probs,
346 + rej_sample_probs).item()
347 +
348 + return reference_vs_rejsample_dist, target_vs_rejsample_dist
349 +
350 + def _estimate_rejection_sampling_pdf(
351 + self,
352 + draft_probs: torch.Tensor,
353 + target_probs: torch.Tensor,
354 + num_samples: int,
355 + ) -> torch.Tensor:
356 + # Repeat draft probs num_samples times.
357 + draft_probs = draft_probs.reshape(1, self.k, self.vocab_size).repeat(
358 + num_samples, 1, 1)
359 +
360 + # Repeat target probs num_samples * k times.
361 + # Rejection sampler requires bonus token probs, but they aren't used.
362 + target_probs = target_probs.reshape(1, 1, self.vocab_size).repeat(
363 + num_samples, self.k, 1)
364 +
365 + # Randomly sample draft token ids from draft probs.
366 + draft_token_ids = torch.multinomial(draft_probs[:, 0, :],
367 + num_samples=1,
368 + replacement=True).reshape(
369 + num_samples, self.k)
370 +
371 + # Bonus tokens not used but required.
372 + bonus_token_ids = torch.zeros((1, self.num_bonus_tokens),
373 + dtype=torch.int64,
374 + device="cuda").repeat(num_samples, 1)
375 +
376 + # Get output tokens via rejection sampling.
377 + output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
378 + bonus_token_ids.to("cuda"),
379 + draft_probs.to("cuda"),
380 + draft_token_ids.to("cuda"))
381 +
382 + # Remove bonus tokens
383 + output_token_ids = output_token_ids[:, :-1].flatten()
384 +
385 + # Estimate probability density function
386 + hist = torch.histogram(output_token_ids.to(dtype=torch.float,
387 + device="cpu"),
388 + bins=self.vocab_size,
389 + range=self.vocab_range,
390 + density=True)
391 +
392 + return hist.hist
vllm/model_executor/layers/rejection_sampler.py new
+392
@@ -0,0 +1,392 @@
1 +from typing import Tuple, Optional
2 +from functools import cached_property
3 +
4 +import torch
5 +import torch.nn as nn
6 +import torch.jit
7 +
8 +
9 +class RejectionSampler(nn.Module):
10 + """Apply modified rejection sampling as described in "Accelerating Large
11 + Language Model Decoding with Speculative Sampling"
12 + https://arxiv.org/pdf/2302.01318.pdf.
13 + """
14 +
15 + def __init__(self, strict_mode: bool = False):
16 + """Create a rejection sampler.
17 +
18 + Args:
19 + strict_mode: Whether or not to perform shape/device/dtype checks
20 + during sampling. This catches correctness issues but adds
21 + nontrivial latency.
22 + """
23 + super().__init__()
24 + self.probs_dtype = torch.float32
25 + self.token_id_dtype = torch.int64
26 + self._strict_mode = strict_mode
27 +
28 + # NOTE: A "bonus token" is accepted iff all proposal tokens are
29 + # accepted. There is always only one possible bonus token. We store this
30 + # value in a variable for readability.
31 + self._num_bonus_tokens = 1
32 +
33 + self.num_accepted_tokens: Optional[torch.Tensor] = None
34 + self.num_emitted_tokens: Optional[torch.Tensor] = None
35 + self.num_draft_tokens: int = 0
36 +
37 + def init_gpu_tensors(self, rank: int) -> None:
38 + assert self.num_accepted_tokens is None
39 + device = f"cuda:{rank}"
40 + self.num_accepted_tokens = torch.tensor(0,
41 + dtype=torch.long,
42 + device=device)
43 + self.num_emitted_tokens = torch.tensor(0,
44 + dtype=torch.long,
45 + device=device)
46 +
47 + def forward(
48 + self,
49 + target_probs: torch.Tensor,
50 + bonus_token_ids: torch.Tensor,
51 + draft_probs: torch.Tensor,
52 + draft_token_ids: torch.Tensor,
53 + ) -> torch.Tensor:
54 + """Sample token ids using rejection sampling. This accepts or rejects
55 + tokens proposed by the draft model using the probability of each token
56 + according to the draft and target models.
57 +
58 + In the worst case where all draft tokens are rejected, it is guaranteed
59 + one correct token will be emitted.
60 +
61 + In the case where all draft tokens are accepted, a bonus token will be
62 + accepted as its cheap to have the target model score this speculative
63 + sequence.
64 +
65 + Args:
66 + target_probs: The probability distribution over token ids given
67 + context according to the target model.
68 + shape = [batch_size, num_speculative_tokens, vocab_size]
69 +
70 + bonus_token_ids: The "bonus" token ids that are accepted iff all
71 + speculative tokens in a sequence are accepted.
72 + shape = [batch_size, num_bonus_tokens]
73 +
74 + draft_probs: The probability distribution over token ids given
75 + context according to the draft model.
76 + shape = [batch_size, num_speculative_tokens, vocab_size]
77 +
78 + draft_token_ids: The token ids that were sampled from the draft
79 + probabilities.
80 + shape = [batch_size, num_speculative_tokens]
81 +
82 + Returns:
83 + output_token_ids: The token ids sampled via rejection sampling,
84 + or -1 if unable to sample a token because the previous token
85 + was rejected.
86 + shape = [batch_size, num_speculative_tokens + num_bonus_tokens]
87 + """
88 + # Only perform shape/dtype/device checking in strict mode, as it adds
89 + # overhead.
90 + if self._strict_mode:
91 + self._raise_if_incorrect_shape(target_probs, bonus_token_ids,
92 + draft_probs, draft_token_ids)
93 + self._raise_if_incorrect_dtype(target_probs, bonus_token_ids,
94 + draft_probs, draft_token_ids)
95 + self._raise_if_inconsistent_device(target_probs, bonus_token_ids,
96 + draft_probs, draft_token_ids)
97 + self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
98 + bonus_token_ids,
99 + draft_token_ids)
100 +
101 + accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
102 + target_probs,
103 + draft_probs,
104 + draft_token_ids,
105 + )
106 +
107 + output_token_ids = self._create_output(
108 + accepted,
109 + recovered_token_ids,
110 + draft_token_ids,
111 + bonus_token_ids,
112 + )
113 + return output_token_ids
114 +
115 + def _batch_modified_rejection_sampling(
116 + self,
117 + target_probs: torch.Tensor, # [batch_size, k, vocab_size]
118 + draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
119 + draft_token_ids: torch.Tensor, # [batch_size, k]
120 + ) -> Tuple[torch.Tensor, torch.Tensor]:
121 + """Perform modified rejection sampling on each sequence.
122 +
123 + Returns:
124 + A tuple of two tensors:
125 + 0: A bool tensor of which tokens in each sequence is accepted.
126 + shape = [batch_size, k]
127 + 1: Token ids sampled from a recovered distribution, to be used
128 + when a token is rejected.
129 + shape = [batch_size, k]
130 + """
131 +
132 + batch_size, k, vocab_size = draft_probs.shape
133 +
134 + # shape [batch_size, k]
135 + accepted = self._get_accepted(target_probs, draft_probs,
136 + draft_token_ids)
137 +
138 + recovered_probs = self._get_recovered_probs(
139 + target_probs, draft_probs).reshape(batch_size * k, vocab_size)
140 +
141 + recovered_token_ids = _multinomial(recovered_probs,
142 + num_samples=1).reshape(
143 + batch_size, k)
144 + return accepted, recovered_token_ids
145 +
146 + def _get_accepted(
147 + self,
148 + target_probs: torch.Tensor, # [batch_size, k, vocab_size]
149 + draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
150 + draft_token_ids: torch.Tensor, # [batch_size, k]
151 + ) -> torch.Tensor:
152 + r"""Create bool matrix over the proposed draft tokens. If
153 + True, then a token can be accepted, else it should be
154 + rejected.
155 +
156 + Given :math:`q(\hat{x}_{n+1}|x_1, \dots, x_n)`, the probability of
157 + :math:`\hat{x}_{n+1}` given context :math:`x_1, \dots, x_n` according
158 + to the target model, and :math:`p(\hat{x}_{n+1}|x_1, \dots, x_n)`, the
159 + same conditional probability according to the draft model, the token
160 + is accepted with probability:
161 +
162 + .. math::
163 + \min\left(1, \frac{q(\hat{x}_{n+1}|x_1, \dots, x_n)}
164 + {p(\hat{x}_{n+1}|x_1, \dots, x_n)}\right)
165 +
166 + This implementation does not apply causality. When using the output,
167 + if a token is rejected, subsequent tokens should not be used.
168 +
169 + Returns a bool tensor of shape [batch_size, k] specifying which tokens
170 + are accepted.
171 + """
172 + batch_size, k, _ = draft_probs.shape
173 + batch_indices = torch.arange(batch_size,
174 + device=target_probs.device)[:, None]
175 + probs_indicies = torch.arange(k, device=target_probs.device)
176 +
177 + # shape [batch_size, k]
178 + selected_draft_probs = draft_probs[batch_indices, probs_indicies,
179 + draft_token_ids]
180 +
181 + # shape [batch_size, k]
182 + selected_target_probs = target_probs[batch_indices, probs_indicies,
183 + draft_token_ids]
184 +
185 + uniform_rand = torch.rand(batch_size,
186 + k,
187 + dtype=self.probs_dtype,
188 + device=target_probs.device)
189 + capped_ratio = torch.minimum(
190 + selected_target_probs / selected_draft_probs,
191 + torch.full((1, ), 1, device=target_probs.device))
192 + accepted = uniform_rand < capped_ratio
193 +
194 + return accepted
195 +
196 + def _get_recovered_probs(
197 + self,
198 + target_probs: torch.Tensor, # [k, vocab_size]
199 + draft_probs: torch.Tensor, # [k, vocab_size]
200 + ) -> torch.Tensor:
201 + r"""Create a probability distribution for each proposed token which can
202 + be sampled if the proposed token is rejected.
203 +
204 + When this routine is applied sequentially, the true distribution of the
205 + target model is recovered (within hardware numerics).
206 +
207 + The probability distribution used in this rejection case is constructed
208 + as follows. Given :math:`q(x|x_1, \dots, x_n)`, the probability of
209 + :math:`x` given context :math:`x_1, \dots, x_n` according to the target
210 + model and :math:`p(x|x_1, \dots, x_n)`, the same conditional probability
211 + according to the draft model:
212 +
213 + .. math::
214 + x_{n+1} \sim (q(x|x_1, \dots, x_n) - p(x|x_1, \dots, x_n))_+
215 +
216 + where :math:`(f(x))_+` is defined as:
217 +
218 + .. math::
219 + (f(x))_+ = \frac{\max(0, f(x))}{\sum_x \max(0, f(x))}
220 +
221 + See https://github.com/vllm-project/vllm/pull/2336 for a visualization
222 + of the draft, target, and recovered probability distributions.
223 +
224 + Returns a tensor of shape [batch_size, k, vocab_size].
225 +
226 + Note: This batches operations on GPU and thus constructs the recovered
227 + distribution for all tokens, even if they are accepted. This causes
228 + division-by-zero errors, so we use self._smallest_positive_value to
229 + avoid that. This introduces some drift to the distribution.
230 + """
231 + _, k, _ = draft_probs.shape
232 +
233 + # shape [batch_size, k, vocab_size]
234 + difference = target_probs - draft_probs
235 +
236 + # TODO(cade): Can we use logprobs instead of probs, and avoid the
237 + # division-by-zero errors without introducing distribution drift?
238 +
239 + # shape [batch_size, k, vocab_size]
240 + f = torch.clamp(difference, min=self._smallest_positive_value)
241 +
242 + # shape [batch_size, k, vocab_size]
243 + recovered_probs = f / torch.sum(f, dim=-1).reshape(-1, k, 1)
244 +
245 + return recovered_probs
246 +
247 + @cached_property
248 + def _smallest_positive_value(self) -> float:
249 + """Return the smallest positive value representable by the probs dtype.
250 + This value is used when constructing a distribution from which to sample
251 + recovered tokens in the first rejection case.
252 +
253 + See _get_recovered_probs for more details
254 +
255 + Note that this isn't actually the smallest positive value representable
256 + by float32, but the smallest positive normal value.
257 + See https://en.wikipedia.org/wiki/Subnormal_number for more information.
258 + """
259 + return torch.finfo(self.probs_dtype).tiny
260 +
261 + def _create_output(
262 + self,
263 + accepted: torch.Tensor, # [batch_size, k]
264 + recovered_token_ids: torch.Tensor, # [batch_size, k]
265 + draft_token_ids: torch.Tensor, # [batch_size, k]
266 + bonus_token_ids: torch.Tensor, # [batch_size]
267 + ) -> torch.Tensor:
268 + """Format output. Returns a matrix of token ids. When
269 + a token is rejected via rejection sampling, all subsequent
270 + token ids are set to -1 for the sequence.
271 +
272 + shape = [batch_size, k + num_bonus_tokens]
273 + """
274 + bonus_token_ids = bonus_token_ids.squeeze()
275 + batch_size, k = recovered_token_ids.shape
276 +
277 + # Determine the index of the first False value for each row.
278 + limits = (accepted == 0).max(1).indices
279 + limits[~(accepted == 0).any(1)] = k
280 +
281 + # Create masks using the indices.
282 + indices = torch.arange(k, device=accepted.device).unsqueeze(0)
283 + accepted_mask = indices < limits.unsqueeze(1)
284 + after_false_mask = indices == limits.unsqueeze(1)
285 +
286 + # Create an extended output tensor
287 + output_with_bonus_tokens = -torch.ones(
288 + (batch_size, k + self._num_bonus_tokens),
289 + dtype=self.token_id_dtype,
290 + device=accepted.device)
291 + output = output_with_bonus_tokens[:, :k]
292 +
293 + # Fill in the first k columns of the output tensor using masks and data
294 + # tensors.
295 + output[:, :k] = torch.where(accepted_mask, draft_token_ids,
296 + -torch.ones_like(draft_token_ids))
297 +
298 + # Fill the last column.
299 + # We check output directly as accepted may have True values inconsistent
300 + # with causal acceptance.
301 + output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
302 + bonus_token_ids, -1)
303 +
304 + # Fill the recovered token ids.
305 + output.mul_(~after_false_mask).add_(
306 + recovered_token_ids.mul(after_false_mask))
307 +
308 + self.num_accepted_tokens += accepted.sum()
309 + self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
310 + self.num_draft_tokens += batch_size * k
311 +
312 + return output_with_bonus_tokens
313 +
314 + def _raise_if_incorrect_shape(
315 + self,
316 + target_probs: torch.Tensor,
317 + bonus_token_ids: torch.Tensor,
318 + draft_probs: torch.Tensor,
319 + draft_token_ids: torch.Tensor,
320 + ) -> None:
321 + (target_batch_size, num_target_probs,
322 + target_vocab_size) = target_probs.shape
323 + bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
324 + draft_batch_size, num_draft_probs, draft_vocab_size = draft_probs.shape
325 + draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
326 +
327 + assert draft_batch_size == target_batch_size
328 + assert num_draft_probs == num_target_probs
329 + assert (draft_vocab_size == target_vocab_size
330 + ), f"{draft_vocab_size=} {target_vocab_size=}"
331 +
332 + assert draft_token_ids_batch_size == draft_batch_size
333 + assert num_draft_token_ids == num_draft_probs
334 +
335 + assert bonus_batch_size == target_batch_size
336 + assert num_bonus_tokens == self._num_bonus_tokens
337 +
338 + def _raise_if_incorrect_dtype(
339 + self,
340 + target_probs: torch.Tensor,
341 + bonus_token_ids: torch.Tensor,
342 + draft_probs: torch.Tensor,
343 + draft_token_ids: torch.Tensor,
344 + ) -> None:
345 + assert all(probs.dtype == self.probs_dtype
346 + for probs in [target_probs, draft_probs])
347 + assert all(token_ids.dtype == self.token_id_dtype
348 + for token_ids in [bonus_token_ids, draft_token_ids])
349 +
350 + def _raise_if_inconsistent_device(
351 + self,
352 + target_probs: torch.Tensor,
353 + bonus_token_ids: torch.Tensor,
354 + draft_probs: torch.Tensor,
355 + draft_token_ids: torch.Tensor,
356 + ) -> None:
357 + devices = [
358 + t.device for t in
359 + [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
360 + ]
361 + assert all([devices[0] == device for device in devices])
362 +
363 + def _raise_if_out_of_bounds_vocab(
364 + self,
365 + vocab_size: int,
366 + bonus_token_ids: torch.Tensor,
367 + draft_token_ids: torch.Tensor,
368 + ) -> None:
369 + assert torch.all(bonus_token_ids < vocab_size)
370 + assert torch.all(bonus_token_ids >= 0)
371 + assert torch.all(draft_token_ids < vocab_size)
372 + assert torch.all(draft_token_ids >= 0)
373 +
374 +
375 +# torch.multinomial forces a GPU<->CPU sync.
376 +# Therefore, we use an optimized implementation instead that skips the sync.
377 +# Note that we always sample with replacement.
378 +# probs will be modified in place, but this is fine, as we pass
379 +# in a copy already.
380 +@torch.jit.script
381 +def _multinomial(
382 + probs: torch.Tensor,
383 + num_samples: int,
384 +) -> torch.Tensor:
385 + if num_samples > 1:
386 + # This is equivalent to torch.repeat_interleaved (which also
387 + # forces a GPU<->CPU sync).
388 + probs = probs[:, None, :].expand(probs.shape[0], num_samples,
389 + probs.shape[1]).contiguous().view(
390 + -1, probs.shape[1])
391 + q = torch.empty_like(probs).exponential_(1.0)
392 + return probs.div_(q).argmax(dim=1).view(-1, num_samples)