A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Speculative decoding 1/9] Optimized rejection sampler (#2336)
Cade Daniel committed
Jan 9, 2024 at 15:38 UTC
79d64c495463665cf80937a05226806785cfa583
2 files changed
+784
tests/samplers/test_rejection_sampler.py
new
+392
@@ -0,0 +1,392 @@
1
+"""Tests for rejection sampling."""
2
+import pytest
3
+from typing import List, Tuple
4
+
5
+import torch
6
+import torch.nn.functional as F
7
+
8
+from vllm.model_executor.utils import set_random_seed
9
+
10
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
+
12
+
13
+def mock_causal_accepted_tensor(
14
+ k: int, last_accepted_indices: torch.Tensor) -> torch.Tensor:
15
+ """Generate an "accepted" tensor which should yield causally-accepted tokens
16
+ up to last accepted indices.
17
+
18
+ Tokens after last_accepted_indices+1 may also be accepted, although they
19
+ will not be causally accepted.
20
+ """
21
+ batch_size = last_accepted_indices.shape[0]
22
+
23
+ accepted = (torch.arange(k).expand(batch_size, k) <=
24
+ last_accepted_indices.unsqueeze(-1).broadcast_to(
25
+ batch_size, k)).to(device="cuda")
26
+
27
+ # Sprinkle accepted values after the contiguous initial accepted values.
28
+ # This replicates the behavior of rejection sampling, which may "accept"
29
+ # a token that cannot be accepted because of causality.
30
+ sprinkle_candidates = (
31
+ torch.arange(k).expand(batch_size, k) >
32
+ last_accepted_indices.unsqueeze(-1).broadcast_to(batch_size, k) + 1)
33
+ sprinkle = torch.rand(batch_size, k, device="cuda") > 0.5
34
+ accepted[sprinkle_candidates] = sprinkle[sprinkle_candidates]
35
+ return accepted
36
+
37
+
38
+@pytest.mark.parametrize("seed", list(range(10)))
39
+@pytest.mark.parametrize(
40
+ "which_tokens_accepted",
41
+ ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
42
+@torch.inference_mode()
43
+def test_correct_output_format(which_tokens_accepted: str, seed: int):
44
+ """Verify the output has correct format given predetermined accepted matrix.
45
+ """
46
+ set_random_seed(seed)
47
+
48
+ batch_size = 10
49
+ k = 5
50
+ vocab_size = 3000
51
+
52
+ if which_tokens_accepted == "all_tokens_accepted":
53
+ accepted = mock_causal_accepted_tensor(
54
+ k, -1 + k * torch.ones((batch_size, ), dtype=torch.long))
55
+ elif which_tokens_accepted == "no_tokens_accepted":
56
+ accepted = mock_causal_accepted_tensor(
57
+ k, -torch.ones((batch_size, ), dtype=torch.long))
58
+ elif which_tokens_accepted == "some_tokens_accepted":
59
+ last_accepted_indices = torch.randint(low=-1,
60
+ high=k,
61
+ size=(batch_size, ))
62
+ accepted = mock_causal_accepted_tensor(k, last_accepted_indices)
63
+ else:
64
+ raise AssertionError()
65
+
66
+ recovered_token_ids = torch.randint(low=0,
67
+ high=vocab_size,
68
+ size=(batch_size, k),
69
+ dtype=torch.int64,
70
+ device="cuda")
71
+ draft_token_ids = torch.randint(low=0,
72
+ high=vocab_size,
73
+ size=(batch_size, k),
74
+ dtype=torch.int64,
75
+ device="cuda")
76
+ bonus_token_ids = torch.randint(low=0,
77
+ high=vocab_size,
78
+ size=(batch_size, 1),
79
+ dtype=torch.int64,
80
+ device="cuda")
81
+
82
+ rejection_sampler = RejectionSampler()
83
+ rejection_sampler.init_gpu_tensors(rank=0)
84
+ output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
85
+ accepted,
86
+ recovered_token_ids,
87
+ draft_token_ids,
88
+ bonus_token_ids,
89
+ )
90
+
91
+ if which_tokens_accepted == "all_tokens_accepted":
92
+ # Expect all tokens to be equal to draft tokens.
93
+ assert torch.equal(output_token_ids[:, :-1], draft_token_ids)
94
+
95
+ # Expect all bonus tokens to be included.
96
+ assert torch.equal(output_token_ids[:, -1:], bonus_token_ids)
97
+ elif which_tokens_accepted == "no_tokens_accepted":
98
+ # Expect first token to be equal to recovered tokens.
99
+ assert torch.equal(output_token_ids[:, 0], recovered_token_ids[:, 0])
100
+
101
+ # Expect everything else to be -1.
102
+ assert torch.equal(output_token_ids[:, 1:],
103
+ torch.ones_like(output_token_ids[:, 1:]) * -1)
104
+ elif which_tokens_accepted == "some_tokens_accepted":
105
+ recovered_plus_bonus = torch.cat(
106
+ (recovered_token_ids, bonus_token_ids), dim=-1)
107
+ # Assert first rejected token is a recovered token or bonus token.
108
+ assert torch.equal(
109
+ recovered_plus_bonus[torch.arange(0, batch_size),
110
+ last_accepted_indices + 1],
111
+ output_token_ids[torch.arange(0, batch_size),
112
+ last_accepted_indices + 1])
113
+
114
+ # Assert every subsequent token is -1.
115
+ subsequent_mask = torch.arange(0, k + 1).expand(
116
+ batch_size, k + 1) >= (last_accepted_indices + 2).unsqueeze(-1)
117
+ assert torch.all(output_token_ids[subsequent_mask] == -1)
118
+
119
+
120
+@pytest.mark.parametrize("k", list(range(1, 6)))
121
+@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
122
+@pytest.mark.parametrize("batch_size", list(range(1, 32)))
123
+@torch.inference_mode()
124
+def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int):
125
+ rejection_sampler = RejectionSampler()
126
+ rejection_sampler.init_gpu_tensors(rank=0)
127
+
128
+ draft_probs = torch.rand(batch_size,
129
+ k,
130
+ vocab_size,
131
+ dtype=torch.float32,
132
+ device="cuda")
133
+ target_probs = torch.rand(batch_size,
134
+ k,
135
+ vocab_size,
136
+ dtype=torch.float32,
137
+ device="cuda")
138
+ bonus_token_ids = torch.randint(low=0,
139
+ high=vocab_size,
140
+ size=(batch_size, 1),
141
+ dtype=torch.int64,
142
+ device="cuda")
143
+ draft_token_ids = torch.randint(low=0,
144
+ high=vocab_size,
145
+ size=(batch_size, k),
146
+ dtype=torch.int64,
147
+ device="cuda")
148
+
149
+ rejection_sampler(target_probs, bonus_token_ids, draft_probs,
150
+ draft_token_ids)
151
+
152
+
153
+@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
154
+@pytest.mark.parametrize("which_token_ids",
155
+ ["bonus_token_ids", "draft_token_ids"])
156
+@torch.inference_mode()
157
+def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
158
+ which_token_ids: str):
159
+ k = 3
160
+ batch_size = 5
161
+ vocab_size = 30_000
162
+
163
+ rejection_sampler = RejectionSampler(strict_mode=True)
164
+ rejection_sampler.init_gpu_tensors(rank=0)
165
+
166
+ draft_probs = torch.rand(batch_size,
167
+ k,
168
+ vocab_size,
169
+ dtype=torch.float32,
170
+ device="cuda")
171
+ target_probs = torch.rand(batch_size,
172
+ k,
173
+ vocab_size,
174
+ dtype=torch.float32,
175
+ device="cuda")
176
+ bonus_token_ids = torch.randint(low=0,
177
+ high=vocab_size,
178
+ size=(batch_size, 1),
179
+ dtype=torch.int64,
180
+ device="cuda")
181
+ draft_token_ids = torch.randint(low=0,
182
+ high=vocab_size,
183
+ size=(batch_size, k),
184
+ dtype=torch.int64,
185
+ device="cuda")
186
+
187
+ oob_token_ids = None
188
+ if which_token_ids == "bonus_token_ids":
189
+ oob_token_ids = bonus_token_ids
190
+ elif which_token_ids == "draft_token_ids":
191
+ oob_token_ids = draft_token_ids
192
+ else:
193
+ raise AssertionError()
194
+
195
+ if above_or_below_vocab_range == "above":
196
+ rogue_token_id = vocab_size + 1
197
+ elif above_or_below_vocab_range == "below":
198
+ rogue_token_id = -1
199
+ else:
200
+ raise AssertionError()
201
+
202
+ oob_token_ids[0][0] = rogue_token_id
203
+
204
+ with pytest.raises(AssertionError):
205
+ rejection_sampler(target_probs, bonus_token_ids, draft_probs,
206
+ draft_token_ids)
207
+
208
+
209
+@pytest.mark.parametrize("draft_and_target_probs_equal", [True, False])
210
+@pytest.mark.parametrize("seed", list(range(5)))
211
+@torch.inference_mode()
212
+def test_rejection_sampling_approximates_target_distribution(
213
+ seed: int, draft_and_target_probs_equal: bool):
214
+ """Verify rejection sampling approximates target distribution,
215
+ despite sampling from a potentially distinct draft distribution.
216
+
217
+ This is done by first creating a random target probability
218
+ distribution and a random draft probability distribution. We then
219
+ sample token ids from the rejection sampler using these draft
220
+ and target distributions. The samples are used to estimate
221
+ the output probability distribution, which we expect to approximate
222
+ the target distribution.
223
+
224
+ A basic distance metric is used to determine similarity between
225
+ distributions.
226
+
227
+ We expect that as we increase the number of samples,
228
+ the distance between the observed distribution and the target
229
+ distribution decreases. To measure this, we compare the distance
230
+ of the observed distribution against both the target distribution
231
+ and a uniform random distribution. We expect the distance between
232
+ the observed distribution and the target distribution to improve
233
+ much more than the distance improvement between the observed
234
+ distribution and the random distribution.
235
+
236
+ When draft_and_target_probs_equal=True, the draft and target
237
+ probabilities are exactly equal. Rejection sampling should
238
+ still work without any NaNs or exceptions.
239
+ """
240
+ set_random_seed(seed)
241
+
242
+ helper = _CorrectnessTestHelper(
243
+ vocab_size=10,
244
+ rejection_sampler=RejectionSampler(),
245
+ )
246
+
247
+ draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
248
+ draft_and_target_probs_equal)
249
+
250
+ sample_sizes = [10, 100, 1_000, 10_000, 100_000]
251
+ distance_wrt_reference = []
252
+ distance_wrt_target = []
253
+
254
+ for num_samples in sample_sizes:
255
+ (reference_vs_rejsample_dist,
256
+ target_vs_rejsample_dist) = helper.run_and_compare_distributions(
257
+ draft_probs,
258
+ target_probs,
259
+ reference_probs,
260
+ num_samples,
261
+ )
262
+
263
+ distance_wrt_reference.append(reference_vs_rejsample_dist)
264
+ distance_wrt_target.append(target_vs_rejsample_dist)
265
+
266
+ relative_change_in_distance_wrt_target = get_ratio_first_to_last(
267
+ distance_wrt_target)
268
+ relative_change_in_distance_wrt_reference = get_ratio_first_to_last(
269
+ distance_wrt_reference)
270
+
271
+ print(f"{num_samples=} {target_vs_rejsample_dist=:.05f} "
272
+ f"{reference_vs_rejsample_dist=:.05f}")
273
+ print(f"{num_samples=} {relative_change_in_distance_wrt_target=:.02f} "
274
+ f"{relative_change_in_distance_wrt_reference=:.02f}")
275
+
276
+ relative_change_in_distance_wrt_target = get_ratio_first_to_last(
277
+ distance_wrt_target)
278
+ relative_change_in_distance_wrt_reference = get_ratio_first_to_last(
279
+ distance_wrt_reference)
280
+
281
+ expected_improvement_multiplier = 20
282
+ assert (relative_change_in_distance_wrt_target >
283
+ relative_change_in_distance_wrt_reference *
284
+ expected_improvement_multiplier)
285
+
286
+
287
+def get_ratio_first_to_last(elements: List[float]) -> float:
288
+ return elements[0] / elements[-1]
289
+
290
+
291
+class _CorrectnessTestHelper:
292
+ """Class that packages together logic required for the unit-level
293
+ rejection sampling correctness test.
294
+ """
295
+
296
+ def __init__(self, vocab_size: int, rejection_sampler: RejectionSampler):
297
+ self.rejection_sampler = rejection_sampler
298
+ self.vocab_size = vocab_size
299
+ self.vocab_range = (0, vocab_size)
300
+
301
+ self.rejection_sampler.init_gpu_tensors(rank=0)
302
+
303
+ # Keep test simple, use k=1
304
+ self.k = 1
305
+
306
+ # Bonus tokens not used, but rejection sampler requires
307
+ # correct shape.
308
+ self.num_bonus_tokens = 1
309
+
310
+ def generate_probs_for_test(
311
+ self, draft_and_target_probs_equal: bool
312
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
313
+ draft_probs, target_probs = [
314
+ F.softmax(
315
+ torch.rand(self.vocab_size, dtype=torch.float32),
316
+ dim=-1,
317
+ ) for _ in range(2)
318
+ ]
319
+
320
+ num_reference_probs = 100
321
+ reference_probs = F.softmax(
322
+ torch.rand(num_reference_probs,
323
+ self.vocab_size,
324
+ dtype=torch.float32),
325
+ dim=-1,
326
+ )
327
+
328
+ if draft_and_target_probs_equal:
329
+ target_probs = draft_probs.clone()
330
+
331
+ return draft_probs, target_probs, reference_probs
332
+
333
+ def run_and_compare_distributions(self, draft_probs: torch.Tensor,
334
+ target_probs: torch.Tensor,
335
+ reference_probs: torch.Tensor,
336
+ num_samples: int) -> Tuple[float, float]:
337
+ # Sample using rejection sampling.
338
+ rej_sample_probs = self._estimate_rejection_sampling_pdf(
339
+ draft_probs, target_probs, num_samples)
340
+
341
+ # Average distance from reference probs.
342
+ reference_vs_rejsample_dist = torch.dist(
343
+ reference_probs,
344
+ rej_sample_probs).item() / reference_probs.shape[0]
345
+ target_vs_rejsample_dist = torch.dist(target_probs,
346
+ rej_sample_probs).item()
347
+
348
+ return reference_vs_rejsample_dist, target_vs_rejsample_dist
349
+
350
+ def _estimate_rejection_sampling_pdf(
351
+ self,
352
+ draft_probs: torch.Tensor,
353
+ target_probs: torch.Tensor,
354
+ num_samples: int,
355
+ ) -> torch.Tensor:
356
+ # Repeat draft probs num_samples times.
357
+ draft_probs = draft_probs.reshape(1, self.k, self.vocab_size).repeat(
358
+ num_samples, 1, 1)
359
+
360
+ # Repeat target probs num_samples * k times.
361
+ # Rejection sampler requires bonus token probs, but they aren't used.
362
+ target_probs = target_probs.reshape(1, 1, self.vocab_size).repeat(
363
+ num_samples, self.k, 1)
364
+
365
+ # Randomly sample draft token ids from draft probs.
366
+ draft_token_ids = torch.multinomial(draft_probs[:, 0, :],
367
+ num_samples=1,
368
+ replacement=True).reshape(
369
+ num_samples, self.k)
370
+
371
+ # Bonus tokens not used but required.
372
+ bonus_token_ids = torch.zeros((1, self.num_bonus_tokens),
373
+ dtype=torch.int64,
374
+ device="cuda").repeat(num_samples, 1)
375
+
376
+ # Get output tokens via rejection sampling.
377
+ output_token_ids = self.rejection_sampler(target_probs.to("cuda"),
378
+ bonus_token_ids.to("cuda"),
379
+ draft_probs.to("cuda"),
380
+ draft_token_ids.to("cuda"))
381
+
382
+ # Remove bonus tokens
383
+ output_token_ids = output_token_ids[:, :-1].flatten()
384
+
385
+ # Estimate probability density function
386
+ hist = torch.histogram(output_token_ids.to(dtype=torch.float,
387
+ device="cpu"),
388
+ bins=self.vocab_size,
389
+ range=self.vocab_range,
390
+ density=True)
391
+
392
+ return hist.hist
vllm/model_executor/layers/rejection_sampler.py
new
+392
@@ -0,0 +1,392 @@
1
+from typing import Tuple, Optional
2
+from functools import cached_property
3
+
4
+import torch
5
+import torch.nn as nn
6
+import torch.jit
7
+
8
+
9
+class RejectionSampler(nn.Module):
10
+ """Apply modified rejection sampling as described in "Accelerating Large
11
+ Language Model Decoding with Speculative Sampling"
12
+ https://arxiv.org/pdf/2302.01318.pdf.
13
+ """
14
+
15
+ def __init__(self, strict_mode: bool = False):
16
+ """Create a rejection sampler.
17
+
18
+ Args:
19
+ strict_mode: Whether or not to perform shape/device/dtype checks
20
+ during sampling. This catches correctness issues but adds
21
+ nontrivial latency.
22
+ """
23
+ super().__init__()
24
+ self.probs_dtype = torch.float32
25
+ self.token_id_dtype = torch.int64
26
+ self._strict_mode = strict_mode
27
+
28
+ # NOTE: A "bonus token" is accepted iff all proposal tokens are
29
+ # accepted. There is always only one possible bonus token. We store this
30
+ # value in a variable for readability.
31
+ self._num_bonus_tokens = 1
32
+
33
+ self.num_accepted_tokens: Optional[torch.Tensor] = None
34
+ self.num_emitted_tokens: Optional[torch.Tensor] = None
35
+ self.num_draft_tokens: int = 0
36
+
37
+ def init_gpu_tensors(self, rank: int) -> None:
38
+ assert self.num_accepted_tokens is None
39
+ device = f"cuda:{rank}"
40
+ self.num_accepted_tokens = torch.tensor(0,
41
+ dtype=torch.long,
42
+ device=device)
43
+ self.num_emitted_tokens = torch.tensor(0,
44
+ dtype=torch.long,
45
+ device=device)
46
+
47
+ def forward(
48
+ self,
49
+ target_probs: torch.Tensor,
50
+ bonus_token_ids: torch.Tensor,
51
+ draft_probs: torch.Tensor,
52
+ draft_token_ids: torch.Tensor,
53
+ ) -> torch.Tensor:
54
+ """Sample token ids using rejection sampling. This accepts or rejects
55
+ tokens proposed by the draft model using the probability of each token
56
+ according to the draft and target models.
57
+
58
+ In the worst case where all draft tokens are rejected, it is guaranteed
59
+ one correct token will be emitted.
60
+
61
+ In the case where all draft tokens are accepted, a bonus token will be
62
+ accepted as its cheap to have the target model score this speculative
63
+ sequence.
64
+
65
+ Args:
66
+ target_probs: The probability distribution over token ids given
67
+ context according to the target model.
68
+ shape = [batch_size, num_speculative_tokens, vocab_size]
69
+
70
+ bonus_token_ids: The "bonus" token ids that are accepted iff all
71
+ speculative tokens in a sequence are accepted.
72
+ shape = [batch_size, num_bonus_tokens]
73
+
74
+ draft_probs: The probability distribution over token ids given
75
+ context according to the draft model.
76
+ shape = [batch_size, num_speculative_tokens, vocab_size]
77
+
78
+ draft_token_ids: The token ids that were sampled from the draft
79
+ probabilities.
80
+ shape = [batch_size, num_speculative_tokens]
81
+
82
+ Returns:
83
+ output_token_ids: The token ids sampled via rejection sampling,
84
+ or -1 if unable to sample a token because the previous token
85
+ was rejected.
86
+ shape = [batch_size, num_speculative_tokens + num_bonus_tokens]
87
+ """
88
+ # Only perform shape/dtype/device checking in strict mode, as it adds
89
+ # overhead.
90
+ if self._strict_mode:
91
+ self._raise_if_incorrect_shape(target_probs, bonus_token_ids,
92
+ draft_probs, draft_token_ids)
93
+ self._raise_if_incorrect_dtype(target_probs, bonus_token_ids,
94
+ draft_probs, draft_token_ids)
95
+ self._raise_if_inconsistent_device(target_probs, bonus_token_ids,
96
+ draft_probs, draft_token_ids)
97
+ self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
98
+ bonus_token_ids,
99
+ draft_token_ids)
100
+
101
+ accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
102
+ target_probs,
103
+ draft_probs,
104
+ draft_token_ids,
105
+ )
106
+
107
+ output_token_ids = self._create_output(
108
+ accepted,
109
+ recovered_token_ids,
110
+ draft_token_ids,
111
+ bonus_token_ids,
112
+ )
113
+ return output_token_ids
114
+
115
+ def _batch_modified_rejection_sampling(
116
+ self,
117
+ target_probs: torch.Tensor, # [batch_size, k, vocab_size]
118
+ draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
119
+ draft_token_ids: torch.Tensor, # [batch_size, k]
120
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
121
+ """Perform modified rejection sampling on each sequence.
122
+
123
+ Returns:
124
+ A tuple of two tensors:
125
+ 0: A bool tensor of which tokens in each sequence is accepted.
126
+ shape = [batch_size, k]
127
+ 1: Token ids sampled from a recovered distribution, to be used
128
+ when a token is rejected.
129
+ shape = [batch_size, k]
130
+ """
131
+
132
+ batch_size, k, vocab_size = draft_probs.shape
133
+
134
+ # shape [batch_size, k]
135
+ accepted = self._get_accepted(target_probs, draft_probs,
136
+ draft_token_ids)
137
+
138
+ recovered_probs = self._get_recovered_probs(
139
+ target_probs, draft_probs).reshape(batch_size * k, vocab_size)
140
+
141
+ recovered_token_ids = _multinomial(recovered_probs,
142
+ num_samples=1).reshape(
143
+ batch_size, k)
144
+ return accepted, recovered_token_ids
145
+
146
+ def _get_accepted(
147
+ self,
148
+ target_probs: torch.Tensor, # [batch_size, k, vocab_size]
149
+ draft_probs: torch.Tensor, # [batch_size, k, vocab_size]
150
+ draft_token_ids: torch.Tensor, # [batch_size, k]
151
+ ) -> torch.Tensor:
152
+ r"""Create bool matrix over the proposed draft tokens. If
153
+ True, then a token can be accepted, else it should be
154
+ rejected.
155
+
156
+ Given :math:`q(\hat{x}_{n+1}|x_1, \dots, x_n)`, the probability of
157
+ :math:`\hat{x}_{n+1}` given context :math:`x_1, \dots, x_n` according
158
+ to the target model, and :math:`p(\hat{x}_{n+1}|x_1, \dots, x_n)`, the
159
+ same conditional probability according to the draft model, the token
160
+ is accepted with probability:
161
+
162
+ .. math::
163
+ \min\left(1, \frac{q(\hat{x}_{n+1}|x_1, \dots, x_n)}
164
+ {p(\hat{x}_{n+1}|x_1, \dots, x_n)}\right)
165
+
166
+ This implementation does not apply causality. When using the output,
167
+ if a token is rejected, subsequent tokens should not be used.
168
+
169
+ Returns a bool tensor of shape [batch_size, k] specifying which tokens
170
+ are accepted.
171
+ """
172
+ batch_size, k, _ = draft_probs.shape
173
+ batch_indices = torch.arange(batch_size,
174
+ device=target_probs.device)[:, None]
175
+ probs_indicies = torch.arange(k, device=target_probs.device)
176
+
177
+ # shape [batch_size, k]
178
+ selected_draft_probs = draft_probs[batch_indices, probs_indicies,
179
+ draft_token_ids]
180
+
181
+ # shape [batch_size, k]
182
+ selected_target_probs = target_probs[batch_indices, probs_indicies,
183
+ draft_token_ids]
184
+
185
+ uniform_rand = torch.rand(batch_size,
186
+ k,
187
+ dtype=self.probs_dtype,
188
+ device=target_probs.device)
189
+ capped_ratio = torch.minimum(
190
+ selected_target_probs / selected_draft_probs,
191
+ torch.full((1, ), 1, device=target_probs.device))
192
+ accepted = uniform_rand < capped_ratio
193
+
194
+ return accepted
195
+
196
+ def _get_recovered_probs(
197
+ self,
198
+ target_probs: torch.Tensor, # [k, vocab_size]
199
+ draft_probs: torch.Tensor, # [k, vocab_size]
200
+ ) -> torch.Tensor:
201
+ r"""Create a probability distribution for each proposed token which can
202
+ be sampled if the proposed token is rejected.
203
+
204
+ When this routine is applied sequentially, the true distribution of the
205
+ target model is recovered (within hardware numerics).
206
+
207
+ The probability distribution used in this rejection case is constructed
208
+ as follows. Given :math:`q(x|x_1, \dots, x_n)`, the probability of
209
+ :math:`x` given context :math:`x_1, \dots, x_n` according to the target
210
+ model and :math:`p(x|x_1, \dots, x_n)`, the same conditional probability
211
+ according to the draft model:
212
+
213
+ .. math::
214
+ x_{n+1} \sim (q(x|x_1, \dots, x_n) - p(x|x_1, \dots, x_n))_+
215
+
216
+ where :math:`(f(x))_+` is defined as:
217
+
218
+ .. math::
219
+ (f(x))_+ = \frac{\max(0, f(x))}{\sum_x \max(0, f(x))}
220
+
221
+ See https://github.com/vllm-project/vllm/pull/2336 for a visualization
222
+ of the draft, target, and recovered probability distributions.
223
+
224
+ Returns a tensor of shape [batch_size, k, vocab_size].
225
+
226
+ Note: This batches operations on GPU and thus constructs the recovered
227
+ distribution for all tokens, even if they are accepted. This causes
228
+ division-by-zero errors, so we use self._smallest_positive_value to
229
+ avoid that. This introduces some drift to the distribution.
230
+ """
231
+ _, k, _ = draft_probs.shape
232
+
233
+ # shape [batch_size, k, vocab_size]
234
+ difference = target_probs - draft_probs
235
+
236
+ # TODO(cade): Can we use logprobs instead of probs, and avoid the
237
+ # division-by-zero errors without introducing distribution drift?
238
+
239
+ # shape [batch_size, k, vocab_size]
240
+ f = torch.clamp(difference, min=self._smallest_positive_value)
241
+
242
+ # shape [batch_size, k, vocab_size]
243
+ recovered_probs = f / torch.sum(f, dim=-1).reshape(-1, k, 1)
244
+
245
+ return recovered_probs
246
+
247
+ @cached_property
248
+ def _smallest_positive_value(self) -> float:
249
+ """Return the smallest positive value representable by the probs dtype.
250
+ This value is used when constructing a distribution from which to sample
251
+ recovered tokens in the first rejection case.
252
+
253
+ See _get_recovered_probs for more details
254
+
255
+ Note that this isn't actually the smallest positive value representable
256
+ by float32, but the smallest positive normal value.
257
+ See https://en.wikipedia.org/wiki/Subnormal_number for more information.
258
+ """
259
+ return torch.finfo(self.probs_dtype).tiny
260
+
261
+ def _create_output(
262
+ self,
263
+ accepted: torch.Tensor, # [batch_size, k]
264
+ recovered_token_ids: torch.Tensor, # [batch_size, k]
265
+ draft_token_ids: torch.Tensor, # [batch_size, k]
266
+ bonus_token_ids: torch.Tensor, # [batch_size]
267
+ ) -> torch.Tensor:
268
+ """Format output. Returns a matrix of token ids. When
269
+ a token is rejected via rejection sampling, all subsequent
270
+ token ids are set to -1 for the sequence.
271
+
272
+ shape = [batch_size, k + num_bonus_tokens]
273
+ """
274
+ bonus_token_ids = bonus_token_ids.squeeze()
275
+ batch_size, k = recovered_token_ids.shape
276
+
277
+ # Determine the index of the first False value for each row.
278
+ limits = (accepted == 0).max(1).indices
279
+ limits[~(accepted == 0).any(1)] = k
280
+
281
+ # Create masks using the indices.
282
+ indices = torch.arange(k, device=accepted.device).unsqueeze(0)
283
+ accepted_mask = indices < limits.unsqueeze(1)
284
+ after_false_mask = indices == limits.unsqueeze(1)
285
+
286
+ # Create an extended output tensor
287
+ output_with_bonus_tokens = -torch.ones(
288
+ (batch_size, k + self._num_bonus_tokens),
289
+ dtype=self.token_id_dtype,
290
+ device=accepted.device)
291
+ output = output_with_bonus_tokens[:, :k]
292
+
293
+ # Fill in the first k columns of the output tensor using masks and data
294
+ # tensors.
295
+ output[:, :k] = torch.where(accepted_mask, draft_token_ids,
296
+ -torch.ones_like(draft_token_ids))
297
+
298
+ # Fill the last column.
299
+ # We check output directly as accepted may have True values inconsistent
300
+ # with causal acceptance.
301
+ output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
302
+ bonus_token_ids, -1)
303
+
304
+ # Fill the recovered token ids.
305
+ output.mul_(~after_false_mask).add_(
306
+ recovered_token_ids.mul(after_false_mask))
307
+
308
+ self.num_accepted_tokens += accepted.sum()
309
+ self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
310
+ self.num_draft_tokens += batch_size * k
311
+
312
+ return output_with_bonus_tokens
313
+
314
+ def _raise_if_incorrect_shape(
315
+ self,
316
+ target_probs: torch.Tensor,
317
+ bonus_token_ids: torch.Tensor,
318
+ draft_probs: torch.Tensor,
319
+ draft_token_ids: torch.Tensor,
320
+ ) -> None:
321
+ (target_batch_size, num_target_probs,
322
+ target_vocab_size) = target_probs.shape
323
+ bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
324
+ draft_batch_size, num_draft_probs, draft_vocab_size = draft_probs.shape
325
+ draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
326
+
327
+ assert draft_batch_size == target_batch_size
328
+ assert num_draft_probs == num_target_probs
329
+ assert (draft_vocab_size == target_vocab_size
330
+ ), f"{draft_vocab_size=} {target_vocab_size=}"
331
+
332
+ assert draft_token_ids_batch_size == draft_batch_size
333
+ assert num_draft_token_ids == num_draft_probs
334
+
335
+ assert bonus_batch_size == target_batch_size
336
+ assert num_bonus_tokens == self._num_bonus_tokens
337
+
338
+ def _raise_if_incorrect_dtype(
339
+ self,
340
+ target_probs: torch.Tensor,
341
+ bonus_token_ids: torch.Tensor,
342
+ draft_probs: torch.Tensor,
343
+ draft_token_ids: torch.Tensor,
344
+ ) -> None:
345
+ assert all(probs.dtype == self.probs_dtype
346
+ for probs in [target_probs, draft_probs])
347
+ assert all(token_ids.dtype == self.token_id_dtype
348
+ for token_ids in [bonus_token_ids, draft_token_ids])
349
+
350
+ def _raise_if_inconsistent_device(
351
+ self,
352
+ target_probs: torch.Tensor,
353
+ bonus_token_ids: torch.Tensor,
354
+ draft_probs: torch.Tensor,
355
+ draft_token_ids: torch.Tensor,
356
+ ) -> None:
357
+ devices = [
358
+ t.device for t in
359
+ [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
360
+ ]
361
+ assert all([devices[0] == device for device in devices])
362
+
363
+ def _raise_if_out_of_bounds_vocab(
364
+ self,
365
+ vocab_size: int,
366
+ bonus_token_ids: torch.Tensor,
367
+ draft_token_ids: torch.Tensor,
368
+ ) -> None:
369
+ assert torch.all(bonus_token_ids < vocab_size)
370
+ assert torch.all(bonus_token_ids >= 0)
371
+ assert torch.all(draft_token_ids < vocab_size)
372
+ assert torch.all(draft_token_ids >= 0)
373
+
374
+
375
+# torch.multinomial forces a GPU<->CPU sync.
376
+# Therefore, we use an optimized implementation instead that skips the sync.
377
+# Note that we always sample with replacement.
378
+# probs will be modified in place, but this is fine, as we pass
379
+# in a copy already.
380
+@torch.jit.script
381
+def _multinomial(
382
+ probs: torch.Tensor,
383
+ num_samples: int,
384
+) -> torch.Tensor:
385
+ if num_samples > 1:
386
+ # This is equivalent to torch.repeat_interleaved (which also
387
+ # forces a GPU<->CPU sync).
388
+ probs = probs[:, None, :].expand(probs.shape[0], num_samples,
389
+ probs.shape[1]).contiguous().view(
390
+ -1, probs.shape[1])
391
+ q = torch.empty_like(probs).exponential_(1.0)
392
+ return probs.div_(q).argmax(dim=1).view(-1, num_samples)