A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Speculative Decoding 1/2 ] Add typical acceptance sampling as one of the sampling techniques in the verifier (#5131)
sroy745 committed
Jun 17, 2024 at 19:29 UTC
fa9e3852290ecb6eaae45befbd629bb060f57fb7
4 files changed
+866
-164
tests/samplers/test_typical_acceptance_sampler.py
new
+464
@@ -0,0 +1,464 @@
1
+"""Tests for rejection sampling."""
2
+
3
+import pytest
4
+import torch
5
+
6
+from vllm.model_executor.layers.typical_acceptance_sampler import (
7
+ TypicalAcceptanceSampler)
8
+from vllm.model_executor.utils import set_random_seed
9
+
10
+CUDA_DEVICES = [f"cuda:{i}" for i in range(1)]
11
+
12
+
13
+def get_zero_temperature_prob_dist(batch_size, k, vocab_size):
14
+ """
15
+ Generates a fake temperature zero probability distribution.
16
+ Returns:
17
+ 1. A fake temperature zero probability distribution of shape
18
+ [batch_size, k, vocab_size]
19
+ 2. Tensor of shape [batch_size, k] containing the token ids
20
+ of the probability 1.0 tokens at each position.
21
+ """
22
+ # Simulate temperature 0 probability distribution for target probabilities
23
+ # and create target probabilities such that only 1 token id has
24
+ # probability 1.0
25
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
26
+ probs = torch.rand(batch_size, k, vocab_size)
27
+ _, zero_temperature_token_ids = torch.max(probs, dim=-1)
28
+ # set the probability of the tokens with ids in zero_temperature_token_ids
29
+ # to 1 and the rest to 0.
30
+ target_probs = torch.zeros_like(probs).scatter_(
31
+ -1, zero_temperature_token_ids.unsqueeze(-1), 1.0)
32
+ return target_probs, zero_temperature_token_ids
33
+
34
+
35
+def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
36
+ token_ids_to_exclude: torch.Tensor):
37
+ """
38
+ Returns a tensor of shape [batch_size, k] of fake draft token ids
39
+ drawn randomly from a vocab of size vocab_size. We however ensure
40
+ that token_ids from token_ids_to_exclude are excluded at the
41
+ corresponding positions.
42
+ """
43
+ draft_token_ids = torch.empty(batch_size, k, dtype=torch.long)
44
+ for i in range(batch_size):
45
+ for j in range(k):
46
+ # Generate a random token ID excluding token_ids_to_exclude[i, j]
47
+ while True:
48
+ token_id = torch.randint(0, vocab_size, (1, )).item()
49
+ if token_id != token_ids_to_exclude[i, j]:
50
+ draft_token_ids[i, j] = token_id
51
+ break
52
+ return draft_token_ids
53
+
54
+
55
+@pytest.mark.parametrize("k", list(range(1, 6)))
56
+@pytest.mark.parametrize("vocab_size", [30_000, 50_000])
57
+@pytest.mark.parametrize("batch_size", list(range(1, 32)))
58
+@pytest.mark.parametrize("device", CUDA_DEVICES)
59
+@torch.inference_mode()
60
+def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
61
+ device: str):
62
+ """
63
+ Tests that the TypicalAcceptancSampler forward succeeds for
64
+ different combinations of k, vocab_size, batch_size and num devices.
65
+ """
66
+ torch.set_default_device(device)
67
+ typical_acceptance_sampler = TypicalAcceptanceSampler()
68
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
69
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
70
+ bonus_token_ids = torch.randint(low=0,
71
+ high=vocab_size,
72
+ size=(batch_size, 1),
73
+ dtype=torch.int64)
74
+ draft_token_ids = torch.randint(low=0,
75
+ high=vocab_size,
76
+ size=(batch_size, k),
77
+ dtype=torch.int64)
78
+ # Verify that sampling succeeds for all cases.
79
+ typical_acceptance_sampler(target_probs, bonus_token_ids, draft_token_ids)
80
+
81
+
82
+@pytest.mark.parametrize("above_or_below_vocab_range", ["above", "below"])
83
+@pytest.mark.parametrize("which_token_ids",
84
+ ["bonus_token_ids", "draft_token_ids"])
85
+@pytest.mark.parametrize("device", CUDA_DEVICES)
86
+@torch.inference_mode()
87
+def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
88
+ which_token_ids: str, device: str):
89
+ """
90
+ Tests that we throw an exception of the token ids fall outside
91
+ the bound of the provided vocabulary.
92
+ """
93
+ k = 3
94
+ batch_size = 5
95
+ vocab_size = 30_000
96
+ torch.set_default_device(device)
97
+ typical_acceptance_sampler = TypicalAcceptanceSampler(strict_mode=True)
98
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
99
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
100
+ bonus_token_ids = torch.randint(low=0,
101
+ high=vocab_size,
102
+ size=(batch_size, 1),
103
+ dtype=torch.int64)
104
+ draft_token_ids = torch.randint(low=0,
105
+ high=vocab_size,
106
+ size=(batch_size, k),
107
+ dtype=torch.int64)
108
+ # Verify that appropriate exceptions are thrown for out
109
+ # of bound vocabs.
110
+ oob_token_ids = None
111
+ if which_token_ids == "bonus_token_ids":
112
+ oob_token_ids = bonus_token_ids
113
+ elif which_token_ids == "draft_token_ids":
114
+ oob_token_ids = draft_token_ids
115
+ else:
116
+ raise AssertionError()
117
+
118
+ if above_or_below_vocab_range == "above":
119
+ rogue_token_id = vocab_size + 1
120
+ elif above_or_below_vocab_range == "below":
121
+ rogue_token_id = -1
122
+ else:
123
+ raise AssertionError()
124
+
125
+ oob_token_ids[0][0] = rogue_token_id
126
+
127
+ with pytest.raises(AssertionError):
128
+ typical_acceptance_sampler(target_probs, bonus_token_ids,
129
+ draft_token_ids)
130
+
131
+
132
+@pytest.mark.parametrize("seed", list(range(10)))
133
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
134
+@pytest.mark.parametrize("device", CUDA_DEVICES)
135
+@torch.inference_mode()
136
+def test_uniform_target_distribution_accepts_all_tokens(
137
+ seed: int, disable_bonus_tokens: bool, device: str):
138
+ """
139
+ Test the TypicalAcceptanceSampler with a uniform target probability
140
+ distribution.
141
+
142
+ This test verifies that when provided with a uniform target probability
143
+ distribution, the TypicalAcceptanceSampler accepts all draft tokens. The
144
+ entropy of the uniform target distribution being high should lead to all
145
+ draft tokens being accepted. The test also ensures that the behavior
146
+ regarding bonus tokens is consistent with the `disable_bonus_tokens`
147
+ flag.
148
+ """
149
+ set_random_seed(seed)
150
+ k = 3
151
+ batch_size = 5
152
+ vocab_size = 30_000
153
+ torch.set_default_device(device)
154
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
155
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
156
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
157
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
158
+ draft_token_ids = torch.randint(low=0,
159
+ high=vocab_size,
160
+ size=(batch_size, k),
161
+ dtype=torch.int64)
162
+ bonus_token_ids = torch.randint(low=0,
163
+ high=vocab_size,
164
+ size=(batch_size, 1),
165
+ dtype=torch.int64)
166
+ output_token_ids = typical_acceptance_sampler(target_probs,
167
+ bonus_token_ids,
168
+ draft_token_ids)
169
+ # We are using a uniform target probability distribution.
170
+ # For a uniform distribution the entropy is very high and it
171
+ # should lead to all draft tokens being accepted. Verify that.
172
+ assert output_token_ids.shape[0] == batch_size
173
+ assert output_token_ids.shape[1] == (k + 1)
174
+ if disable_bonus_tokens:
175
+ assert torch.all(output_token_ids[:, -1] == -1)
176
+ else:
177
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
178
+
179
+ assert torch.all(output_token_ids[:, :k] == draft_token_ids)
180
+
181
+
182
+@pytest.mark.parametrize("seed", list(range(10)))
183
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
184
+@pytest.mark.parametrize("device", CUDA_DEVICES)
185
+@torch.inference_mode()
186
+def test_temperature_zero_target_distribution(seed: int,
187
+ disable_bonus_tokens: bool,
188
+ device: str):
189
+ """
190
+ Test the TypicalAcceptanceSampler with a zero-temperature target
191
+ probability distribution.
192
+
193
+ This test verifies that when using a zero-temperature target probability
194
+ distribution, where only one token has a probability of 1.0, the
195
+ TypicalAcceptanceSampler correctly rejects all draft tokens that do not
196
+ match this probability. Additionally, it ensures that when all draft
197
+ tokens are rejected, the sampler falls back to greedy sampling to select a
198
+ single token from the target distribution.
199
+ """
200
+ set_random_seed(seed)
201
+ k = 3
202
+ batch_size = 5
203
+ vocab_size = 30_000
204
+ torch.set_default_device(device)
205
+
206
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
207
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
208
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
209
+ # Simulate temperature 0 probability distribution for target probabilities
210
+ # and create target probabilities such that only 1 token id has
211
+ # probability 1.0
212
+ target_probs, zero_temperature_token_ids = get_zero_temperature_prob_dist(
213
+ batch_size, k, vocab_size)
214
+ # Populate draft_token_ids such that they exclude the token_ids
215
+ # with probability = 1.0
216
+ draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
217
+ zero_temperature_token_ids)
218
+ bonus_token_ids = torch.randint(low=0,
219
+ high=vocab_size,
220
+ size=(batch_size, 1),
221
+ dtype=torch.int64)
222
+ # The target probaility distribution is a temperature zero distribution
223
+ # with zero entroy. Since our draft token ids don't match the probability
224
+ # 1.0 tokens in the target distribution we will reject all of them and
225
+ # fallback to the greedy sampling for selecting 1 token for each sequence.
226
+ # Verify the same.
227
+ output_token_ids = typical_acceptance_sampler(target_probs,
228
+ bonus_token_ids,
229
+ draft_token_ids)
230
+ assert output_token_ids.shape[0] == batch_size
231
+ assert output_token_ids.shape[1] == (k + 1)
232
+ assert torch.all(output_token_ids[:, -1] == -1)
233
+ assert torch.all(output_token_ids[:, 0] == zero_temperature_token_ids[:,
234
+ 0])
235
+
236
+
237
+@pytest.mark.parametrize("seed", list(range(10)))
238
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
239
+@pytest.mark.parametrize("device", CUDA_DEVICES)
240
+@torch.inference_mode()
241
+def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
242
+ device: str):
243
+ """
244
+ Test the TypicalAcceptanceSampler with a mixed target probability
245
+ distribution.
246
+
247
+ This test ensures that the TypicalAcceptanceSampler handles a mixed
248
+ target probability distribution correctly. Specifically, it uses a
249
+ zero-temperature distribution for some sequences and a uniform
250
+ distribution for others. The test verifies that:
251
+
252
+ - For sequences with a zero-temperature distribution, only the token
253
+ with a probability of 1.0 is accepted, and all other tokens are rejected.
254
+ - For sequences with a uniform distribution, all draft tokens are
255
+ accepted.
256
+ - When `disable_bonus_tokens` is False, the bonus tokens are also accepted
257
+ for sequences with a uniform distribution.
258
+ """
259
+ set_random_seed(seed)
260
+ k = 3
261
+ batch_size = 4
262
+ vocab_size = 30_000
263
+ torch.set_default_device(device)
264
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
265
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
266
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
267
+ # For sequences 0 and 2 set the distribution to a temperature
268
+ # zero distribution. For sequences 1 and 3 set it to a uniform
269
+ # distribution.
270
+ target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
271
+ batch_size, k, vocab_size))
272
+ draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
273
+ zero_temperature_token_ids)
274
+ uniform_probs = torch.rand(2, k, vocab_size, dtype=torch.float32)
275
+ target_probs[[1, 3]] = uniform_probs
276
+ bonus_token_ids = torch.randint(low=0,
277
+ high=vocab_size,
278
+ size=(batch_size, 1),
279
+ dtype=torch.int64)
280
+ output_token_ids = typical_acceptance_sampler(target_probs,
281
+ bonus_token_ids,
282
+ draft_token_ids)
283
+ # verify the shape of output_token_ids
284
+ assert output_token_ids.shape[0] == batch_size
285
+ assert output_token_ids.shape[1] == (k + 1)
286
+ # For sequences 0 and 2 verify that only 1 token is accepted
287
+ # which is the token with probability 1.0 in the target distribution
288
+ # at position 0.
289
+ assert torch.all(output_token_ids[[0, 2], 1:] == -1)
290
+ assert (torch.all(output_token_ids[[0, 2],
291
+ 0] == zero_temperature_token_ids[[0, 2],
292
+ 0]))
293
+ # For sequences 1 and 3 verify that all tokens are accepted since the
294
+ # target probability distribution is uniform. In addition verify that
295
+ # if disable_bonus_tokens is false then we also accept the bonus tokens.
296
+ assert torch.all(
297
+ output_token_ids[[1, 3], :-1] == draft_token_ids[[1, 3], :])
298
+ if disable_bonus_tokens:
299
+ assert torch.all(output_token_ids[[1, 3], -1] == -1)
300
+ else:
301
+ assert torch.all(output_token_ids[[1, 3], -1] != -1)
302
+
303
+
304
+@pytest.mark.parametrize("seed", list(range(10)))
305
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
306
+@pytest.mark.parametrize("device", CUDA_DEVICES)
307
+@torch.inference_mode()
308
+def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
309
+ device: str):
310
+ """
311
+ Test the TypicalAcceptanceSampler's behavior when only a subset of draft
312
+ tokens should be accepted.
313
+
314
+ This test verifies that the TypicalAcceptanceSampler correctly accepts or
315
+ rejects draft tokens based on a zero-temperature target probability
316
+ distribution. Specifically, it ensures that:
317
+
318
+ - When all draft tokens match tokens with a probability of 1.0 in the
319
+ target distribution, all draft tokens are accepted.
320
+ - When only some draft tokens match tokens with a probability of 1.0 in
321
+ the target distribution, only those matching tokens are accepted, and the
322
+ rest are rejected.
323
+ """
324
+ set_random_seed(seed)
325
+ k = 5
326
+ batch_size = 1
327
+ vocab_size = 30_000
328
+ torch.set_default_device(device)
329
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
330
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
331
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
332
+ # Create a temperature zero target probability distribution and ensure
333
+ # all draft token ids correspond to the tokens with 1.0 probability.
334
+ # Verify that all of them are accepted.
335
+ target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
336
+ batch_size, k, vocab_size))
337
+ draft_token_ids = zero_temperature_token_ids
338
+ bonus_token_ids = torch.randint(low=0,
339
+ high=vocab_size,
340
+ size=(batch_size, 1),
341
+ dtype=torch.int64)
342
+ output_token_ids = typical_acceptance_sampler(target_probs,
343
+ bonus_token_ids,
344
+ draft_token_ids)
345
+ assert output_token_ids.shape[0] == batch_size
346
+ assert output_token_ids.shape[1] == (k + 1)
347
+ assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
348
+ if disable_bonus_tokens:
349
+ assert torch.all(output_token_ids[:, -1] == -1)
350
+ else:
351
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
352
+ # Next only keep the first 2 draft tokens same as the zero temperature
353
+ # tokens. For the remaining 3 choose some other tokens. In the
354
+ # response we will expect the first 2 tokens to be the same as the
355
+ # draft tokens and the rest as -1
356
+ draft_token_ids_to_replace = get_draft_token_ids(
357
+ batch_size, k, vocab_size, zero_temperature_token_ids)
358
+ draft_token_ids = torch.cat(
359
+ (draft_token_ids[:, :2], draft_token_ids_to_replace[:, -3:]), dim=1)
360
+ output_token_ids = typical_acceptance_sampler(target_probs,
361
+ bonus_token_ids,
362
+ draft_token_ids)
363
+ assert output_token_ids.shape[0] == batch_size
364
+ assert output_token_ids.shape[1] == (k + 1)
365
+ assert torch.all(output_token_ids[:, :2] == draft_token_ids[:, :2])
366
+ assert torch.all(output_token_ids[:, -3:] == -1)
367
+
368
+
369
+@pytest.mark.parametrize("seed", list(range(1)))
370
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
371
+@pytest.mark.parametrize("device", CUDA_DEVICES)
372
+@torch.inference_mode()
373
+def test_accept_tokens_set_non_default_posteriors(seed: int,
374
+ disable_bonus_tokens: bool,
375
+ device: str):
376
+ """
377
+ Test the TypicalAcceptanceSampler with custom posterior thresholds and
378
+ alpha values. This test verifies that by modifying the posterior
379
+ thresholds and alpha values we can change the acceptance behavior of the
380
+ sampler.
381
+ """
382
+ set_random_seed(seed)
383
+ k = 5
384
+ batch_size = 1
385
+ vocab_size = 30_000
386
+ torch.set_default_device(device)
387
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
388
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
389
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
390
+ # Simulate temperature 0 probability distribution for target
391
+ # probabilities and create target probabilities such that only 1 token
392
+ # id has probability 1.0 and others have a very low probability of
393
+ # 0.00001. Populate draft_token_ids such that they exclude the token_ids
394
+ # with probability = 1.0. Without any changes to the posterior thresholds
395
+ # none of the draft tokens are accepted.
396
+ target_probs, zero_temperature_token_ids = (get_zero_temperature_prob_dist(
397
+ batch_size, k, vocab_size))
398
+ target_probs[target_probs == 0] = 0.00001
399
+ draft_token_ids = get_draft_token_ids(batch_size, k, vocab_size,
400
+ zero_temperature_token_ids)
401
+ bonus_token_ids = torch.randint(low=0,
402
+ high=vocab_size,
403
+ size=(batch_size, 1),
404
+ dtype=torch.int64)
405
+ output_token_ids = typical_acceptance_sampler(target_probs,
406
+ bonus_token_ids,
407
+ draft_token_ids)
408
+ assert output_token_ids.shape[0] == batch_size
409
+ assert output_token_ids.shape[1] == (k + 1)
410
+ assert torch.all(output_token_ids[:, 1:-1] == -1)
411
+
412
+ # Change the posterior threshold values to 0.0 so that we will
413
+ # now accept even draft tokens with very low probability in the
414
+ # target distribution. Simulate and verify the same.
415
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
416
+ strict_mode=True,
417
+ disable_bonus_tokens=disable_bonus_tokens,
418
+ posterior_threshold=0.0,
419
+ posterior_alpha=0.0)
420
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
421
+ output_token_ids = typical_acceptance_sampler(target_probs,
422
+ bonus_token_ids,
423
+ draft_token_ids)
424
+ assert output_token_ids.shape[0] == batch_size
425
+ assert output_token_ids.shape[1] == (k + 1)
426
+ assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
427
+ if disable_bonus_tokens:
428
+ assert torch.all(output_token_ids[:, -1] == -1)
429
+ else:
430
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
431
+
432
+
433
+@pytest.mark.parametrize("seed", list(range(10)))
434
+@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
435
+@pytest.mark.parametrize("device", CUDA_DEVICES)
436
+@torch.inference_mode()
437
+def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
438
+ device: str):
439
+ """
440
+ Test the TypicalAcceptanceSampler's method for generating
441
+ replacement token IDs.
442
+
443
+ This test verifies that the `_replacement_token_ids` method of the
444
+ TypicalAcceptanceSampler correctly identifies the token IDs to be used
445
+ as replacements based on the target probability distribution.
446
+ Specifically, it ensures that the method correctly identifies the
447
+ tokens with the highest probability for each sequence in the batch.
448
+ """
449
+ set_random_seed(seed)
450
+ k = 10
451
+ batch_size = 5
452
+ vocab_size = 30_000
453
+ torch.set_default_device(device)
454
+ typical_acceptance_sampler = TypicalAcceptanceSampler(
455
+ strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
456
+ typical_acceptance_sampler.init_gpu_tensors(rank=0)
457
+ target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
458
+ expected_replacement_tokens = -torch.ones(
459
+ (batch_size, k), dtype=torch.long)
460
+ expected_replacement_tokens[:, 0] = torch.argmax(target_probs[:, 0, :],
461
+ dim=1)
462
+ actual_replacement_tokens = (
463
+ typical_acceptance_sampler._replacement_token_ids(target_probs))
464
+ assert torch.all(expected_replacement_tokens == actual_replacement_tokens)
vllm/model_executor/layers/rejection_sampler.py
+10
-164
@@ -1,12 +1,15 @@
1
from functools import cached_property
2
-from typing import Optional, Tuple
2
+from typing import Tuple
3
4
import torch
5
import torch.jit
6
import torch.nn as nn
7
8
+from vllm.model_executor.layers.spec_decode_base_sampler import (
9
+ SpecDecodeBaseSampler)
10
9
-class RejectionSampler(nn.Module):
11
+
12
+class RejectionSampler(SpecDecodeBaseSampler, nn.Module):
13
"""Apply modified rejection sampling as described in "Accelerating Large
14
Language Model Decoding with Speculative Sampling"
15
https://arxiv.org/pdf/2302.01318.pdf.
@@ -22,39 +25,11 @@ class RejectionSampler(nn.Module):
25
Require when bonus tokens will cause corrupt KV cache for
26
proposal methods that require KV cache.
27
strict_mode: Whether or not to perform shape/device/dtype checks
25
- during sampling. This catches correctness issues but adds
26
- nontrivial latency.
28
+ during sampling. This catches correctness issues but adds
29
+ nontrivial latency.
30
"""
28
- super().__init__()
29
- self._disable_bonus_tokens = disable_bonus_tokens
30
- self._strict_mode = strict_mode
31
-
32
- # NOTE: A "bonus token" is accepted iff all proposal tokens are
33
- # accepted. There is always only one possible bonus token. We store this
34
- # value in a variable for readability.
35
- self._num_bonus_tokens = 1
36
-
37
- self.num_accepted_tokens: Optional[torch.Tensor] = None
38
- self.num_emitted_tokens: Optional[torch.Tensor] = None
39
- self.num_draft_tokens: int = 0
40
-
41
- def init_gpu_tensors(self, rank: int) -> None:
42
- assert self.num_accepted_tokens is None
43
- device = f"cuda:{rank}"
44
- self.num_accepted_tokens = torch.tensor(0,
45
- dtype=torch.long,
46
- device=device)
47
- self.num_emitted_tokens = torch.tensor(0,
48
- dtype=torch.long,
49
- device=device)
50
-
51
- @property
52
- def probs_dtype(self):
53
- return torch.float32
54
-
55
- @property
56
- def token_id_dtype(self):
57
- return torch.int64
31
+ SpecDecodeBaseSampler.__init__(self, disable_bonus_tokens, strict_mode)
32
+ nn.Module.__init__(self)
33
34
def forward(
35
self,
@@ -100,15 +75,8 @@ class RejectionSampler(nn.Module):
75
# Only perform shape/dtype/device checking in strict mode, as it adds
76
# overhead.
77
if self._strict_mode:
103
- self._raise_if_incorrect_shape(target_probs, bonus_token_ids,
104
- draft_probs, draft_token_ids)
105
- self._raise_if_incorrect_dtype(target_probs, bonus_token_ids,
78
+ self._raise_if_incorrect_input(target_probs, bonus_token_ids,
79
draft_probs, draft_token_ids)
107
- self._raise_if_inconsistent_device(target_probs, bonus_token_ids,
108
- draft_probs, draft_token_ids)
109
- self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
110
- bonus_token_ids,
111
- draft_token_ids)
80
81
accepted, recovered_token_ids = self._batch_modified_rejection_sampling(
82
target_probs,
@@ -272,128 +240,6 @@ class RejectionSampler(nn.Module):
240
"""
241
return torch.finfo(self.probs_dtype).tiny
242
275
- def _create_output(
276
- self,
277
- accepted: torch.Tensor, # [batch_size, k]
278
- recovered_token_ids: torch.Tensor, # [batch_size, k]
279
- draft_token_ids: torch.Tensor, # [batch_size, k]
280
- bonus_token_ids: torch.Tensor, # [batch_size]
281
- ) -> torch.Tensor:
282
- """Format output. Returns a matrix of token ids. When
283
- a token is rejected via rejection sampling, all subsequent
284
- token ids are set to -1 for the sequence.
285
-
286
- shape = [batch_size, k + num_bonus_tokens]
287
- """
288
- bonus_token_ids = bonus_token_ids.squeeze()
289
- batch_size, k = recovered_token_ids.shape
290
-
291
- # Determine the index of the first False value for each row.
292
- limits = (accepted == 0).max(1).indices
293
- limits[~(accepted == 0).any(1)] = k
294
-
295
- # Create masks using the indices.
296
- indices = torch.arange(k, device=accepted.device).unsqueeze(0)
297
- accepted_mask = indices < limits.unsqueeze(1)
298
- after_false_mask = indices == limits.unsqueeze(1)
299
-
300
- # Create an extended output tensor
301
- output_with_bonus_tokens = -torch.ones(
302
- (batch_size, k + self._num_bonus_tokens),
303
- dtype=self.token_id_dtype,
304
- device=accepted.device)
305
- output = output_with_bonus_tokens[:, :k]
306
-
307
- # Fill in the first k columns of the output tensor using masks and data
308
- # tensors.
309
- torch.where(accepted_mask,
310
- draft_token_ids,
311
- -torch.ones_like(draft_token_ids),
312
- out=output)
313
-
314
- # Fill the last column.
315
- # We check output directly as accepted may have True values inconsistent
316
- # with causal acceptance.
317
- output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
318
- bonus_token_ids, -1)
319
-
320
- # We disable bonus tokens because it causes corrupt KV cache for
321
- # proposal methods that require KV cache. We can fix it by "prefilling"
322
- # the bonus token in the proposer. The following issue tracks the fix.
323
- # https://github.com/vllm-project/vllm/issues/4212
324
- if self._disable_bonus_tokens:
325
- output_with_bonus_tokens[:, -1] = -1
326
-
327
- # Fill the recovered token ids.
328
- output.mul_(~after_false_mask).add_(
329
- recovered_token_ids.mul(after_false_mask))
330
-
331
- self.num_accepted_tokens += accepted.sum()
332
- self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
333
- self.num_draft_tokens += batch_size * k
334
-
335
- return output_with_bonus_tokens
336
-
337
- def _raise_if_incorrect_shape(
338
- self,
339
- target_probs: torch.Tensor,
340
- bonus_token_ids: torch.Tensor,
341
- draft_probs: torch.Tensor,
342
- draft_token_ids: torch.Tensor,
343
- ) -> None:
344
- (target_batch_size, num_target_probs,
345
- target_vocab_size) = target_probs.shape
346
- bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
347
- draft_batch_size, num_draft_probs, draft_vocab_size = draft_probs.shape
348
- draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
349
-
350
- assert draft_batch_size == target_batch_size
351
- assert num_draft_probs == num_target_probs
352
- assert (draft_vocab_size == target_vocab_size
353
- ), f"{draft_vocab_size=} {target_vocab_size=}"
354
-
355
- assert draft_token_ids_batch_size == draft_batch_size
356
- assert num_draft_token_ids == num_draft_probs
357
-
358
- assert bonus_batch_size == target_batch_size
359
- assert num_bonus_tokens == self._num_bonus_tokens
360
-
361
- def _raise_if_incorrect_dtype(
362
- self,
363
- target_probs: torch.Tensor,
364
- bonus_token_ids: torch.Tensor,
365
- draft_probs: torch.Tensor,
366
- draft_token_ids: torch.Tensor,
367
- ) -> None:
368
- assert all(probs.dtype == self.probs_dtype
369
- for probs in [target_probs, draft_probs])
370
- assert all(token_ids.dtype == self.token_id_dtype
371
- for token_ids in [bonus_token_ids, draft_token_ids])
372
-
373
- def _raise_if_inconsistent_device(
374
- self,
375
- target_probs: torch.Tensor,
376
- bonus_token_ids: torch.Tensor,
377
- draft_probs: torch.Tensor,
378
- draft_token_ids: torch.Tensor,
379
- ) -> None:
380
- devices = [
381
- t.device for t in
382
- [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
383
- ]
384
- assert all([devices[0] == device for device in devices])
385
-
386
- def _raise_if_out_of_bounds_vocab(
387
- self,
388
- vocab_size: int,
389
- bonus_token_ids: torch.Tensor,
390
- draft_token_ids: torch.Tensor,
391
- ) -> None:
392
- assert torch.all(bonus_token_ids < vocab_size)
393
- assert torch.all(bonus_token_ids >= 0)
394
- assert torch.all(draft_token_ids < vocab_size)
395
- assert torch.all(draft_token_ids >= 0)
396
-
243
244
# torch.multinomial forces a GPU<->CPU sync.
245
# Therefore, we use an optimized implementation instead that skips the sync.
vllm/model_executor/layers/spec_decode_base_sampler.py
new
+206
@@ -0,0 +1,206 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+
6
+class SpecDecodeBaseSampler():
7
+ """Base class for samplers used for Speculative Decoding verification
8
+ step.
9
+ """
10
+
11
+ def __init__(self,
12
+ disable_bonus_tokens: bool = True,
13
+ strict_mode: bool = False):
14
+ """Base class constructor.
15
+ Args:
16
+ disable_bonus_tokens: Whether or not to disable the bonus token.
17
+ Require when bonus tokens will cause corrupt KV cache for
18
+ proposal methods that require KV cache.
19
+ strict_mode: Whether or not to perform shape/device/dtype checks
20
+ during sampling. This catches correctness issues but adds
21
+ nontrivial latency.
22
+ """
23
+ super().__init__()
24
+ self._disable_bonus_tokens = disable_bonus_tokens
25
+ self._strict_mode = strict_mode
26
+
27
+ # NOTE: A "bonus token" is accepted iff all proposal tokens are
28
+ # accepted. There is always only one possible bonus token. We store this
29
+ # value in a variable for readability.
30
+ self._num_bonus_tokens = 1
31
+
32
+ self.num_accepted_tokens: Optional[torch.Tensor] = None
33
+ self.num_emitted_tokens: Optional[torch.Tensor] = None
34
+ self.num_draft_tokens: int = 0
35
+
36
+ def init_gpu_tensors(self, rank: int) -> None:
37
+ assert self.num_accepted_tokens is None
38
+ device = f"cuda:{rank}"
39
+ self.num_accepted_tokens = torch.tensor(0,
40
+ dtype=torch.long,
41
+ device=device)
42
+ self.num_emitted_tokens = torch.tensor(0,
43
+ dtype=torch.long,
44
+ device=device)
45
+
46
+ @property
47
+ def probs_dtype(self):
48
+ return torch.float32
49
+
50
+ @property
51
+ def token_id_dtype(self):
52
+ return torch.int64
53
+
54
+ def _create_output(
55
+ self,
56
+ accepted: torch.Tensor, # [batch_size, k]
57
+ substitute_token_ids: torch.Tensor, # [batch_size, k]
58
+ draft_token_ids: torch.Tensor, # [batch_size, k]
59
+ bonus_token_ids: torch.Tensor, # [batch_size]
60
+ ) -> torch.Tensor:
61
+ """Format output. Returns a matrix of token ids. When
62
+ a token is rejected via sampling, all subsequent token ids are
63
+ set to -1 for the sequence.
64
+
65
+ Args:
66
+ accepted: A boolean tensor indicating if the corresponding
67
+ draft token in draft_token_ids should be accepted or not.
68
+ substitute_token_ids: A tensor of token_ids that can be used
69
+ as substitutes for the draft token ids if the proposed token
70
+ is rejected.
71
+ draft_token_ids: A tensor of token ids speculated by the
72
+ draft model.
73
+ bonus_token_ids: Token ids to use as the bonus token if
74
+ all the draft tokens are accepted.
75
+ Returns:
76
+ A tensor containing the accepted token ids. The shape of the
77
+ tensor is [batch_size, k + num_bonus_tokens]
78
+ """
79
+ batch_size, k = substitute_token_ids.shape
80
+ bonus_token_ids = bonus_token_ids.squeeze()
81
+ # Determine the index of the first False value for each row.
82
+ limits = (accepted == 0).max(1).indices
83
+ limits[~(accepted == 0).any(1)] = k
84
+
85
+ # Create masks using the indices.
86
+ indices = torch.arange(k, device=accepted.device).unsqueeze(0)
87
+ accepted_mask = indices < limits.unsqueeze(1)
88
+ after_false_mask = indices == limits.unsqueeze(1)
89
+
90
+ # Create an extended output tensor
91
+ output_with_bonus_tokens = -torch.ones(
92
+ (batch_size, k + self._num_bonus_tokens),
93
+ dtype=self.token_id_dtype,
94
+ device=accepted.device)
95
+ output = output_with_bonus_tokens[:, :k]
96
+
97
+ # Fill in the first k columns of the output tensor using masks and data
98
+ # tensors.
99
+ output[:, :k] = torch.where(accepted_mask, draft_token_ids,
100
+ -torch.ones_like(draft_token_ids))
101
+
102
+ # Fill the last column.
103
+ # We check output directly as accepted may have True values inconsistent
104
+ # with causal acceptance.
105
+ output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
106
+ bonus_token_ids, -1)
107
+
108
+ # We disable bonus tokens because it causes corrupt KV cache for
109
+ # proposal methods that require KV cache. We can fix it by "prefilling"
110
+ # the bonus token in the proposer. The following issue tracks the fix.
111
+ # https://github.com/vllm-project/vllm/issues/4212
112
+ if self._disable_bonus_tokens:
113
+ output_with_bonus_tokens[:, -1] = -1
114
+
115
+ # Fill the recovered token ids.
116
+ output.mul_(~after_false_mask).add_(
117
+ substitute_token_ids.mul(after_false_mask))
118
+
119
+ self.num_accepted_tokens += accepted.sum()
120
+ self.num_emitted_tokens += (output_with_bonus_tokens != -1).sum()
121
+ self.num_draft_tokens += batch_size * k
122
+
123
+ return output_with_bonus_tokens
124
+
125
+ def _raise_if_incorrect_input(
126
+ self,
127
+ target_probs: torch.Tensor,
128
+ draft_token_ids: torch.Tensor,
129
+ bonus_token_ids: torch.Tensor,
130
+ draft_probs: Optional[torch.Tensor] = None,
131
+ ) -> None:
132
+ self._raise_if_incorrect_shape(target_probs, draft_token_ids,
133
+ bonus_token_ids, draft_probs)
134
+ self._raise_if_incorrect_dtype(target_probs, draft_token_ids,
135
+ bonus_token_ids, draft_probs)
136
+ self._raise_if_inconsistent_device(target_probs, draft_token_ids,
137
+ bonus_token_ids, draft_probs)
138
+ self._raise_if_out_of_bounds_vocab(target_probs.shape[-1],
139
+ draft_token_ids, bonus_token_ids)
140
+
141
+ def _raise_if_incorrect_shape(
142
+ self,
143
+ target_probs: torch.Tensor,
144
+ draft_token_ids: torch.Tensor,
145
+ bonus_token_ids: torch.Tensor,
146
+ draft_probs: Optional[torch.Tensor] = None,
147
+ ) -> None:
148
+ (target_batch_size, num_target_probs,
149
+ target_vocab_size) = target_probs.shape
150
+
151
+ # validate the shape of draft token ids.
152
+ draft_token_ids_batch_size, num_draft_token_ids = draft_token_ids.shape
153
+ assert draft_token_ids_batch_size == target_batch_size
154
+ assert num_draft_token_ids == num_target_probs
155
+
156
+ # validate the shape of bonus token ids
157
+ bonus_batch_size, num_bonus_tokens = bonus_token_ids.shape
158
+ assert bonus_batch_size == target_batch_size
159
+ assert num_bonus_tokens == self._num_bonus_tokens
160
+
161
+ # validate the shape of draft probs if it is set
162
+ if draft_probs is not None:
163
+ (draft_batch_size, num_draft_probs,
164
+ draft_vocab_size) = draft_probs.shape
165
+ assert draft_batch_size == target_batch_size
166
+ assert num_draft_probs == num_target_probs
167
+ assert (draft_vocab_size == target_vocab_size
168
+ ), f"{draft_vocab_size=} {target_vocab_size=}"
169
+
170
+ def _raise_if_incorrect_dtype(
171
+ self,
172
+ target_probs: torch.Tensor,
173
+ draft_token_ids: torch.Tensor,
174
+ bonus_token_ids: torch.Tensor,
175
+ draft_probs: Optional[torch.Tensor] = None,
176
+ ) -> None:
177
+ assert target_probs.dtype == self.probs_dtype
178
+ assert draft_token_ids.dtype == self.token_id_dtype
179
+ assert bonus_token_ids.dtype == self.token_id_dtype
180
+ if draft_probs is not None:
181
+ assert draft_probs.dtype == self.probs_dtype
182
+
183
+ def _raise_if_inconsistent_device(
184
+ self,
185
+ target_probs: torch.Tensor,
186
+ draft_token_ids: torch.Tensor,
187
+ bonus_token_ids: torch.Tensor,
188
+ draft_probs: Optional[torch.Tensor] = None,
189
+ ) -> None:
190
+ devices = [
191
+ t.device for t in
192
+ [target_probs, bonus_token_ids, draft_probs, draft_token_ids]
193
+ if t is not None
194
+ ]
195
+ assert all([devices[0] == device for device in devices])
196
+
197
+ def _raise_if_out_of_bounds_vocab(
198
+ self,
199
+ vocab_size: int,
200
+ draft_token_ids: torch.Tensor,
201
+ bonus_token_ids: torch.Tensor,
202
+ ) -> None:
203
+ assert torch.all(bonus_token_ids < vocab_size)
204
+ assert torch.all(bonus_token_ids >= 0)
205
+ assert torch.all(draft_token_ids < vocab_size)
206
+ assert torch.all(draft_token_ids >= 0)
vllm/model_executor/layers/typical_acceptance_sampler.py
new
+186
@@ -0,0 +1,186 @@
1
+import torch
2
+import torch.jit
3
+import torch.nn as nn
4
+
5
+from vllm.model_executor.layers.spec_decode_base_sampler import (
6
+ SpecDecodeBaseSampler)
7
+
8
+
9
+class TypicalAcceptanceSampler(SpecDecodeBaseSampler, nn.Module):
10
+ """Apply typical acceptance sampling as described in section 3.3.1 in
11
+ "MEDUSA: Simple LLM Inference Acceleration Framework with
12
+ Multiple Decoding Heads"
13
+ https://arxiv.org/pdf/2401.10774
14
+ """
15
+
16
+ def __init__(
17
+ self,
18
+ disable_bonus_tokens: bool = False,
19
+ strict_mode: bool = False,
20
+ posterior_threshold: float = 0.09,
21
+ posterior_alpha: float = 0.3,
22
+ ):
23
+ """Create a Typical Acceptance Sampler.
24
+
25
+ Args:
26
+ disable_bonus_tokens: Whether or not to disable the bonus token.
27
+ Require when bonus tokens will cause corrupt KV cache for
28
+ proposal methods that require KV cache.
29
+ strict_mode: Whether or not to perform shape/device/dtype checks
30
+ during sampling. This catches correctness issues but adds
31
+ nontrivial latency.
32
+ posterior_threshold : A threshold value that sets a lower bound
33
+ on the posterior probability of a token in target model for it
34
+ to be accepted. Default is 0.09
35
+ posterior_alpha : A scaling factor for the entropy-based
36
+ threshold in typical acceptance sampling. Typically defaults to
37
+ sqrt of posterior_threshold and is set to 0.3.
38
+ """
39
+ SpecDecodeBaseSampler.__init__(
40
+ self,
41
+ disable_bonus_tokens=disable_bonus_tokens,
42
+ strict_mode=strict_mode)
43
+ nn.Module.__init__(self)
44
+ self._posterior_threshold = posterior_threshold
45
+ self._posterior_alpha = posterior_alpha
46
+
47
+ def forward(
48
+ self,
49
+ target_probs: torch.Tensor,
50
+ bonus_token_ids: torch.Tensor,
51
+ draft_token_ids: torch.Tensor,
52
+ ) -> torch.Tensor:
53
+ """Sample token ids using typical acceptance sampling. This accepts
54
+ or rejects tokens proposed by the draft model using the probability
55
+ of each token according to the draft and target models.
56
+
57
+ In the worst case where all draft tokens are rejected, it is guaranteed
58
+ one token will be emitted.
59
+
60
+ In the case where all draft tokens are accepted, the bonus token will be
61
+ accepted conditioned on self._disable_bonus_tokens being false.
62
+
63
+ Args:
64
+ target_probs: The probability distribution over token ids given
65
+ context according to the target model.
66
+ shape = [batch_size, num_speculative_tokens, vocab_size]
67
+
68
+ bonus_token_ids: The "bonus" token ids that are accepted iff all
69
+ speculative tokens in a sequence are accepted.
70
+ shape = [batch_size, num_bonus_tokens]
71
+
72
+ draft_token_ids: The token ids that were sampled from the draft
73
+ probabilities.
74
+ shape = [batch_size, num_speculative_tokens]
75
+
76
+ Returns:
77
+ output_token_ids: The token ids sampled via rejection sampling,
78
+ or -1 if unable to sample a token because the previous token
79
+ was rejected.
80
+ shape = [batch_size, num_speculative_tokens + num_bonus_tokens]
81
+ """
82
+ # Only perform shape/dtype/device checking in strict mode, as it adds
83
+ # overhead.
84
+ if self._strict_mode:
85
+ self._raise_if_incorrect_input(target_probs, draft_token_ids,
86
+ bonus_token_ids)
87
+ accepted = self._evaluate_accepted_tokens(target_probs,
88
+ draft_token_ids)
89
+ recovered_token_ids = self._replacement_token_ids(target_probs)
90
+ output_token_ids = self._create_output(accepted, recovered_token_ids,
91
+ draft_token_ids,
92
+ bonus_token_ids)
93
+ return output_token_ids
94
+
95
+ def _evaluate_accepted_tokens(self, target_probs, draft_token_ids):
96
+ r"""
97
+ Evaluates and returns a mask of accepted tokens based on the
98
+ posterior probabilities.
99
+
100
+ Parameters:
101
+ ----------
102
+ target_probs : torch.Tensor
103
+ A tensor of shape (batch_size, k, vocab_size) representing
104
+ the probabilities of each token in the vocabulary for each
105
+ position in the proposed sequence. This is the distribution
106
+ generated by the target model.
107
+ draft_token_ids : torch.Tensor
108
+ A tensor of shape (batch_size, k) representing the proposed
109
+ token ids.
110
+
111
+ A draft token_id x_{n+k} is accepted if it satisfies the
112
+ following condition
113
+
114
+ .. math::
115
+ p_{\text{original}}(x_{n+k} | x_1, x_2, \dots, x_{n+k-1}) >
116
+ \min \left( \epsilon, \delta * \exp \left(
117
+ -H(p_{\text{original}}(
118
+ \cdot | x_1, x_2, \ldots, x_{n+k-1})) \right) \right)
119
+
120
+ where :math:`p_{\text{original}}` corresponds to target_probs
121
+ and :math:`\epsilon` and :math:`\delta` correspond to hyperparameters
122
+ specified using self._posterior_threshold and self._posterior_alpha
123
+
124
+ This method computes the posterior probabilities for the given
125
+ draft token ids based on the provided target probabilities. It
126
+ calculates the entropy of the posterior distribution and determines
127
+ a dynamic threshold for each token position using the provided
128
+ posterior_threshold and posterior_alpha values. The method then
129
+ returns a boolean mask indicating which tokens can be accepted.
130
+
131
+ Returns:
132
+ -------
133
+ torch.Tensor
134
+ A boolean tensor of shape (batch_size, k) where each element
135
+ indicates whether the corresponding draft token has been accepted
136
+ or rejected. True indicates acceptance and false indicates
137
+ rejection.
138
+
139
+ """
140
+ device = target_probs.device
141
+ candidates_prob = torch.gather(
142
+ target_probs, dim=-1,
143
+ index=draft_token_ids.unsqueeze(-1)).squeeze(-1)
144
+ # A small constant added to prevent computing the logarithm of zero,
145
+ # which can lead to undefined values.
146
+ epsilon = 1e-5
147
+ posterior_entropy = -torch.sum(
148
+ target_probs * torch.log(target_probs + epsilon), dim=-1)
149
+ threshold = torch.minimum(
150
+ torch.ones_like(posterior_entropy, device=device) *
151
+ self._posterior_threshold,
152
+ torch.exp(-posterior_entropy) * self._posterior_alpha,
153
+ )
154
+ accepted_mask = candidates_prob > threshold
155
+ return accepted_mask
156
+
157
+ def _replacement_token_ids(self, target_probs):
158
+ """
159
+ Generate one replacement token ID for each sequence based on target
160
+ probabilities. The replacement token is used as the fallback option
161
+ if typical acceptance sampling does not accept any draft tokens for
162
+ that particular sequence.
163
+
164
+ This method computes the token IDs to be replaced by selecting the
165
+ token with the highest probability for each sequence in the first
166
+ position. The rest of the output is filled with -1.
167
+
168
+ Parameters
169
+ ----------
170
+ target_probs : torch.Tensor
171
+ A tensor of shape (batch_size, k, vocab_size) containing
172
+ the target probability distribution
173
+
174
+ Returns
175
+ -------
176
+ torch.Tensor
177
+ A tensor of shape (batch_size, k) with the replacement
178
+ token IDs. Only the first column is set, and the rest of the
179
+ columns are filled with -1.
180
+ """
181
+ max_indices = torch.argmax(target_probs[:, 0, :], dim=1)
182
+ output = -torch.ones((target_probs.shape[0], target_probs.shape[1]),
183
+ dtype=self.token_id_dtype,
184
+ device=target_probs.device)
185
+ output[:, 0] = max_indices
186
+ return output