A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[SpecDec][Misc] Cleanup, remove bonus token logic. (#8701)
Lily Liu committed
Sep 22, 2024 at 12:34 UTC
c6bd70d7728b50f358cb5cb6e66e02b75aeb3d20
7 files changed
+33
-115
tests/samplers/test_rejection_sampler.py
+7
-23
@@ -42,18 +42,13 @@ def mock_causal_accepted_tensor(
42
@pytest.mark.parametrize(
43
"which_tokens_accepted",
44
["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
45
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
45
@pytest.mark.parametrize("device", CUDA_DEVICES)
46
@pytest.mark.parametrize("use_flashinfer", [True, False])
47
@torch.inference_mode()
48
def test_correct_output_format(which_tokens_accepted: str, seed: int,
50
- disable_bonus_tokens: bool, device: str,
51
- use_flashinfer: bool):
49
+ device: str, use_flashinfer: bool):
50
"""Verify the output has correct format given predetermined accepted matrix.
51
"""
54
- if use_flashinfer and disable_bonus_tokens:
55
- pytest.skip("Flashinfer rejection sampler must enable bonus token.")
56
-
52
set_random_seed(seed)
53
torch.set_default_device(device)
54
@@ -88,9 +83,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
83
size=(batch_size, 1),
84
dtype=torch.int64)
85
91
- rejection_sampler = RejectionSampler(
92
- disable_bonus_tokens=disable_bonus_tokens,
93
- use_flashinfer=use_flashinfer)
86
+ rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
87
rejection_sampler.init_gpu_tensors(device=device)
88
output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
89
accepted,
@@ -100,10 +93,6 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
93
)
94
95
expected_bonus_token_ids = bonus_token_ids.clone()
103
- # If bonus tokens disabled. Verify they are set to -1.
104
- # See https://github.com/vllm-project/vllm/issues/4212
105
- if disable_bonus_tokens:
106
- expected_bonus_token_ids = expected_bonus_token_ids * 0 - 1
96
97
if which_tokens_accepted == "all_tokens_accepted":
98
# Expect all tokens to be equal to draft tokens.
@@ -143,8 +132,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
132
def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
133
device: str, use_flashinfer: bool):
134
torch.set_default_device(device)
146
- rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
147
- use_flashinfer=use_flashinfer)
135
+ rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
136
rejection_sampler.init_gpu_tensors(device=device)
137
138
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -177,8 +165,7 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
165
frac_seeded: float, n_rep: int, device: str,
166
use_flashinfer: bool):
167
torch.set_default_device(device)
180
- rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
181
- use_flashinfer=use_flashinfer)
168
+ rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
169
rejection_sampler.init_gpu_tensors(device=device)
170
171
draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -251,8 +238,7 @@ def test_compare_nonflashinfer_backend(k: int, vocab_size: int,
238
}
239
240
for use_flashinfer in [True, False]:
254
- rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
255
- use_flashinfer=use_flashinfer)
241
+ rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
242
rejection_sampler.init_gpu_tensors(device=device)
243
# We use seeded sequences to ensure the same tokens are accepted
244
# for both flashinfer and nonflashinfer backends.
@@ -282,8 +268,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
268
vocab_size = 30_000
269
torch.set_default_device(device)
270
285
- rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
286
- use_flashinfer=use_flashinfer,
271
+ rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer,
272
strict_mode=True)
273
rejection_sampler.init_gpu_tensors(device=device)
274
@@ -359,8 +344,7 @@ def test_rejection_sampling_approximates_target_distribution(
344
set_random_seed(seed)
345
helper = _CorrectnessTestHelper(
346
vocab_size=10,
362
- rejection_sampler=RejectionSampler(disable_bonus_tokens=False,
363
- use_flashinfer=use_flashinfer),
347
+ rejection_sampler=RejectionSampler(use_flashinfer=use_flashinfer),
348
)
349
350
draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
tests/samplers/test_typical_acceptance_sampler.py
+20
-59
@@ -55,14 +55,13 @@ def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
55
def get_acceptance_sampler(
56
posterior_threshold: float = 0.03,
57
posterior_alpha: float = 0.9,
58
- disable_bonus_tokens: bool = False,
58
strict_mode: bool = False,
59
) -> TypicalAcceptanceSampler:
60
"""
61
Initializes and returns a TypicalAcceptanceSampler.
62
"""
63
return TypicalAcceptanceSampler(posterior_threshold, posterior_alpha,
65
- disable_bonus_tokens, strict_mode)
64
+ strict_mode)
65
66
67
@pytest.mark.parametrize("k", list(range(1, 6)))
@@ -154,11 +153,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
153
154
155
@pytest.mark.parametrize("seed", list(range(10)))
157
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
156
@pytest.mark.parametrize("device", CUDA_DEVICES)
157
@torch.inference_mode()
158
def test_uniform_target_distribution_accepts_all_tokens(
161
- seed: int, disable_bonus_tokens: bool, device: str):
159
+ seed: int, device: str):
160
"""
161
Test the TypicalAcceptanceSampler with a uniform target probability
162
distribution.
@@ -166,17 +164,14 @@ def test_uniform_target_distribution_accepts_all_tokens(
164
This test verifies that when provided with a uniform target probability
165
distribution, the TypicalAcceptanceSampler accepts all draft tokens. The
166
entropy of the uniform target distribution being high should lead to all
169
- draft tokens being accepted. The test also ensures that the behavior
170
- regarding bonus tokens is consistent with the `disable_bonus_tokens`
171
- flag.
167
+ draft tokens being accepted.
168
"""
169
set_random_seed(seed)
170
k = 3
171
batch_size = 5
172
vocab_size = 30_000
173
torch.set_default_device(device)
178
- typical_acceptance_sampler = get_acceptance_sampler(
179
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
175
typical_acceptance_sampler.init_gpu_tensors(device=device)
176
target_with_bonus_probs = torch.rand(batch_size,
177
k + 1,
@@ -200,21 +195,15 @@ def test_uniform_target_distribution_accepts_all_tokens(
195
# should lead to all draft tokens being accepted. Verify that.
196
assert output_token_ids.shape[0] == batch_size
197
assert output_token_ids.shape[1] == (k + 1)
203
- if disable_bonus_tokens:
204
- assert torch.all(output_token_ids[:, -1] == -1)
205
- else:
206
- assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
198
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
199
200
assert torch.all(output_token_ids[:, :k] == draft_token_ids)
201
202
203
@pytest.mark.parametrize("seed", list(range(10)))
212
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
204
@pytest.mark.parametrize("device", CUDA_DEVICES)
205
@torch.inference_mode()
215
-def test_temperature_zero_target_distribution(seed: int,
216
- disable_bonus_tokens: bool,
217
- device: str):
206
+def test_temperature_zero_target_distribution(seed: int, device: str):
207
"""
208
Test the TypicalAcceptanceSampler with a zero-temperature target
209
probability distribution.
@@ -232,8 +221,7 @@ def test_temperature_zero_target_distribution(seed: int,
221
vocab_size = 30_000
222
torch.set_default_device(device)
223
235
- typical_acceptance_sampler = get_acceptance_sampler(
236
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
224
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
225
typical_acceptance_sampler.init_gpu_tensors(device=device)
226
# Simulate temperature 0 probability distribution for target probabilities
227
# and create target probabilities such that only 1 token id has
@@ -267,11 +255,9 @@ def test_temperature_zero_target_distribution(seed: int,
255
256
257
@pytest.mark.parametrize("seed", list(range(10)))
270
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
258
@pytest.mark.parametrize("device", CUDA_DEVICES)
259
@torch.inference_mode()
273
-def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
274
- device: str):
260
+def test_mixed_target_distribution(seed: int, device: str):
261
"""
262
Test the TypicalAcceptanceSampler with a mixed target probability
263
distribution.
@@ -285,16 +271,13 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
271
with a probability of 1.0 is accepted, and all other tokens are rejected.
272
- For sequences with a uniform distribution, all draft tokens are
273
accepted.
288
- - When `disable_bonus_tokens` is False, the bonus tokens are also accepted
289
- for sequences with a uniform distribution.
274
"""
275
set_random_seed(seed)
276
k = 3
277
batch_size = 4
278
vocab_size = 30_000
279
torch.set_default_device(device)
296
- typical_acceptance_sampler = get_acceptance_sampler(
297
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
280
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
281
typical_acceptance_sampler.init_gpu_tensors(device=device)
282
# For sequences 0 and 2 set the distribution to a temperature
283
# zero distribution. For sequences 1 and 3 set it to a uniform
@@ -328,21 +311,16 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
311
0]))
312
# For sequences 1 and 3 verify that all tokens are accepted since the
313
# target probability distribution is uniform. In addition verify that
331
- # if disable_bonus_tokens is false then we also accept the bonus tokens.
314
+ # we also accept the bonus tokens.
315
assert torch.all(
316
output_token_ids[[1, 3], :-1] == draft_token_ids[[1, 3], :])
334
- if disable_bonus_tokens:
335
- assert torch.all(output_token_ids[[1, 3], -1] == -1)
336
- else:
337
- assert torch.all(output_token_ids[[1, 3], -1] != -1)
317
+ assert torch.all(output_token_ids[[1, 3], -1] != -1)
318
319
320
@pytest.mark.parametrize("seed", list(range(10)))
341
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
321
@pytest.mark.parametrize("device", CUDA_DEVICES)
322
@torch.inference_mode()
344
-def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
345
- device: str):
323
+def test_accept_tokens_partially(seed: int, device: str):
324
"""
325
Test the TypicalAcceptanceSampler's behavior when only a subset of draft
326
tokens should be accepted.
@@ -362,8 +340,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
340
batch_size = 1
341
vocab_size = 30_000
342
torch.set_default_device(device)
365
- typical_acceptance_sampler = get_acceptance_sampler(
366
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
343
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
344
typical_acceptance_sampler.init_gpu_tensors(device=device)
345
# Create a temperature zero target probability distribution and ensure
346
# all draft token ids correspond to the tokens with 1.0 probability.
@@ -384,10 +361,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
361
assert output_token_ids.shape[0] == batch_size
362
assert output_token_ids.shape[1] == (k + 1)
363
assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
387
- if disable_bonus_tokens:
388
- assert torch.all(output_token_ids[:, -1] == -1)
389
- else:
390
- assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
364
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
365
# Next only keep the first 2 draft tokens same as the zero temperature
366
# tokens. For the remaining 3 choose some other tokens. In the
367
# response we will expect the first 2 tokens to be the same as the
@@ -408,12 +382,9 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
382
383
384
@pytest.mark.parametrize("seed", list(range(1)))
411
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
385
@pytest.mark.parametrize("device", CUDA_DEVICES)
386
@torch.inference_mode()
414
-def test_accept_tokens_set_non_default_posteriors(seed: int,
415
- disable_bonus_tokens: bool,
416
- device: str):
387
+def test_accept_tokens_set_non_default_posteriors(seed: int, device: str):
388
"""
389
Test the TypicalAcceptanceSampler with custom posterior thresholds and
390
alpha values. This test verifies that by modifying the posterior
@@ -425,8 +396,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
396
batch_size = 1
397
vocab_size = 30_000
398
torch.set_default_device(device)
428
- typical_acceptance_sampler = get_acceptance_sampler(
429
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
399
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
400
typical_acceptance_sampler.init_gpu_tensors(device=device)
401
# Simulate temperature 0 probability distribution for target
402
# probabilities and create target probabilities such that only 1 token
@@ -457,10 +427,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
427
# now accept even draft tokens with very low probability in the
428
# target distribution. Simulate and verify the same.
429
typical_acceptance_sampler = TypicalAcceptanceSampler(
460
- strict_mode=True,
461
- disable_bonus_tokens=disable_bonus_tokens,
462
- posterior_threshold=0.0,
463
- posterior_alpha=0.0)
430
+ strict_mode=True, posterior_threshold=0.0, posterior_alpha=0.0)
431
typical_acceptance_sampler.init_gpu_tensors(device=device)
432
output_token_ids = typical_acceptance_sampler(
433
target_probs,
@@ -470,18 +437,13 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
437
assert output_token_ids.shape[0] == batch_size
438
assert output_token_ids.shape[1] == (k + 1)
439
assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
473
- if disable_bonus_tokens:
474
- assert torch.all(output_token_ids[:, -1] == -1)
475
- else:
476
- assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
440
+ assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
441
442
443
@pytest.mark.parametrize("seed", list(range(10)))
480
-@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
444
@pytest.mark.parametrize("device", CUDA_DEVICES)
445
@torch.inference_mode()
483
-def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
484
- device: str):
446
+def test_replacement_token_ids(seed: int, device: str):
447
"""
448
Test the TypicalAcceptanceSampler's method for generating
449
replacement token IDs.
@@ -497,8 +459,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
459
batch_size = 5
460
vocab_size = 30_000
461
torch.set_default_device(device)
500
- typical_acceptance_sampler = get_acceptance_sampler(
501
- strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
462
+ typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
463
typical_acceptance_sampler.init_gpu_tensors(device=device)
464
target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
465
expected_replacement_tokens = -torch.ones(
tests/spec_decode/e2e/test_medusa_correctness.py
+1
-1
@@ -31,7 +31,7 @@ MAIN_MODEL = "JackFram/llama-68m"
31
# speculative model
32
SPEC_MODEL = "abhigoyal/vllm-medusa-llama-68m-random"
33
34
-# max. number of speculative tokens: this corresponds to
34
+# max number of speculative tokens: this corresponds to
35
# num_heads in the config.json of the speculator model.
36
MAX_SPEC_TOKENS = 5
37
vllm/model_executor/layers/rejection_sampler.py
+1
-8
@@ -31,15 +31,11 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
31
"""
32
33
def __init__(self,
34
- disable_bonus_tokens: bool = True,
34
strict_mode: bool = False,
35
use_flashinfer: Optional[bool] = None):
36
"""Create a rejection sampler.
37
38
Args:
40
- disable_bonus_tokens: Whether or not to disable the bonus token.
41
- Require when bonus tokens will cause corrupt KV cache for
42
- proposal methods that require KV cache.
39
strict_mode: Whether or not to perform shape/device/dtype checks
40
during sampling. This catches correctness issues but adds
41
nontrivial latency.
@@ -48,8 +44,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
44
None, we will use the default value from the environment variable.
45
This parameter is only used for testing purposes.
46
"""
51
- super().__init__(disable_bonus_tokens=disable_bonus_tokens,
52
- strict_mode=strict_mode)
47
+ super().__init__(strict_mode=strict_mode)
48
if use_flashinfer is None:
49
self.use_flashinfer = envs.VLLM_USE_FLASHINFER_SAMPLER and (
50
chain_speculative_sampling is not None)
@@ -57,8 +52,6 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
52
self.use_flashinfer = use_flashinfer
53
54
if self.use_flashinfer:
60
- assert not disable_bonus_tokens, \
61
- "flashinfer will enable bonus token by default"
55
logger.info("Use flashinfer for rejection sampling.")
56
else:
57
logger.info("Use pytorch for rejection sampling.")
vllm/model_executor/layers/spec_decode_base_sampler.py
+1
-14
@@ -11,20 +11,14 @@ class SpecDecodeBaseSampler(nn.Module):
11
step.
12
"""
13
14
- def __init__(self,
15
- disable_bonus_tokens: bool = True,
16
- strict_mode: bool = False):
14
+ def __init__(self, strict_mode: bool = False):
15
"""Base class constructor.
16
Args:
19
- disable_bonus_tokens: Whether or not to disable the bonus token.
20
- Require when bonus tokens will cause corrupt KV cache for
21
- proposal methods that require KV cache.
17
strict_mode: Whether or not to perform shape/device/dtype checks
18
during sampling. This catches correctness issues but adds
19
nontrivial latency.
20
"""
21
super().__init__()
27
- self._disable_bonus_tokens = disable_bonus_tokens
22
self._strict_mode = strict_mode
23
24
# NOTE: A "bonus token" is accepted iff all proposal tokens are
@@ -111,13 +105,6 @@ class SpecDecodeBaseSampler(nn.Module):
105
output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
106
bonus_token_ids, -1)
107
114
- # We disable bonus tokens because it causes corrupt KV cache for
115
- # proposal methods that require KV cache. We can fix it by "prefilling"
116
- # the bonus token in the proposer. The following issue tracks the fix.
117
- # https://github.com/vllm-project/vllm/issues/4212
118
- if self._disable_bonus_tokens:
119
- output_with_bonus_tokens[:, -1] = -1
120
-
108
# Fill the recovered token ids.
109
output.mul_(~after_false_mask).add_(
110
substitute_token_ids.mul(after_false_mask))
vllm/model_executor/layers/typical_acceptance_sampler.py
+2
-7
@@ -16,15 +16,11 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
16
self,
17
posterior_threshold: float,
18
posterior_alpha: float,
19
- disable_bonus_tokens: bool = False,
19
strict_mode: bool = False,
20
):
21
"""Create a Typical Acceptance Sampler.
22
23
Args:
25
- disable_bonus_tokens: Whether or not to disable the bonus token.
26
- Require when bonus tokens will cause corrupt KV cache for
27
- proposal methods that require KV cache.
24
strict_mode: Whether or not to perform shape/device/dtype checks
25
during sampling. This catches correctness issues but adds
26
nontrivial latency.
@@ -36,8 +32,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
32
"""
33
self._posterior_threshold = posterior_threshold
34
self._posterior_alpha = posterior_alpha
39
- super().__init__(disable_bonus_tokens=disable_bonus_tokens,
40
- strict_mode=strict_mode)
35
+ super().__init__(strict_mode=strict_mode)
36
37
def forward(
38
self,
@@ -54,7 +49,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
49
one token will be emitted.
50
51
In the case where all draft tokens are accepted, the bonus token will be
57
- accepted conditioned on self._disable_bonus_tokens being false.
52
+ accepted.
53
54
Args:
55
target_probs: The probability distribution over token ids given
vllm/spec_decode/spec_decode_worker.py
+1
-3
@@ -164,11 +164,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
164
165
spec_decode_sampler: SpecDecodeBaseSampler = None
166
if draft_token_acceptance_method == "rejection_sampler":
167
- spec_decode_sampler = RejectionSampler(
168
- disable_bonus_tokens=False, )
167
+ spec_decode_sampler = RejectionSampler()
168
elif draft_token_acceptance_method == "typical_acceptance_sampler":
169
spec_decode_sampler = TypicalAcceptanceSampler(
171
- disable_bonus_tokens=False,
170
posterior_threshold=\
171
typical_acceptance_sampler_posterior_threshold,
172
posterior_alpha=typical_acceptance_sampler_posterior_alpha,