@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6bd70d772
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[SpecDec][Misc] Cleanup, remove bonus token logic. (#8701)

Lily Liu committed Sep 22, 2024 at 12:34 UTC c6bd70d7728b50f358cb5cb6e66e02b75aeb3d20
7 files changed +33 -115
tests/samplers/test_rejection_sampler.py
+7 -23
@@ -42,18 +42,13 @@ def mock_causal_accepted_tensor(
42 @pytest.mark.parametrize(
43 "which_tokens_accepted",
44 ["all_tokens_accepted", "no_tokens_accepted", "some_tokens_accepted"])
45 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
45 @pytest.mark.parametrize("device", CUDA_DEVICES)
46 @pytest.mark.parametrize("use_flashinfer", [True, False])
47 @torch.inference_mode()
48 def test_correct_output_format(which_tokens_accepted: str, seed: int,
50 - disable_bonus_tokens: bool, device: str,
51 - use_flashinfer: bool):
49 + device: str, use_flashinfer: bool):
50 """Verify the output has correct format given predetermined accepted matrix.
51 """
54 - if use_flashinfer and disable_bonus_tokens:
55 - pytest.skip("Flashinfer rejection sampler must enable bonus token.")
56 -
52 set_random_seed(seed)
53 torch.set_default_device(device)
54
@@ -88,9 +83,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
83 size=(batch_size, 1),
84 dtype=torch.int64)
85
91 - rejection_sampler = RejectionSampler(
92 - disable_bonus_tokens=disable_bonus_tokens,
93 - use_flashinfer=use_flashinfer)
86 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
87 rejection_sampler.init_gpu_tensors(device=device)
88 output_token_ids = rejection_sampler._create_output( # pylint: disable=protected-access
89 accepted,
@@ -100,10 +93,6 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
93 )
94
95 expected_bonus_token_ids = bonus_token_ids.clone()
103 - # If bonus tokens disabled. Verify they are set to -1.
104 - # See https://github.com/vllm-project/vllm/issues/4212
105 - if disable_bonus_tokens:
106 - expected_bonus_token_ids = expected_bonus_token_ids * 0 - 1
96
97 if which_tokens_accepted == "all_tokens_accepted":
98 # Expect all tokens to be equal to draft tokens.
@@ -143,8 +132,7 @@ def test_correct_output_format(which_tokens_accepted: str, seed: int,
132 def test_no_crash_with_varying_dims(k: int, vocab_size: int, batch_size: int,
133 device: str, use_flashinfer: bool):
134 torch.set_default_device(device)
146 - rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
147 - use_flashinfer=use_flashinfer)
135 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
136 rejection_sampler.init_gpu_tensors(device=device)
137
138 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -177,8 +165,7 @@ def test_deterministic_when_seeded(k: int, vocab_size: int, batch_size: int,
165 frac_seeded: float, n_rep: int, device: str,
166 use_flashinfer: bool):
167 torch.set_default_device(device)
180 - rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
181 - use_flashinfer=use_flashinfer)
168 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
169 rejection_sampler.init_gpu_tensors(device=device)
170
171 draft_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
@@ -251,8 +238,7 @@ def test_compare_nonflashinfer_backend(k: int, vocab_size: int,
238 }
239
240 for use_flashinfer in [True, False]:
254 - rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
255 - use_flashinfer=use_flashinfer)
241 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer)
242 rejection_sampler.init_gpu_tensors(device=device)
243 # We use seeded sequences to ensure the same tokens are accepted
244 # for both flashinfer and nonflashinfer backends.
@@ -282,8 +268,7 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
268 vocab_size = 30_000
269 torch.set_default_device(device)
270
285 - rejection_sampler = RejectionSampler(disable_bonus_tokens=False,
286 - use_flashinfer=use_flashinfer,
271 + rejection_sampler = RejectionSampler(use_flashinfer=use_flashinfer,
272 strict_mode=True)
273 rejection_sampler.init_gpu_tensors(device=device)
274
@@ -359,8 +344,7 @@ def test_rejection_sampling_approximates_target_distribution(
344 set_random_seed(seed)
345 helper = _CorrectnessTestHelper(
346 vocab_size=10,
362 - rejection_sampler=RejectionSampler(disable_bonus_tokens=False,
363 - use_flashinfer=use_flashinfer),
347 + rejection_sampler=RejectionSampler(use_flashinfer=use_flashinfer),
348 )
349
350 draft_probs, target_probs, reference_probs = helper.generate_probs_for_test(
tests/samplers/test_typical_acceptance_sampler.py
+20 -59
@@ -55,14 +55,13 @@ def get_draft_token_ids(batch_size: int, k: int, vocab_size: int,
55 def get_acceptance_sampler(
56 posterior_threshold: float = 0.03,
57 posterior_alpha: float = 0.9,
58 - disable_bonus_tokens: bool = False,
58 strict_mode: bool = False,
59 ) -> TypicalAcceptanceSampler:
60 """
61 Initializes and returns a TypicalAcceptanceSampler.
62 """
63 return TypicalAcceptanceSampler(posterior_threshold, posterior_alpha,
65 - disable_bonus_tokens, strict_mode)
64 + strict_mode)
65
66
67 @pytest.mark.parametrize("k", list(range(1, 6)))
@@ -154,11 +153,10 @@ def test_raises_when_vocab_oob(above_or_below_vocab_range: str,
153
154
155 @pytest.mark.parametrize("seed", list(range(10)))
157 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
156 @pytest.mark.parametrize("device", CUDA_DEVICES)
157 @torch.inference_mode()
158 def test_uniform_target_distribution_accepts_all_tokens(
161 - seed: int, disable_bonus_tokens: bool, device: str):
159 + seed: int, device: str):
160 """
161 Test the TypicalAcceptanceSampler with a uniform target probability
162 distribution.
@@ -166,17 +164,14 @@ def test_uniform_target_distribution_accepts_all_tokens(
164 This test verifies that when provided with a uniform target probability
165 distribution, the TypicalAcceptanceSampler accepts all draft tokens. The
166 entropy of the uniform target distribution being high should lead to all
169 - draft tokens being accepted. The test also ensures that the behavior
170 - regarding bonus tokens is consistent with the `disable_bonus_tokens`
171 - flag.
167 + draft tokens being accepted.
168 """
169 set_random_seed(seed)
170 k = 3
171 batch_size = 5
172 vocab_size = 30_000
173 torch.set_default_device(device)
178 - typical_acceptance_sampler = get_acceptance_sampler(
179 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
174 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
175 typical_acceptance_sampler.init_gpu_tensors(device=device)
176 target_with_bonus_probs = torch.rand(batch_size,
177 k + 1,
@@ -200,21 +195,15 @@ def test_uniform_target_distribution_accepts_all_tokens(
195 # should lead to all draft tokens being accepted. Verify that.
196 assert output_token_ids.shape[0] == batch_size
197 assert output_token_ids.shape[1] == (k + 1)
203 - if disable_bonus_tokens:
204 - assert torch.all(output_token_ids[:, -1] == -1)
205 - else:
206 - assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
198 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids.squeeze())
199
200 assert torch.all(output_token_ids[:, :k] == draft_token_ids)
201
202
203 @pytest.mark.parametrize("seed", list(range(10)))
212 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
204 @pytest.mark.parametrize("device", CUDA_DEVICES)
205 @torch.inference_mode()
215 -def test_temperature_zero_target_distribution(seed: int,
216 - disable_bonus_tokens: bool,
217 - device: str):
206 +def test_temperature_zero_target_distribution(seed: int, device: str):
207 """
208 Test the TypicalAcceptanceSampler with a zero-temperature target
209 probability distribution.
@@ -232,8 +221,7 @@ def test_temperature_zero_target_distribution(seed: int,
221 vocab_size = 30_000
222 torch.set_default_device(device)
223
235 - typical_acceptance_sampler = get_acceptance_sampler(
236 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
224 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
225 typical_acceptance_sampler.init_gpu_tensors(device=device)
226 # Simulate temperature 0 probability distribution for target probabilities
227 # and create target probabilities such that only 1 token id has
@@ -267,11 +255,9 @@ def test_temperature_zero_target_distribution(seed: int,
255
256
257 @pytest.mark.parametrize("seed", list(range(10)))
270 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
258 @pytest.mark.parametrize("device", CUDA_DEVICES)
259 @torch.inference_mode()
273 -def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
274 - device: str):
260 +def test_mixed_target_distribution(seed: int, device: str):
261 """
262 Test the TypicalAcceptanceSampler with a mixed target probability
263 distribution.
@@ -285,16 +271,13 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
271 with a probability of 1.0 is accepted, and all other tokens are rejected.
272 - For sequences with a uniform distribution, all draft tokens are
273 accepted.
288 - - When `disable_bonus_tokens` is False, the bonus tokens are also accepted
289 - for sequences with a uniform distribution.
274 """
275 set_random_seed(seed)
276 k = 3
277 batch_size = 4
278 vocab_size = 30_000
279 torch.set_default_device(device)
296 - typical_acceptance_sampler = get_acceptance_sampler(
297 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
280 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
281 typical_acceptance_sampler.init_gpu_tensors(device=device)
282 # For sequences 0 and 2 set the distribution to a temperature
283 # zero distribution. For sequences 1 and 3 set it to a uniform
@@ -328,21 +311,16 @@ def test_mixed_target_distribution(seed: int, disable_bonus_tokens: bool,
311 0]))
312 # For sequences 1 and 3 verify that all tokens are accepted since the
313 # target probability distribution is uniform. In addition verify that
331 - # if disable_bonus_tokens is false then we also accept the bonus tokens.
314 + # we also accept the bonus tokens.
315 assert torch.all(
316 output_token_ids[[1, 3], :-1] == draft_token_ids[[1, 3], :])
334 - if disable_bonus_tokens:
335 - assert torch.all(output_token_ids[[1, 3], -1] == -1)
336 - else:
337 - assert torch.all(output_token_ids[[1, 3], -1] != -1)
317 + assert torch.all(output_token_ids[[1, 3], -1] != -1)
318
319
320 @pytest.mark.parametrize("seed", list(range(10)))
341 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
321 @pytest.mark.parametrize("device", CUDA_DEVICES)
322 @torch.inference_mode()
344 -def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
345 - device: str):
323 +def test_accept_tokens_partially(seed: int, device: str):
324 """
325 Test the TypicalAcceptanceSampler's behavior when only a subset of draft
326 tokens should be accepted.
@@ -362,8 +340,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
340 batch_size = 1
341 vocab_size = 30_000
342 torch.set_default_device(device)
365 - typical_acceptance_sampler = get_acceptance_sampler(
366 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
343 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
344 typical_acceptance_sampler.init_gpu_tensors(device=device)
345 # Create a temperature zero target probability distribution and ensure
346 # all draft token ids correspond to the tokens with 1.0 probability.
@@ -384,10 +361,7 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
361 assert output_token_ids.shape[0] == batch_size
362 assert output_token_ids.shape[1] == (k + 1)
363 assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
387 - if disable_bonus_tokens:
388 - assert torch.all(output_token_ids[:, -1] == -1)
389 - else:
390 - assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
364 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
365 # Next only keep the first 2 draft tokens same as the zero temperature
366 # tokens. For the remaining 3 choose some other tokens. In the
367 # response we will expect the first 2 tokens to be the same as the
@@ -408,12 +382,9 @@ def test_accept_tokens_partially(seed: int, disable_bonus_tokens: bool,
382
383
384 @pytest.mark.parametrize("seed", list(range(1)))
411 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
385 @pytest.mark.parametrize("device", CUDA_DEVICES)
386 @torch.inference_mode()
414 -def test_accept_tokens_set_non_default_posteriors(seed: int,
415 - disable_bonus_tokens: bool,
416 - device: str):
387 +def test_accept_tokens_set_non_default_posteriors(seed: int, device: str):
388 """
389 Test the TypicalAcceptanceSampler with custom posterior thresholds and
390 alpha values. This test verifies that by modifying the posterior
@@ -425,8 +396,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
396 batch_size = 1
397 vocab_size = 30_000
398 torch.set_default_device(device)
428 - typical_acceptance_sampler = get_acceptance_sampler(
429 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
399 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
400 typical_acceptance_sampler.init_gpu_tensors(device=device)
401 # Simulate temperature 0 probability distribution for target
402 # probabilities and create target probabilities such that only 1 token
@@ -457,10 +427,7 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
427 # now accept even draft tokens with very low probability in the
428 # target distribution. Simulate and verify the same.
429 typical_acceptance_sampler = TypicalAcceptanceSampler(
460 - strict_mode=True,
461 - disable_bonus_tokens=disable_bonus_tokens,
462 - posterior_threshold=0.0,
463 - posterior_alpha=0.0)
430 + strict_mode=True, posterior_threshold=0.0, posterior_alpha=0.0)
431 typical_acceptance_sampler.init_gpu_tensors(device=device)
432 output_token_ids = typical_acceptance_sampler(
433 target_probs,
@@ -470,18 +437,13 @@ def test_accept_tokens_set_non_default_posteriors(seed: int,
437 assert output_token_ids.shape[0] == batch_size
438 assert output_token_ids.shape[1] == (k + 1)
439 assert torch.all(output_token_ids[:, 0:-1] == draft_token_ids)
473 - if disable_bonus_tokens:
474 - assert torch.all(output_token_ids[:, -1] == -1)
475 - else:
476 - assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
440 + assert torch.all(output_token_ids[:, -1] == bonus_token_ids)
441
442
443 @pytest.mark.parametrize("seed", list(range(10)))
480 -@pytest.mark.parametrize("disable_bonus_tokens", [True, False])
444 @pytest.mark.parametrize("device", CUDA_DEVICES)
445 @torch.inference_mode()
483 -def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
484 - device: str):
446 +def test_replacement_token_ids(seed: int, device: str):
447 """
448 Test the TypicalAcceptanceSampler's method for generating
449 replacement token IDs.
@@ -497,8 +459,7 @@ def test_replacement_token_ids(seed: int, disable_bonus_tokens: bool,
459 batch_size = 5
460 vocab_size = 30_000
461 torch.set_default_device(device)
500 - typical_acceptance_sampler = get_acceptance_sampler(
501 - strict_mode=True, disable_bonus_tokens=disable_bonus_tokens)
462 + typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
463 typical_acceptance_sampler.init_gpu_tensors(device=device)
464 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
465 expected_replacement_tokens = -torch.ones(
tests/spec_decode/e2e/test_medusa_correctness.py
+1 -1
@@ -31,7 +31,7 @@ MAIN_MODEL = "JackFram/llama-68m"
31 # speculative model
32 SPEC_MODEL = "abhigoyal/vllm-medusa-llama-68m-random"
33
34 -# max. number of speculative tokens: this corresponds to
34 +# max number of speculative tokens: this corresponds to
35 # num_heads in the config.json of the speculator model.
36 MAX_SPEC_TOKENS = 5
37
vllm/model_executor/layers/rejection_sampler.py
+1 -8
@@ -31,15 +31,11 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
31 """
32
33 def __init__(self,
34 - disable_bonus_tokens: bool = True,
34 strict_mode: bool = False,
35 use_flashinfer: Optional[bool] = None):
36 """Create a rejection sampler.
37
38 Args:
40 - disable_bonus_tokens: Whether or not to disable the bonus token.
41 - Require when bonus tokens will cause corrupt KV cache for
42 - proposal methods that require KV cache.
39 strict_mode: Whether or not to perform shape/device/dtype checks
40 during sampling. This catches correctness issues but adds
41 nontrivial latency.
@@ -48,8 +44,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
44 None, we will use the default value from the environment variable.
45 This parameter is only used for testing purposes.
46 """
51 - super().__init__(disable_bonus_tokens=disable_bonus_tokens,
52 - strict_mode=strict_mode)
47 + super().__init__(strict_mode=strict_mode)
48 if use_flashinfer is None:
49 self.use_flashinfer = envs.VLLM_USE_FLASHINFER_SAMPLER and (
50 chain_speculative_sampling is not None)
@@ -57,8 +52,6 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
52 self.use_flashinfer = use_flashinfer
53
54 if self.use_flashinfer:
60 - assert not disable_bonus_tokens, \
61 - "flashinfer will enable bonus token by default"
55 logger.info("Use flashinfer for rejection sampling.")
56 else:
57 logger.info("Use pytorch for rejection sampling.")
vllm/model_executor/layers/spec_decode_base_sampler.py
+1 -14
@@ -11,20 +11,14 @@ class SpecDecodeBaseSampler(nn.Module):
11 step.
12 """
13
14 - def __init__(self,
15 - disable_bonus_tokens: bool = True,
16 - strict_mode: bool = False):
14 + def __init__(self, strict_mode: bool = False):
15 """Base class constructor.
16 Args:
19 - disable_bonus_tokens: Whether or not to disable the bonus token.
20 - Require when bonus tokens will cause corrupt KV cache for
21 - proposal methods that require KV cache.
17 strict_mode: Whether or not to perform shape/device/dtype checks
18 during sampling. This catches correctness issues but adds
19 nontrivial latency.
20 """
21 super().__init__()
27 - self._disable_bonus_tokens = disable_bonus_tokens
22 self._strict_mode = strict_mode
23
24 # NOTE: A "bonus token" is accepted iff all proposal tokens are
@@ -111,13 +105,6 @@ class SpecDecodeBaseSampler(nn.Module):
105 output_with_bonus_tokens[:, -1] = torch.where(output[:, -1] != -1,
106 bonus_token_ids, -1)
107
114 - # We disable bonus tokens because it causes corrupt KV cache for
115 - # proposal methods that require KV cache. We can fix it by "prefilling"
116 - # the bonus token in the proposer. The following issue tracks the fix.
117 - # https://github.com/vllm-project/vllm/issues/4212
118 - if self._disable_bonus_tokens:
119 - output_with_bonus_tokens[:, -1] = -1
120 -
108 # Fill the recovered token ids.
109 output.mul_(~after_false_mask).add_(
110 substitute_token_ids.mul(after_false_mask))
vllm/model_executor/layers/typical_acceptance_sampler.py
+2 -7
@@ -16,15 +16,11 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
16 self,
17 posterior_threshold: float,
18 posterior_alpha: float,
19 - disable_bonus_tokens: bool = False,
19 strict_mode: bool = False,
20 ):
21 """Create a Typical Acceptance Sampler.
22
23 Args:
25 - disable_bonus_tokens: Whether or not to disable the bonus token.
26 - Require when bonus tokens will cause corrupt KV cache for
27 - proposal methods that require KV cache.
24 strict_mode: Whether or not to perform shape/device/dtype checks
25 during sampling. This catches correctness issues but adds
26 nontrivial latency.
@@ -36,8 +32,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
32 """
33 self._posterior_threshold = posterior_threshold
34 self._posterior_alpha = posterior_alpha
39 - super().__init__(disable_bonus_tokens=disable_bonus_tokens,
40 - strict_mode=strict_mode)
35 + super().__init__(strict_mode=strict_mode)
36
37 def forward(
38 self,
@@ -54,7 +49,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
49 one token will be emitted.
50
51 In the case where all draft tokens are accepted, the bonus token will be
57 - accepted conditioned on self._disable_bonus_tokens being false.
52 + accepted.
53
54 Args:
55 target_probs: The probability distribution over token ids given
vllm/spec_decode/spec_decode_worker.py
+1 -3
@@ -164,11 +164,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
164
165 spec_decode_sampler: SpecDecodeBaseSampler = None
166 if draft_token_acceptance_method == "rejection_sampler":
167 - spec_decode_sampler = RejectionSampler(
168 - disable_bonus_tokens=False, )
167 + spec_decode_sampler = RejectionSampler()
168 elif draft_token_acceptance_method == "typical_acceptance_sampler":
169 spec_decode_sampler = TypicalAcceptanceSampler(
171 - disable_bonus_tokens=False,
170 posterior_threshold=\
171 typical_acceptance_sampler_posterior_threshold,
172 posterior_alpha=typical_acceptance_sampler_posterior_alpha,