@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f7bb58427
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Fix typical acceptance sampler with correct recovered token ids (#8562)

jiqing-feng committed Sep 24, 2024 at 03:32 UTC 5f7bb584272ee15147a411b887e7ababd6b9b9d0
2 files changed +17 -28
tests/samplers/test_typical_acceptance_sampler.py
+8 -9
@@ -365,7 +365,7 @@ def test_accept_tokens_partially(seed: int, device: str):
365 # Next only keep the first 2 draft tokens same as the zero temperature
366 # tokens. For the remaining 3 choose some other tokens. In the
367 # response we will expect the first 2 tokens to be the same as the
368 - # draft tokens and the rest as -1
368 + # draft tokens and the recovered token and rest as -1
369 draft_token_ids_to_replace = get_draft_token_ids(
370 batch_size, k, vocab_size, zero_temperature_token_ids)
371 draft_token_ids = torch.cat(
@@ -378,6 +378,8 @@ def test_accept_tokens_partially(seed: int, device: str):
378 assert output_token_ids.shape[0] == batch_size
379 assert output_token_ids.shape[1] == (k + 1)
380 assert torch.all(output_token_ids[:, :2] == draft_token_ids[:, :2])
381 + assert torch.all(
382 + output_token_ids[:, 2] == target_with_bonus_probs.argmax(-1)[:, 2])
383 assert torch.all(output_token_ids[:, -3:] == -1)
384
385
@@ -443,14 +445,14 @@ def test_accept_tokens_set_non_default_posteriors(seed: int, device: str):
445 @pytest.mark.parametrize("seed", list(range(10)))
446 @pytest.mark.parametrize("device", CUDA_DEVICES)
447 @torch.inference_mode()
446 -def test_replacement_token_ids(seed: int, device: str):
448 +def test_get_recovered_token_ids(seed: int, device: str):
449 """
450 Test the TypicalAcceptanceSampler's method for generating
451 replacement token IDs.
452
451 - This test verifies that the `_replacement_token_ids` method of the
453 + This test verifies that the `_get_recovered_token_ids` method of the
454 TypicalAcceptanceSampler correctly identifies the token IDs to be used
453 - as replacements based on the target probability distribution.
455 + as recovered token IDs based on the target probability distribution.
456 Specifically, it ensures that the method correctly identifies the
457 tokens with the highest probability for each sequence in the batch.
458 """
@@ -462,10 +464,7 @@ def test_replacement_token_ids(seed: int, device: str):
464 typical_acceptance_sampler = get_acceptance_sampler(strict_mode=True)
465 typical_acceptance_sampler.init_gpu_tensors(device=device)
466 target_probs = torch.rand(batch_size, k, vocab_size, dtype=torch.float32)
465 - expected_replacement_tokens = -torch.ones(
466 - (batch_size, k), dtype=torch.long)
467 - expected_replacement_tokens[:, 0] = torch.argmax(target_probs[:, 0, :],
468 - dim=1)
467 + expected_replacement_tokens = torch.argmax(target_probs, dim=-1)
468 actual_replacement_tokens = (
470 - typical_acceptance_sampler._replacement_token_ids(target_probs))
469 + typical_acceptance_sampler._get_recovered_token_ids(target_probs))
470 assert torch.all(expected_replacement_tokens == actual_replacement_tokens)
vllm/model_executor/layers/typical_acceptance_sampler.py
+9 -19
@@ -80,7 +80,7 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
80 target_probs = target_with_bonus_probs[:, :-1]
81 accepted = self._evaluate_accepted_tokens(target_probs,
82 draft_token_ids)
83 - recovered_token_ids = self._replacement_token_ids(target_probs)
83 + recovered_token_ids = self._get_recovered_token_ids(target_probs)
84 output_token_ids = self._create_output(accepted, recovered_token_ids,
85 draft_token_ids,
86 bonus_token_ids)
@@ -148,16 +148,10 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
148 accepted_mask = candidates_prob > threshold
149 return accepted_mask
150
151 - def _replacement_token_ids(self, target_probs):
151 + def _get_recovered_token_ids(self, target_probs):
152 """
153 - Generate one replacement token ID for each sequence based on target
154 - probabilities. The replacement token is used as the fallback option
155 - if typical acceptance sampling does not accept any draft tokens for
156 - that particular sequence.
157 -
158 - This method computes the token IDs to be replaced by selecting the
159 - token with the highest probability for each sequence in the first
160 - position. The rest of the output is filled with -1.
153 + The recovered token ids will fill the first unmatched token
154 + by the target token.
155
156 Parameters
157 ----------
@@ -168,13 +162,9 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
162 Returns
163 -------
164 torch.Tensor
171 - A tensor of shape (batch_size, k) with the replacement
172 - token IDs. Only the first column is set, and the rest of the
173 - columns are filled with -1.
165 + A tensor of shape (batch_size, k) with the recovered token
166 + ids which are selected from target probs.
167 """
175 - max_indices = torch.argmax(target_probs[:, 0, :], dim=1)
176 - output = -torch.ones((target_probs.shape[0], target_probs.shape[1]),
177 - dtype=self.token_id_dtype,
178 - device=target_probs.device)
179 - output[:, 0] = max_indices
180 - return output
168 + max_indices = torch.argmax(target_probs, dim=-1)
169 +
170 + return max_indices