A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix speculative decoding with MLPSpeculator with padded vocabulary (#7218)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Aug 8, 2024 at 23:08 UTC
99b4cf5f23457b8c21e7e1b4d5d4384de867b521
4 files changed
+66
-5
tests/spec_decode/e2e/test_mlp_correctness.py
+60
@@ -19,8 +19,12 @@ With those tests, we can say at least, MLPSpeculator would not break the
19
correctess for the target model outputs.
20
"""
21
22
+from unittest.mock import patch
23
+
24
import pytest
25
26
+from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
27
+
28
from .conftest import (run_equality_correctness_test,
29
run_greedy_equality_correctness_test)
30
@@ -178,6 +182,62 @@ def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
182
force_output_len=True)
183
184
185
+@pytest.mark.parametrize(
186
+ "common_llm_kwargs",
187
+ [{
188
+ "block_size": 8,
189
+ # 2 for small prompt, 256//8 for generated.
190
+ "num_gpu_blocks_override": 2 + 256 // 8,
191
+ "max_model_len": (2 + 256 // 8) * 8,
192
+
193
+ # Skip cuda graph recording for fast test.
194
+ "enforce_eager": True,
195
+
196
+ # Required for spec decode.
197
+ "use_v2_block_manager": True,
198
+
199
+ # Precision
200
+ "dtype": PRECISION,
201
+
202
+ # Main model
203
+ "model": MAIN_MODEL,
204
+ }])
205
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
206
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
207
+@pytest.mark.parametrize("test_llm_kwargs", [
208
+ {
209
+ "speculative_model": SPEC_MODEL,
210
+ },
211
+])
212
+@pytest.mark.parametrize(
213
+ "output_len",
214
+ [
215
+ # Use small output len for fast test.
216
+ 128,
217
+ ])
218
+@pytest.mark.parametrize("batch_size", [4])
219
+@pytest.mark.parametrize("seed", [1])
220
+def test_mlp_e2e_greedy_correctness_with_padding(baseline_llm_generator,
221
+ test_llm_generator,
222
+ batch_size: int,
223
+ output_len: int):
224
+ """Verify greedy equality when the vocab dimension is padded
225
+ """
226
+
227
+ # Default pad_to is 64, test model has vocab_size of 32000
228
+ def patched_pad_vocab_size(vocab_size, pad_to=None):
229
+ return pad_vocab_size(vocab_size, pad_to=32064)
230
+
231
+ with patch(
232
+ "vllm.model_executor.layers.vocab_parallel_embedding.pad_vocab_size",
233
+ patched_pad_vocab_size):
234
+ run_greedy_equality_correctness_test(baseline_llm_generator,
235
+ test_llm_generator,
236
+ batch_size,
237
+ max_output_len=output_len,
238
+ force_output_len=True)
239
+
240
+
241
@pytest.mark.parametrize(
242
"common_llm_kwargs",
243
[{
vllm/model_executor/layers/logits_processor.py
+1
-1
@@ -91,7 +91,7 @@ class LogitsProcessor(nn.Module):
91
logits = tensor_model_parallel_all_gather(logits)
92
# Remove paddings in vocab (if any).
93
if logits is not None:
94
- logits = logits[:, :self.org_vocab_size]
94
+ logits = logits[..., :self.org_vocab_size]
95
return logits
96
97
def extra_repr(self) -> str:
vllm/model_executor/layers/rejection_sampler.py
+2
-2
@@ -78,8 +78,8 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
78
# Only perform shape/dtype/device checking in strict mode, as it adds
79
# overhead.
80
if self._strict_mode:
81
- self._raise_if_incorrect_input(target_probs, bonus_token_ids,
82
- draft_probs, draft_token_ids)
81
+ self._raise_if_incorrect_input(target_probs, draft_token_ids,
82
+ bonus_token_ids, draft_probs)
83
84
accepted, recovered_token_ids = (
85
self._batch_modified_rejection_sampling(
vllm/model_executor/models/mlp_speculator.py
+3
-2
@@ -175,13 +175,14 @@ class MLPSpeculator(nn.Module):
175
states.add_(z, alpha=self.emb_weight / self.state_weight)
176
177
states = self.activation(self.ln[head_index](states)) # b k d
178
- # TODO: not yet supporting top_k_tokens_per_head
178
previous_hidden_states = states
179
+ # TODO: not yet supporting top_k_tokens_per_head
180
+ states = states.flatten(0, 1)
181
182
logits = self.logits_processor(self.head[head_index], states,
183
sampling_metadata)
184
184
- output = self.sampler(logits.flatten(0, 1), sampling_metadata)
185
+ output = self.sampler(logits, sampling_metadata)
186
last_tokens = output.sampled_token_ids
187
next_tokens.append(output)
188