A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Speculative Decoding] Fixing hidden states handling in batch expansion (#7508)
Abhinav Goyal committed
Aug 20, 2024 at 06:28 UTC
312f7612328fb8db9fb2dd9f94f2ea021c035357
6 files changed
+139
-41
tests/spec_decode/e2e/conftest.py
+16
-9
@@ -288,15 +288,17 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
288
ensure_all_accepted=ensure_all_accepted)
289
290
291
-def run_equality_correctness_test(baseline_llm_generator,
292
- test_llm_generator,
293
- batch_size,
294
- max_output_len,
295
- force_output_len: bool,
296
- temperature: float,
297
- seeded: bool,
298
- print_tokens: bool = False,
299
- ensure_all_accepted: bool = False):
291
+def run_equality_correctness_test(
292
+ baseline_llm_generator,
293
+ test_llm_generator,
294
+ batch_size,
295
+ max_output_len,
296
+ force_output_len: bool,
297
+ temperature: float,
298
+ seeded: bool,
299
+ print_tokens: bool = False,
300
+ ensure_all_accepted: bool = False,
301
+ expected_acceptance_rate: Optional[float] = None):
302
"""Helper method that compares the outputs of both the baseline LLM and
303
the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
304
the same when temperature is zero (or when temperature is > 0 and seeded).
@@ -357,5 +359,10 @@ def run_equality_correctness_test(baseline_llm_generator,
359
print(f'{i=} {spec_token_ids=}')
360
assert baseline_token_ids == spec_token_ids
361
362
+ print(f'{acceptance_rate=}')
363
+
364
if ensure_all_accepted:
365
assert acceptance_rate == 1.0
366
+
367
+ if expected_acceptance_rate is not None:
368
+ assert acceptance_rate >= expected_acceptance_rate - 1e-2
tests/spec_decode/e2e/test_mlp_correctness.py
+42
@@ -82,6 +82,48 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
82
force_output_len=True)
83
84
85
+@pytest.mark.parametrize(
86
+ "common_llm_kwargs",
87
+ [{
88
+ # Skip cuda graph recording for fast test.
89
+ "enforce_eager": True,
90
+
91
+ # Required for spec decode.
92
+ "use_v2_block_manager": True,
93
+
94
+ # Print spec metrics.
95
+ "disable_log_stats": False,
96
+
97
+ # Precision
98
+ "dtype": PRECISION,
99
+
100
+ # Main model
101
+ "model": MAIN_MODEL,
102
+ }])
103
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
104
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
105
+@pytest.mark.parametrize("test_llm_kwargs", [
106
+ {
107
+ "speculative_model": SPEC_MODEL,
108
+ },
109
+])
110
+@pytest.mark.parametrize("output_len", [2048])
111
+@pytest.mark.parametrize("batch_size", [1, 32])
112
+@pytest.mark.parametrize("seed", [1])
113
+def test_mlp_e2e_acceptance_rate(baseline_llm_generator, test_llm_generator,
114
+ batch_size: int, output_len: int):
115
+ """Verify acceptance rate with different batch size and large output
116
+ length."""
117
+ run_equality_correctness_test(baseline_llm_generator,
118
+ test_llm_generator,
119
+ batch_size,
120
+ max_output_len=output_len,
121
+ temperature=0.0,
122
+ seeded=True,
123
+ force_output_len=True,
124
+ expected_acceptance_rate=0.48)
125
+
126
+
127
@pytest.mark.parametrize(
128
"common_llm_kwargs",
129
[{
vllm/spec_decode/batch_expansion.py
+60
-26
@@ -1,6 +1,6 @@
1
from array import array
2
from itertools import chain, count
3
-from typing import Iterator, List, Tuple
3
+from typing import Iterator, List, Optional, Tuple
4
5
import torch
6
@@ -88,21 +88,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
88
assert len(target_sampler_output) == 1, "expected single-step output"
89
target_sampler_output = target_sampler_output[0]
90
91
- all_tokens, all_probs, spec_logprobs = self._contract_batch(
92
- contracted_bs=len(execute_model_req.seq_group_metadata_list),
93
- target_sampler_output=target_sampler_output,
94
- proposals=proposals,
95
- num_scoring_tokens=num_scoring_tokens,
96
- non_spec_indices=non_spec_indices,
97
- spec_indices=spec_indices,
98
- k=execute_model_req.num_lookahead_slots,
99
- )
91
+ (all_tokens, all_probs, spec_logprobs,
92
+ all_hidden_states) = self._contract_batch(
93
+ contracted_bs=len(execute_model_req.seq_group_metadata_list),
94
+ target_sampler_output=target_sampler_output,
95
+ proposals=proposals,
96
+ num_scoring_tokens=num_scoring_tokens,
97
+ non_spec_indices=non_spec_indices,
98
+ spec_indices=spec_indices,
99
+ k=execute_model_req.num_lookahead_slots,
100
+ )
101
102
return SpeculativeScores(
103
probs=all_probs,
104
token_ids=all_tokens,
105
logprobs=spec_logprobs,
105
- hidden_states=target_sampler_output.hidden_states,
106
+ hidden_states=all_hidden_states,
107
)
108
109
def _expand_batch(
@@ -145,10 +146,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
146
num_scoring_tokens)
147
148
def _contract_batch(
148
- self, contracted_bs: int, target_sampler_output: SamplerOutput,
149
- proposals: SpeculativeProposals, num_scoring_tokens: int,
150
- non_spec_indices: List[int], spec_indices: List[int],
151
- k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
149
+ self, contracted_bs: int, target_sampler_output: SamplerOutput,
150
+ proposals: SpeculativeProposals, num_scoring_tokens: int,
151
+ non_spec_indices: List[int], spec_indices: List[int], k: int
152
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
153
+ Optional[torch.Tensor]]:
154
"""Contract the expanded batch back into its original size.
155
This maps the scores of speculative tokens back to their original
156
sequences.
@@ -156,9 +158,10 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
158
contracted_bs is the original batch size, and the batch size that the
159
target_sampler_output will be contracted to.
160
"""
159
- (target_token_ids, target_probs, target_logprobs,
161
+ (target_token_ids, target_probs, target_logprobs, target_hidden_states,
162
non_spec_target_token_ids, non_spec_target_probs,
161
- non_spec_target_logprobs) = self._split_scoring_output(
163
+ non_spec_target_logprobs,
164
+ non_spec_target_hidden_states) = self._split_scoring_output(
165
target_sampler_output, num_scoring_tokens)
166
167
# Map distinct sequences used to score each token
@@ -176,23 +179,40 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
179
self._vocab_size)
180
target_logprobs = target_logprobs.reshape(target_probs.shape)
181
182
+ if target_hidden_states is not None:
183
+ target_hidden_states = target_hidden_states.reshape(
184
+ spec_expanded_bs, k + 1, target_hidden_states.shape[-1])
185
+
186
all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
187
fill_value=-1)
188
all_probs = target_probs.new_zeros(*all_tokens.shape, self._vocab_size)
189
all_logprobs = target_logprobs.new_full(size=all_probs.shape,
190
fill_value=-float("inf"))
191
192
+ if target_sampler_output.hidden_states is not None:
193
+ all_hidden_states = target_hidden_states.new_zeros(
194
+ size=(contracted_bs, k + 1, target_hidden_states.shape[-1]))
195
+ else:
196
+ all_hidden_states = None
197
+
198
if non_spec_indices:
199
all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
200
all_probs[non_spec_indices, :1, :] = non_spec_target_probs
201
all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
202
203
+ if all_hidden_states is not None:
204
+ all_hidden_states[
205
+ non_spec_indices, :1, :] = non_spec_target_hidden_states
206
+
207
if spec_indices:
208
all_tokens[spec_indices] = target_token_ids
209
all_probs[spec_indices] = target_probs
210
all_logprobs[spec_indices] = target_logprobs
211
195
- return all_tokens, all_probs, all_logprobs
212
+ if all_hidden_states is not None:
213
+ all_hidden_states[spec_indices] = target_hidden_states
214
+
215
+ return all_tokens, all_probs, all_logprobs, all_hidden_states
216
217
def _create_scoring_model_input(
218
self,
@@ -327,8 +347,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
347
348
def _split_scoring_output(
349
self, sampler_output: SamplerOutput, num_scoring_tokens: int
330
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
331
- torch.Tensor, torch.Tensor]:
350
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
351
+ Optional[torch.Tensor], torch.Tensor, torch.Tensor,
352
+ torch.Tensor, Optional[torch.Tensor]]:
353
"""Split the target model output into speculative and non-speculative
354
output.
355
"""
@@ -353,24 +374,37 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
374
non_spec_logprobs,
375
) = sampler_output.logprobs.split(split_sizes)
376
377
+ if sampler_output.hidden_states is not None:
378
+ (
379
+ spec_hidden_states,
380
+ non_spec_hidden_states,
381
+ ) = sampler_output.hidden_states.split(split_sizes)
382
+ else:
383
+ spec_hidden_states, non_spec_hidden_states = None, None
384
+
385
# Convert scores to tensors.
386
sampler_output.sampled_token_probs = spec_probs
387
sampler_output.sampled_token_ids = spec_sampled_tokens
388
sampler_output.logprobs = spec_logprobs
360
- (target_token_ids, target_probs,
361
- target_logprobs) = sampler_output_to_torch([sampler_output], True)
389
+ sampler_output.hidden_states = spec_hidden_states
390
+ (target_token_ids, target_probs, target_logprobs,
391
+ target_hidden_states) = sampler_output_to_torch([sampler_output],
392
+ True)
393
394
# Convert non-speculative output tokens to tensors.
395
sampler_output.sampled_token_probs = non_spec_probs
396
sampler_output.sampled_token_ids = non_spec_sampled_tokens
397
sampler_output.logprobs = non_spec_logprobs
398
+ sampler_output.hidden_states = non_spec_hidden_states
399
(non_spec_target_token_ids, non_spec_target_probs,
368
- non_spec_target_logprobs) = sampler_output_to_torch([sampler_output],
369
- True)
400
+ non_spec_target_logprobs,
401
+ non_spec_target_hidden_states) = sampler_output_to_torch(
402
+ [sampler_output], True)
403
404
return (target_token_ids, target_probs, target_logprobs,
372
- non_spec_target_token_ids, non_spec_target_probs,
373
- non_spec_target_logprobs)
405
+ target_hidden_states, non_spec_target_token_ids,
406
+ non_spec_target_probs, non_spec_target_logprobs,
407
+ non_spec_target_hidden_states)
408
409
def _create_target_seq_id_iterator(
410
self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/spec_decode_worker.py
+2
-3
@@ -646,9 +646,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
646
hidden_states = proposal_scores.hidden_states
647
if hidden_states is not None:
648
# Contract hidden states based on accepted tokens
649
- hs_size = hidden_states.shape[1]
650
- hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
651
- hs_size)
649
+ hs_size = hidden_states.shape[-1]
650
+
651
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
652
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
653
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
vllm/spec_decode/top1_proposer.py
+1
-1
@@ -242,7 +242,7 @@ class Top1Proposer(SpeculativeProposer):
242
return proposal_tokens, proposal_probs, proposal_lens_tensor
243
244
sampler_output = maybe_sampler_output
245
- proposal_tokens, proposal_probs, _ = sampler_output_to_torch(
245
+ proposal_tokens, proposal_probs, *_ = sampler_output_to_torch(
246
sampler_output, sampler_transposed)
247
248
# Now, reformat the output GPU tensors such that each sequence has
vllm/spec_decode/util.py
+18
-2
@@ -123,7 +123,7 @@ def split_batch_by_proposal_len(
123
124
def sampler_output_to_torch(
125
sampler_output_list: List[SamplerOutput], sampler_transposed: bool
126
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
126
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
127
"""Utility function which converts a list of SamplerOutput to tensors.
128
129
sampler_transposed here is used as the indicator for whether
@@ -169,7 +169,23 @@ def sampler_output_to_torch(
169
if sampler_transposed:
170
sampled_token_ids = sampled_token_ids.transpose(0, 1)
171
172
- return sampled_token_ids, sampled_token_probs, sampled_token_logprobs
172
+ if sampler_output_list[0].hidden_states is not None:
173
+ # shape: [batch_size, num_sampler_output, hidden_dim]
174
+ sampled_hidden_states = torch.stack(
175
+ [
176
+ sampler_output.hidden_states
177
+ for sampler_output in sampler_output_list
178
+ ],
179
+ dim=0,
180
+ )
181
+
182
+ if sampler_transposed:
183
+ sampled_hidden_states = sampled_hidden_states.transpose(0, 1)
184
+ else:
185
+ sampled_hidden_states = None
186
+
187
+ return (sampled_token_ids, sampled_token_probs, sampled_token_logprobs,
188
+ sampled_hidden_states)
189
190
191
def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,