@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 312f761232
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Speculative Decoding] Fixing hidden states handling in batch expansion (#7508)

Abhinav Goyal committed Aug 20, 2024 at 06:28 UTC 312f7612328fb8db9fb2dd9f94f2ea021c035357
6 files changed +139 -41
tests/spec_decode/e2e/conftest.py
+16 -9
@@ -288,15 +288,17 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
288 ensure_all_accepted=ensure_all_accepted)
289
290
291 -def run_equality_correctness_test(baseline_llm_generator,
292 - test_llm_generator,
293 - batch_size,
294 - max_output_len,
295 - force_output_len: bool,
296 - temperature: float,
297 - seeded: bool,
298 - print_tokens: bool = False,
299 - ensure_all_accepted: bool = False):
291 +def run_equality_correctness_test(
292 + baseline_llm_generator,
293 + test_llm_generator,
294 + batch_size,
295 + max_output_len,
296 + force_output_len: bool,
297 + temperature: float,
298 + seeded: bool,
299 + print_tokens: bool = False,
300 + ensure_all_accepted: bool = False,
301 + expected_acceptance_rate: Optional[float] = None):
302 """Helper method that compares the outputs of both the baseline LLM and
303 the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
304 the same when temperature is zero (or when temperature is > 0 and seeded).
@@ -357,5 +359,10 @@ def run_equality_correctness_test(baseline_llm_generator,
359 print(f'{i=} {spec_token_ids=}')
360 assert baseline_token_ids == spec_token_ids
361
362 + print(f'{acceptance_rate=}')
363 +
364 if ensure_all_accepted:
365 assert acceptance_rate == 1.0
366 +
367 + if expected_acceptance_rate is not None:
368 + assert acceptance_rate >= expected_acceptance_rate - 1e-2
tests/spec_decode/e2e/test_mlp_correctness.py
+42
@@ -82,6 +82,48 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
82 force_output_len=True)
83
84
85 +@pytest.mark.parametrize(
86 + "common_llm_kwargs",
87 + [{
88 + # Skip cuda graph recording for fast test.
89 + "enforce_eager": True,
90 +
91 + # Required for spec decode.
92 + "use_v2_block_manager": True,
93 +
94 + # Print spec metrics.
95 + "disable_log_stats": False,
96 +
97 + # Precision
98 + "dtype": PRECISION,
99 +
100 + # Main model
101 + "model": MAIN_MODEL,
102 + }])
103 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
104 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
105 +@pytest.mark.parametrize("test_llm_kwargs", [
106 + {
107 + "speculative_model": SPEC_MODEL,
108 + },
109 +])
110 +@pytest.mark.parametrize("output_len", [2048])
111 +@pytest.mark.parametrize("batch_size", [1, 32])
112 +@pytest.mark.parametrize("seed", [1])
113 +def test_mlp_e2e_acceptance_rate(baseline_llm_generator, test_llm_generator,
114 + batch_size: int, output_len: int):
115 + """Verify acceptance rate with different batch size and large output
116 + length."""
117 + run_equality_correctness_test(baseline_llm_generator,
118 + test_llm_generator,
119 + batch_size,
120 + max_output_len=output_len,
121 + temperature=0.0,
122 + seeded=True,
123 + force_output_len=True,
124 + expected_acceptance_rate=0.48)
125 +
126 +
127 @pytest.mark.parametrize(
128 "common_llm_kwargs",
129 [{
vllm/spec_decode/batch_expansion.py
+60 -26
@@ -1,6 +1,6 @@
1 from array import array
2 from itertools import chain, count
3 -from typing import Iterator, List, Tuple
3 +from typing import Iterator, List, Optional, Tuple
4
5 import torch
6
@@ -88,21 +88,22 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
88 assert len(target_sampler_output) == 1, "expected single-step output"
89 target_sampler_output = target_sampler_output[0]
90
91 - all_tokens, all_probs, spec_logprobs = self._contract_batch(
92 - contracted_bs=len(execute_model_req.seq_group_metadata_list),
93 - target_sampler_output=target_sampler_output,
94 - proposals=proposals,
95 - num_scoring_tokens=num_scoring_tokens,
96 - non_spec_indices=non_spec_indices,
97 - spec_indices=spec_indices,
98 - k=execute_model_req.num_lookahead_slots,
99 - )
91 + (all_tokens, all_probs, spec_logprobs,
92 + all_hidden_states) = self._contract_batch(
93 + contracted_bs=len(execute_model_req.seq_group_metadata_list),
94 + target_sampler_output=target_sampler_output,
95 + proposals=proposals,
96 + num_scoring_tokens=num_scoring_tokens,
97 + non_spec_indices=non_spec_indices,
98 + spec_indices=spec_indices,
99 + k=execute_model_req.num_lookahead_slots,
100 + )
101
102 return SpeculativeScores(
103 probs=all_probs,
104 token_ids=all_tokens,
105 logprobs=spec_logprobs,
105 - hidden_states=target_sampler_output.hidden_states,
106 + hidden_states=all_hidden_states,
107 )
108
109 def _expand_batch(
@@ -145,10 +146,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
146 num_scoring_tokens)
147
148 def _contract_batch(
148 - self, contracted_bs: int, target_sampler_output: SamplerOutput,
149 - proposals: SpeculativeProposals, num_scoring_tokens: int,
150 - non_spec_indices: List[int], spec_indices: List[int],
151 - k: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
149 + self, contracted_bs: int, target_sampler_output: SamplerOutput,
150 + proposals: SpeculativeProposals, num_scoring_tokens: int,
151 + non_spec_indices: List[int], spec_indices: List[int], k: int
152 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
153 + Optional[torch.Tensor]]:
154 """Contract the expanded batch back into its original size.
155 This maps the scores of speculative tokens back to their original
156 sequences.
@@ -156,9 +158,10 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
158 contracted_bs is the original batch size, and the batch size that the
159 target_sampler_output will be contracted to.
160 """
159 - (target_token_ids, target_probs, target_logprobs,
161 + (target_token_ids, target_probs, target_logprobs, target_hidden_states,
162 non_spec_target_token_ids, non_spec_target_probs,
161 - non_spec_target_logprobs) = self._split_scoring_output(
163 + non_spec_target_logprobs,
164 + non_spec_target_hidden_states) = self._split_scoring_output(
165 target_sampler_output, num_scoring_tokens)
166
167 # Map distinct sequences used to score each token
@@ -176,23 +179,40 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
179 self._vocab_size)
180 target_logprobs = target_logprobs.reshape(target_probs.shape)
181
182 + if target_hidden_states is not None:
183 + target_hidden_states = target_hidden_states.reshape(
184 + spec_expanded_bs, k + 1, target_hidden_states.shape[-1])
185 +
186 all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
187 fill_value=-1)
188 all_probs = target_probs.new_zeros(*all_tokens.shape, self._vocab_size)
189 all_logprobs = target_logprobs.new_full(size=all_probs.shape,
190 fill_value=-float("inf"))
191
192 + if target_sampler_output.hidden_states is not None:
193 + all_hidden_states = target_hidden_states.new_zeros(
194 + size=(contracted_bs, k + 1, target_hidden_states.shape[-1]))
195 + else:
196 + all_hidden_states = None
197 +
198 if non_spec_indices:
199 all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
200 all_probs[non_spec_indices, :1, :] = non_spec_target_probs
201 all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
202
203 + if all_hidden_states is not None:
204 + all_hidden_states[
205 + non_spec_indices, :1, :] = non_spec_target_hidden_states
206 +
207 if spec_indices:
208 all_tokens[spec_indices] = target_token_ids
209 all_probs[spec_indices] = target_probs
210 all_logprobs[spec_indices] = target_logprobs
211
195 - return all_tokens, all_probs, all_logprobs
212 + if all_hidden_states is not None:
213 + all_hidden_states[spec_indices] = target_hidden_states
214 +
215 + return all_tokens, all_probs, all_logprobs, all_hidden_states
216
217 def _create_scoring_model_input(
218 self,
@@ -327,8 +347,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
347
348 def _split_scoring_output(
349 self, sampler_output: SamplerOutput, num_scoring_tokens: int
330 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
331 - torch.Tensor, torch.Tensor]:
350 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
351 + Optional[torch.Tensor], torch.Tensor, torch.Tensor,
352 + torch.Tensor, Optional[torch.Tensor]]:
353 """Split the target model output into speculative and non-speculative
354 output.
355 """
@@ -353,24 +374,37 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
374 non_spec_logprobs,
375 ) = sampler_output.logprobs.split(split_sizes)
376
377 + if sampler_output.hidden_states is not None:
378 + (
379 + spec_hidden_states,
380 + non_spec_hidden_states,
381 + ) = sampler_output.hidden_states.split(split_sizes)
382 + else:
383 + spec_hidden_states, non_spec_hidden_states = None, None
384 +
385 # Convert scores to tensors.
386 sampler_output.sampled_token_probs = spec_probs
387 sampler_output.sampled_token_ids = spec_sampled_tokens
388 sampler_output.logprobs = spec_logprobs
360 - (target_token_ids, target_probs,
361 - target_logprobs) = sampler_output_to_torch([sampler_output], True)
389 + sampler_output.hidden_states = spec_hidden_states
390 + (target_token_ids, target_probs, target_logprobs,
391 + target_hidden_states) = sampler_output_to_torch([sampler_output],
392 + True)
393
394 # Convert non-speculative output tokens to tensors.
395 sampler_output.sampled_token_probs = non_spec_probs
396 sampler_output.sampled_token_ids = non_spec_sampled_tokens
397 sampler_output.logprobs = non_spec_logprobs
398 + sampler_output.hidden_states = non_spec_hidden_states
399 (non_spec_target_token_ids, non_spec_target_probs,
368 - non_spec_target_logprobs) = sampler_output_to_torch([sampler_output],
369 - True)
400 + non_spec_target_logprobs,
401 + non_spec_target_hidden_states) = sampler_output_to_torch(
402 + [sampler_output], True)
403
404 return (target_token_ids, target_probs, target_logprobs,
372 - non_spec_target_token_ids, non_spec_target_probs,
373 - non_spec_target_logprobs)
405 + target_hidden_states, non_spec_target_token_ids,
406 + non_spec_target_probs, non_spec_target_logprobs,
407 + non_spec_target_hidden_states)
408
409 def _create_target_seq_id_iterator(
410 self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/spec_decode_worker.py
+2 -3
@@ -646,9 +646,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
646 hidden_states = proposal_scores.hidden_states
647 if hidden_states is not None:
648 # Contract hidden states based on accepted tokens
649 - hs_size = hidden_states.shape[1]
650 - hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
651 - hs_size)
649 + hs_size = hidden_states.shape[-1]
650 +
651 accepted_index = accepted_token_ids + 1 # Convert -1 to 0
652 accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
653 index = accepted_index[:, None, None].expand(-1, 1, hs_size)
vllm/spec_decode/top1_proposer.py
+1 -1
@@ -242,7 +242,7 @@ class Top1Proposer(SpeculativeProposer):
242 return proposal_tokens, proposal_probs, proposal_lens_tensor
243
244 sampler_output = maybe_sampler_output
245 - proposal_tokens, proposal_probs, _ = sampler_output_to_torch(
245 + proposal_tokens, proposal_probs, *_ = sampler_output_to_torch(
246 sampler_output, sampler_transposed)
247
248 # Now, reformat the output GPU tensors such that each sequence has
vllm/spec_decode/util.py
+18 -2
@@ -123,7 +123,7 @@ def split_batch_by_proposal_len(
123
124 def sampler_output_to_torch(
125 sampler_output_list: List[SamplerOutput], sampler_transposed: bool
126 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
126 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
127 """Utility function which converts a list of SamplerOutput to tensors.
128
129 sampler_transposed here is used as the indicator for whether
@@ -169,7 +169,23 @@ def sampler_output_to_torch(
169 if sampler_transposed:
170 sampled_token_ids = sampled_token_ids.transpose(0, 1)
171
172 - return sampled_token_ids, sampled_token_probs, sampled_token_logprobs
172 + if sampler_output_list[0].hidden_states is not None:
173 + # shape: [batch_size, num_sampler_output, hidden_dim]
174 + sampled_hidden_states = torch.stack(
175 + [
176 + sampler_output.hidden_states
177 + for sampler_output in sampler_output_list
178 + ],
179 + dim=0,
180 + )
181 +
182 + if sampler_transposed:
183 + sampled_hidden_states = sampled_hidden_states.transpose(0, 1)
184 + else:
185 + sampled_hidden_states = None
186 +
187 + return (sampled_token_ids, sampled_token_probs, sampled_token_logprobs,
188 + sampled_hidden_states)
189
190
191 def maybe_mock_device_tensors(sampler_output: SamplerOutput, batch_size: int,