@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1856aff4d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Spec Decoding] Streamline batch expansion tensor manipulation (#7851)

Nick Hill committed Aug 25, 2024 at 15:45 UTC 1856aff4d66833b258ce64132413ab8a18cc18a6
5 files changed +118 -125
tests/spec_decode/test_utils.py
+13 -18
@@ -55,10 +55,9 @@ def fake_sequence_group_metadata():
55
56 def test_filter_zero_length_proposals(fake_sequence_group_metadata):
57 proposal_lens = [0, 1, 0]
58 - filtered_groups, indices = split_batch_by_proposal_len(
59 - fake_sequence_group_metadata,
60 - proposal_lens,
61 - select_proposal_len_zero=True)
58 + _, (filtered_groups,
59 + indices) = split_batch_by_proposal_len(fake_sequence_group_metadata,
60 + proposal_lens)
61
62 expected_groups = [
63 fake_sequence_group_metadata[0], fake_sequence_group_metadata[2]
@@ -71,10 +70,9 @@ def test_filter_zero_length_proposals(fake_sequence_group_metadata):
70
71 def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
72 proposal_lens = [0, 1, 2]
74 - filtered_groups, indices = split_batch_by_proposal_len(
75 - fake_sequence_group_metadata,
76 - proposal_lens,
77 - select_proposal_len_zero=False)
73 + (filtered_groups,
74 + indices), _ = split_batch_by_proposal_len(fake_sequence_group_metadata,
75 + proposal_lens)
76
77 expected_groups = [
78 fake_sequence_group_metadata[1], fake_sequence_group_metadata[2]
@@ -86,8 +84,7 @@ def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
84
85
86 def test_empty_inputs():
89 - filtered_groups, indices = split_batch_by_proposal_len(
90 - [], [], select_proposal_len_zero=True)
87 + _, (filtered_groups, indices) = split_batch_by_proposal_len([], [])
88
89 assert filtered_groups == []
90 assert indices == []
@@ -95,10 +92,9 @@ def test_empty_inputs():
92
93 def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
94 proposal_lens = [0, 0, 0]
98 - filtered_groups, indices = split_batch_by_proposal_len(
99 - fake_sequence_group_metadata,
100 - proposal_lens,
101 - select_proposal_len_zero=False)
95 + (filtered_groups,
96 + indices), _ = split_batch_by_proposal_len(fake_sequence_group_metadata,
97 + proposal_lens)
98
99 assert filtered_groups == []
100 assert indices == []
@@ -106,10 +102,9 @@ def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
102
103 def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
104 proposal_lens = [1, 1, 1]
109 - filtered_groups, indices = split_batch_by_proposal_len(
110 - fake_sequence_group_metadata,
111 - proposal_lens,
112 - select_proposal_len_zero=True)
105 + _, (filtered_groups,
106 + indices) = split_batch_by_proposal_len(fake_sequence_group_metadata,
107 + proposal_lens)
108
109 assert filtered_groups == []
110 assert indices == []
vllm/spec_decode/batch_expansion.py
+79 -64
@@ -10,8 +10,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
10 get_all_seq_ids)
11 from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 SpeculativeScorer, SpeculativeScores)
13 -from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
14 - split_batch_by_proposal_len)
13 +from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
14 from vllm.worker.worker_base import WorkerBase
15
16 SeqId = int
@@ -88,17 +87,25 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
87 assert len(target_sampler_output) == 1, "expected single-step output"
88 target_sampler_output = target_sampler_output[0]
89
91 - (all_tokens, all_probs, spec_logprobs,
92 - all_hidden_states) = self._contract_batch(
93 - contracted_bs=len(execute_model_req.seq_group_metadata_list),
94 - target_sampler_output=target_sampler_output,
95 - proposals=proposals,
96 - num_scoring_tokens=num_scoring_tokens,
97 - non_spec_indices=non_spec_indices,
98 - spec_indices=spec_indices,
99 - k=execute_model_req.num_lookahead_slots,
100 - )
101 -
90 + if not non_spec_indices:
91 + # All sequence groups in batch have spec decoding enabled
92 + contracted = self._contract_batch_all_spec(
93 + target_sampler_output=target_sampler_output,
94 + proposals=proposals,
95 + )
96 + else:
97 + # Batch has a mix of spec decode enabled and disabled seq groups
98 + contracted = self._contract_batch(
99 + contracted_bs=len(execute_model_req.seq_group_metadata_list),
100 + target_sampler_output=target_sampler_output,
101 + proposals=proposals,
102 + num_scoring_tokens=num_scoring_tokens,
103 + non_spec_indices=non_spec_indices,
104 + spec_indices=spec_indices,
105 + k=execute_model_req.num_lookahead_slots,
106 + )
107 +
108 + all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted
109 return SpeculativeScores(
110 probs=all_probs,
111 token_ids=all_tokens,
@@ -121,14 +128,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
128 # proposal len. This adds some complexity (splitting the batch into spec
129 # and non spec sequences) and should be removed in the future. It can be
130 # done by supporting per-sequence proposal lens.
124 - spec_seqs, spec_indices = split_batch_by_proposal_len(
125 - seq_group_metadata_list,
126 - proposal_lens_list,
127 - select_proposal_len_zero=False)
128 - non_spec_seqs, non_spec_indices = split_batch_by_proposal_len(
129 - seq_group_metadata_list,
130 - proposal_lens_list,
131 - select_proposal_len_zero=True)
131 + (spec_seqs, spec_indices), (non_spec_seqs, non_spec_indices) = \
132 + split_batch_by_proposal_len(
133 + seq_group_metadata_list, proposal_lens_list)
134
135 target_seq_group_metadata_list = self._create_scoring_model_input(
136 seq_group_metadata_list=spec_seqs,
@@ -171,7 +173,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
173 # The number of tokens in the expanded batch used for speculation is
174 # equal to the total expanded batch size minus the number of samples for
175 # non-speculative sequences.
174 - non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
176 + non_spec_expanded_bs = len(non_spec_target_token_ids)
177 spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
178
179 target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
@@ -181,7 +183,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
183
184 if target_hidden_states is not None:
185 target_hidden_states = target_hidden_states.reshape(
184 - spec_expanded_bs, k + 1, target_hidden_states.shape[-1])
186 + *target_token_ids.shape, target_hidden_states.shape[-1])
187
188 all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
189 fill_value=-1)
@@ -196,24 +198,58 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
198 all_hidden_states = None
199
200 if non_spec_indices:
199 - all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
200 - all_probs[non_spec_indices, :1, :] = non_spec_target_probs
201 - all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
202 -
201 + all_tokens[non_spec_indices, :1] = \
202 + non_spec_target_token_ids.unsqueeze(1)
203 + all_probs[non_spec_indices, :1, :] = \
204 + non_spec_target_probs.unsqueeze(1)
205 + all_logprobs[non_spec_indices, :1, :] = \
206 + non_spec_target_logprobs.unsqueeze(1)
207 if all_hidden_states is not None:
204 - all_hidden_states[
205 - non_spec_indices, :1, :] = non_spec_target_hidden_states
208 + assert non_spec_target_hidden_states is not None
209 + all_hidden_states[non_spec_indices, :1, :] = \
210 + non_spec_target_hidden_states.unsqueeze(1)
211
212 if spec_indices:
213 all_tokens[spec_indices] = target_token_ids
214 all_probs[spec_indices] = target_probs
215 all_logprobs[spec_indices] = target_logprobs
211 -
216 if all_hidden_states is not None:
217 all_hidden_states[spec_indices] = target_hidden_states
218
219 return all_tokens, all_probs, all_logprobs, all_hidden_states
220
221 + def _contract_batch_all_spec(
222 + self,
223 + target_sampler_output: SamplerOutput,
224 + proposals: SpeculativeProposals,
225 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
226 + Optional[torch.Tensor]]:
227 + """Contract the expanded batch back into its original size.
228 + This maps the scores of speculative tokens back to their original
229 + sequences.
230 +
231 + It assumes all sequences in the batch were previously expanded.
232 + """
233 +
234 + # Map distinct sequences used to score each token
235 + # of shape [batch_size * k + 1] back to [batch_size, k + 1].
236 + contracted_bs, k = proposals.proposal_token_ids.shape
237 +
238 + # Reshape tensors to original batch size
239 + target_token_ids = target_sampler_output.sampled_token_ids.reshape(
240 + contracted_bs, k + 1)
241 + target_probs = target_sampler_output.sampled_token_probs.reshape(
242 + *target_token_ids.shape, self._vocab_size)
243 + target_logprobs = target_sampler_output.logprobs.reshape(
244 + target_probs.shape)
245 + target_hidden_states = target_sampler_output.hidden_states
246 + if target_hidden_states is not None:
247 + target_hidden_states = target_hidden_states.reshape(
248 + *target_token_ids.shape, target_hidden_states.shape[-1])
249 +
250 + return (target_token_ids, target_probs, target_logprobs,
251 + target_hidden_states)
252 +
253 def _create_scoring_model_input(
254 self,
255 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -345,8 +381,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
381 token_chunk_size=1,
382 )
383
384 + @staticmethod
385 def _split_scoring_output(
349 - self, sampler_output: SamplerOutput, num_scoring_tokens: int
386 + sampler_output: SamplerOutput, num_scoring_tokens: int
387 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
388 Optional[torch.Tensor], torch.Tensor, torch.Tensor,
389 torch.Tensor, Optional[torch.Tensor]]:
@@ -361,10 +398,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
398 #
399 # First samples are from speculative scoring, latter samples are non-
400 # speculative samples.
364 - split_sizes = [
365 - num_scoring_tokens,
366 - sampler_output.sampled_token_ids.numel() - num_scoring_tokens
367 - ]
401 + split_sizes = (num_scoring_tokens,
402 + sampler_output.sampled_token_ids.numel() -
403 + num_scoring_tokens)
404 (spec_probs, non_spec_probs
405 ) = sampler_output.sampled_token_probs.split(split_sizes)
406 (spec_sampled_tokens, non_spec_sampled_tokens
@@ -382,32 +418,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
418 else:
419 spec_hidden_states, non_spec_hidden_states = None, None
420
385 - # Convert scores to tensors.
386 - sampler_output.sampled_token_probs = spec_probs
387 - sampler_output.sampled_token_ids = spec_sampled_tokens
388 - sampler_output.logprobs = spec_logprobs
389 - sampler_output.hidden_states = spec_hidden_states
390 - (target_token_ids, target_probs, target_logprobs,
391 - target_hidden_states) = sampler_output_to_torch([sampler_output],
392 - True)
393 -
394 - # Convert non-speculative output tokens to tensors.
395 - sampler_output.sampled_token_probs = non_spec_probs
396 - sampler_output.sampled_token_ids = non_spec_sampled_tokens
397 - sampler_output.logprobs = non_spec_logprobs
398 - sampler_output.hidden_states = non_spec_hidden_states
399 - (non_spec_target_token_ids, non_spec_target_probs,
400 - non_spec_target_logprobs,
401 - non_spec_target_hidden_states) = sampler_output_to_torch(
402 - [sampler_output], True)
403 -
404 - return (target_token_ids, target_probs, target_logprobs,
405 - target_hidden_states, non_spec_target_token_ids,
406 - non_spec_target_probs, non_spec_target_logprobs,
407 - non_spec_target_hidden_states)
421 + return (spec_sampled_tokens, spec_probs, spec_logprobs,
422 + spec_hidden_states, non_spec_sampled_tokens, non_spec_probs,
423 + non_spec_logprobs, non_spec_hidden_states)
424
425 + @staticmethod
426 def _create_target_seq_id_iterator(
410 - self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
427 + seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
428 """Create an iterator for creating target sequence ids.
429 Target sequence ids are distinct from sequence ids because we create a
430 distinct target sequence id for each proposal token to be scored.
@@ -417,8 +434,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
434 """
435 return count(start=max(seq_ids) + 1)
436
437 + @staticmethod
438 def _get_token_ids_to_score(
421 - self,
439 full_spec_token_ids: List[TokenId] # shape: [k]
440 ) -> List[List[TokenId]]:
441 """Given an int tensor of proposal token ids, return a list of
@@ -439,8 +456,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
456 empty_token_ids: List[TokenId] = []
457
458 token_ids_to_score = [empty_token_ids]
442 - token_ids_to_score.extend([
443 - full_spec_token_ids[:i + 1]
444 - for i in range(len(full_spec_token_ids))
445 - ])
459 + token_ids_to_score.extend(full_spec_token_ids[:i + 1]
460 + for i in range(len(full_spec_token_ids)))
461 return token_ids_to_score
vllm/spec_decode/spec_decode_worker.py
+9 -16
@@ -365,12 +365,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
365 # used during the prefill phase.
366 # 2. Auto-disable enabled: The running queue size exceeds
367 # the specified threshold.
368 - # 3. No request: There are no requests in the batch.
368 + # 3. No request: There are no requests in the batch, or
369 + # none of the requests in the batch have spec decoding enabled.
370 # In any of these cases, the proposer and scorer workers
371 # are called normally.
371 - no_spec = num_lookahead_slots == 0 or len(
372 - execute_model_req.seq_group_metadata_list
373 - ) == 0 or disable_all_speculation
372 + no_spec = num_lookahead_slots == 0 or disable_all_speculation or all(
373 + sgm.num_speculative_tokens == 0
374 + for sgm in execute_model_req.seq_group_metadata_list)
375
376 # Broadcast how many lookahead slots are scheduled for this step, and
377 # whether all speculation is disabled, to all non-driver workers.
@@ -415,10 +416,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
416 self, execute_model_req: ExecuteModelRequest) -> bool:
417 # When the batch size is too large, disable speculative decoding
418 # to stop trading off throughput for latency.
418 - disable_all_speculation = (execute_model_req.running_queue_size >=
419 - self.disable_by_batch_size)
420 -
421 - return disable_all_speculation
419 + return (execute_model_req.running_queue_size >=
420 + self.disable_by_batch_size)
421
422 def _maybe_disable_speculative_tokens(
423 self, disable_all_speculation: bool,
@@ -621,14 +620,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
620 # proposal len. This adds some complexity (splitting the batch into spec
621 # and non spec sequences) and should be removed in the future. It can be
622 # done by supporting per-sequence proposal lens.
624 - _, spec_indices = split_batch_by_proposal_len(
625 - seq_group_metadata_list,
626 - proposal_lens_list,
627 - select_proposal_len_zero=False)
628 - _, non_spec_indices = split_batch_by_proposal_len(
629 - seq_group_metadata_list,
630 - proposal_lens_list,
631 - select_proposal_len_zero=True)
623 + (_, spec_indices), (_, non_spec_indices) = split_batch_by_proposal_len(
624 + seq_group_metadata_list, proposal_lens_list)
625 original_indices = spec_indices + non_spec_indices
626
627 # Get probabilities of target model, excluding bonus token.
vllm/spec_decode/top1_proposer.py
+1 -1
@@ -138,7 +138,7 @@ class Top1Proposer(SpeculativeProposer):
138
139 # Currently only proposal lens of 0 or the global batch proposal len
140 # are supported.
141 - # If max_proposal_len is defined, then we shall no exceed this
141 + # If max_proposal_len is defined, then we shall not exceed this
142 # quota for nonzero_proposal
143 new_k = 0
144 if (self.max_proposal_len is None
vllm/spec_decode/util.py
+16 -26
@@ -1,6 +1,6 @@
1 import time
2 from contextlib import contextmanager
3 -from typing import Dict, List, Optional, Tuple
3 +from typing import Dict, List, Optional, Sequence, Tuple
4
5 import torch
6
@@ -98,33 +98,26 @@ def create_sequence_group_output(
98
99 def split_batch_by_proposal_len(
100 seq_group_metadata_list: List[SequenceGroupMetadata],
101 - proposal_lens: List[int], select_proposal_len_zero: bool
102 -) -> Tuple[List[SequenceGroupMetadata], List[int]]:
101 + proposal_lens: List[int],
102 +) -> Tuple[Tuple[List[SequenceGroupMetadata], List[int]], Tuple[
103 + List[SequenceGroupMetadata], List[int]]]:
104 """Utility function that splits a batch based on whether the proposal len is
105 zero or not. We should remove this once vLLM supports per-sequence proposal
106 lens in a batch.
107 """
108
108 - if select_proposal_len_zero:
109 - predicate = lambda proposal_len: proposal_len == 0
110 - else:
111 - predicate = lambda proposal_len: proposal_len != 0
112 -
113 - indices = [
114 - i for i, (_, proposal_len
115 - ) in enumerate(zip(seq_group_metadata_list, proposal_lens))
116 - if predicate(proposal_len)
117 - ]
118 - seq_groups = [
119 - seq_group for seq_group, proposal_len in zip(
120 - seq_group_metadata_list, proposal_lens) if predicate(proposal_len)
121 - ]
122 -
123 - return seq_groups, indices
109 + nonzero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], [])
110 + zero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], [])
111 + for i, (seq_group, proposal_len) in enumerate(
112 + zip(seq_group_metadata_list, proposal_lens)):
113 + seq_groups, indices = nonzero_lists if proposal_len else zero_lists
114 + seq_groups.append(seq_group)
115 + indices.append(i)
116 + return nonzero_lists, zero_lists
117
118
119 def sampler_output_to_torch(
127 - sampler_output_list: List[SamplerOutput], sampler_transposed: bool
120 + sampler_output_list: Sequence[SamplerOutput], sampler_transposed: bool
121 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
122 """Utility function which converts a list of SamplerOutput to tensors.
123
@@ -148,18 +141,12 @@ def sampler_output_to_torch(
141 dim=0,
142 )
143
151 - if sampler_transposed:
152 - sampled_token_probs = sampled_token_probs.transpose(0, 1)
153 -
144 # shape: [batch_size, num_sampler_output, vocab_size]
145 sampled_token_logprobs = torch.stack(
146 [sampler_output.logprobs for sampler_output in sampler_output_list],
147 dim=0,
148 )
149
160 - if sampler_transposed:
161 - sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
162 -
150 # shape: [batch_size, num_sampler_output]
151 sampled_token_ids = torch.stack(
152 [
@@ -168,7 +155,10 @@ def sampler_output_to_torch(
155 ],
156 dim=0,
157 )
158 +
159 if sampler_transposed:
160 + sampled_token_probs = sampled_token_probs.transpose(0, 1)
161 + sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
162 sampled_token_ids = sampled_token_ids.transpose(0, 1)
163
164 if sampler_output_list[0].hidden_states is not None: