A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Spec Decoding] Streamline batch expansion tensor manipulation (#7851)
Nick Hill committed
Aug 25, 2024 at 15:45 UTC
1856aff4d66833b258ce64132413ab8a18cc18a6
5 files changed
+118
-125
tests/spec_decode/test_utils.py
+13
-18
@@ -55,10 +55,9 @@ def fake_sequence_group_metadata():
55
56
def test_filter_zero_length_proposals(fake_sequence_group_metadata):
57
proposal_lens = [0, 1, 0]
58
- filtered_groups, indices = split_batch_by_proposal_len(
59
- fake_sequence_group_metadata,
60
- proposal_lens,
61
- select_proposal_len_zero=True)
58
+ _, (filtered_groups,
59
+ indices) = split_batch_by_proposal_len(fake_sequence_group_metadata,
60
+ proposal_lens)
61
62
expected_groups = [
63
fake_sequence_group_metadata[0], fake_sequence_group_metadata[2]
@@ -71,10 +70,9 @@ def test_filter_zero_length_proposals(fake_sequence_group_metadata):
70
71
def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
72
proposal_lens = [0, 1, 2]
74
- filtered_groups, indices = split_batch_by_proposal_len(
75
- fake_sequence_group_metadata,
76
- proposal_lens,
77
- select_proposal_len_zero=False)
73
+ (filtered_groups,
74
+ indices), _ = split_batch_by_proposal_len(fake_sequence_group_metadata,
75
+ proposal_lens)
76
77
expected_groups = [
78
fake_sequence_group_metadata[1], fake_sequence_group_metadata[2]
@@ -86,8 +84,7 @@ def test_filter_non_zero_length_proposals(fake_sequence_group_metadata):
84
85
86
def test_empty_inputs():
89
- filtered_groups, indices = split_batch_by_proposal_len(
90
- [], [], select_proposal_len_zero=True)
87
+ _, (filtered_groups, indices) = split_batch_by_proposal_len([], [])
88
89
assert filtered_groups == []
90
assert indices == []
@@ -95,10 +92,9 @@ def test_empty_inputs():
92
93
def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
94
proposal_lens = [0, 0, 0]
98
- filtered_groups, indices = split_batch_by_proposal_len(
99
- fake_sequence_group_metadata,
100
- proposal_lens,
101
- select_proposal_len_zero=False)
95
+ (filtered_groups,
96
+ indices), _ = split_batch_by_proposal_len(fake_sequence_group_metadata,
97
+ proposal_lens)
98
99
assert filtered_groups == []
100
assert indices == []
@@ -106,10 +102,9 @@ def test_all_zero_with_non_zero_filter(fake_sequence_group_metadata):
102
103
def test_all_non_zero_with_zero_filter(fake_sequence_group_metadata):
104
proposal_lens = [1, 1, 1]
109
- filtered_groups, indices = split_batch_by_proposal_len(
110
- fake_sequence_group_metadata,
111
- proposal_lens,
112
- select_proposal_len_zero=True)
105
+ _, (filtered_groups,
106
+ indices) = split_batch_by_proposal_len(fake_sequence_group_metadata,
107
+ proposal_lens)
108
109
assert filtered_groups == []
110
assert indices == []
vllm/spec_decode/batch_expansion.py
+79
-64
@@ -10,8 +10,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
10
get_all_seq_ids)
11
from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
SpeculativeScorer, SpeculativeScores)
13
-from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
14
- split_batch_by_proposal_len)
13
+from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
14
from vllm.worker.worker_base import WorkerBase
15
16
SeqId = int
@@ -88,17 +87,25 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
87
assert len(target_sampler_output) == 1, "expected single-step output"
88
target_sampler_output = target_sampler_output[0]
89
91
- (all_tokens, all_probs, spec_logprobs,
92
- all_hidden_states) = self._contract_batch(
93
- contracted_bs=len(execute_model_req.seq_group_metadata_list),
94
- target_sampler_output=target_sampler_output,
95
- proposals=proposals,
96
- num_scoring_tokens=num_scoring_tokens,
97
- non_spec_indices=non_spec_indices,
98
- spec_indices=spec_indices,
99
- k=execute_model_req.num_lookahead_slots,
100
- )
101
-
90
+ if not non_spec_indices:
91
+ # All sequence groups in batch have spec decoding enabled
92
+ contracted = self._contract_batch_all_spec(
93
+ target_sampler_output=target_sampler_output,
94
+ proposals=proposals,
95
+ )
96
+ else:
97
+ # Batch has a mix of spec decode enabled and disabled seq groups
98
+ contracted = self._contract_batch(
99
+ contracted_bs=len(execute_model_req.seq_group_metadata_list),
100
+ target_sampler_output=target_sampler_output,
101
+ proposals=proposals,
102
+ num_scoring_tokens=num_scoring_tokens,
103
+ non_spec_indices=non_spec_indices,
104
+ spec_indices=spec_indices,
105
+ k=execute_model_req.num_lookahead_slots,
106
+ )
107
+
108
+ all_tokens, all_probs, spec_logprobs, all_hidden_states = contracted
109
return SpeculativeScores(
110
probs=all_probs,
111
token_ids=all_tokens,
@@ -121,14 +128,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
128
# proposal len. This adds some complexity (splitting the batch into spec
129
# and non spec sequences) and should be removed in the future. It can be
130
# done by supporting per-sequence proposal lens.
124
- spec_seqs, spec_indices = split_batch_by_proposal_len(
125
- seq_group_metadata_list,
126
- proposal_lens_list,
127
- select_proposal_len_zero=False)
128
- non_spec_seqs, non_spec_indices = split_batch_by_proposal_len(
129
- seq_group_metadata_list,
130
- proposal_lens_list,
131
- select_proposal_len_zero=True)
131
+ (spec_seqs, spec_indices), (non_spec_seqs, non_spec_indices) = \
132
+ split_batch_by_proposal_len(
133
+ seq_group_metadata_list, proposal_lens_list)
134
135
target_seq_group_metadata_list = self._create_scoring_model_input(
136
seq_group_metadata_list=spec_seqs,
@@ -171,7 +173,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
173
# The number of tokens in the expanded batch used for speculation is
174
# equal to the total expanded batch size minus the number of samples for
175
# non-speculative sequences.
174
- non_spec_expanded_bs, _ = non_spec_target_token_ids.shape
176
+ non_spec_expanded_bs = len(non_spec_target_token_ids)
177
spec_expanded_bs = expanded_batch_size - non_spec_expanded_bs
178
179
target_token_ids = target_token_ids.reshape(spec_expanded_bs, k + 1)
@@ -181,7 +183,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
183
184
if target_hidden_states is not None:
185
target_hidden_states = target_hidden_states.reshape(
184
- spec_expanded_bs, k + 1, target_hidden_states.shape[-1])
186
+ *target_token_ids.shape, target_hidden_states.shape[-1])
187
188
all_tokens = target_token_ids.new_full(size=(contracted_bs, k + 1),
189
fill_value=-1)
@@ -196,24 +198,58 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
198
all_hidden_states = None
199
200
if non_spec_indices:
199
- all_tokens[non_spec_indices, :1] = non_spec_target_token_ids
200
- all_probs[non_spec_indices, :1, :] = non_spec_target_probs
201
- all_logprobs[non_spec_indices, :1, :] = non_spec_target_logprobs
202
-
201
+ all_tokens[non_spec_indices, :1] = \
202
+ non_spec_target_token_ids.unsqueeze(1)
203
+ all_probs[non_spec_indices, :1, :] = \
204
+ non_spec_target_probs.unsqueeze(1)
205
+ all_logprobs[non_spec_indices, :1, :] = \
206
+ non_spec_target_logprobs.unsqueeze(1)
207
if all_hidden_states is not None:
204
- all_hidden_states[
205
- non_spec_indices, :1, :] = non_spec_target_hidden_states
208
+ assert non_spec_target_hidden_states is not None
209
+ all_hidden_states[non_spec_indices, :1, :] = \
210
+ non_spec_target_hidden_states.unsqueeze(1)
211
212
if spec_indices:
213
all_tokens[spec_indices] = target_token_ids
214
all_probs[spec_indices] = target_probs
215
all_logprobs[spec_indices] = target_logprobs
211
-
216
if all_hidden_states is not None:
217
all_hidden_states[spec_indices] = target_hidden_states
218
219
return all_tokens, all_probs, all_logprobs, all_hidden_states
220
221
+ def _contract_batch_all_spec(
222
+ self,
223
+ target_sampler_output: SamplerOutput,
224
+ proposals: SpeculativeProposals,
225
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
226
+ Optional[torch.Tensor]]:
227
+ """Contract the expanded batch back into its original size.
228
+ This maps the scores of speculative tokens back to their original
229
+ sequences.
230
+
231
+ It assumes all sequences in the batch were previously expanded.
232
+ """
233
+
234
+ # Map distinct sequences used to score each token
235
+ # of shape [batch_size * k + 1] back to [batch_size, k + 1].
236
+ contracted_bs, k = proposals.proposal_token_ids.shape
237
+
238
+ # Reshape tensors to original batch size
239
+ target_token_ids = target_sampler_output.sampled_token_ids.reshape(
240
+ contracted_bs, k + 1)
241
+ target_probs = target_sampler_output.sampled_token_probs.reshape(
242
+ *target_token_ids.shape, self._vocab_size)
243
+ target_logprobs = target_sampler_output.logprobs.reshape(
244
+ target_probs.shape)
245
+ target_hidden_states = target_sampler_output.hidden_states
246
+ if target_hidden_states is not None:
247
+ target_hidden_states = target_hidden_states.reshape(
248
+ *target_token_ids.shape, target_hidden_states.shape[-1])
249
+
250
+ return (target_token_ids, target_probs, target_logprobs,
251
+ target_hidden_states)
252
+
253
def _create_scoring_model_input(
254
self,
255
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -345,8 +381,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
381
token_chunk_size=1,
382
)
383
384
+ @staticmethod
385
def _split_scoring_output(
349
- self, sampler_output: SamplerOutput, num_scoring_tokens: int
386
+ sampler_output: SamplerOutput, num_scoring_tokens: int
387
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor,
388
Optional[torch.Tensor], torch.Tensor, torch.Tensor,
389
torch.Tensor, Optional[torch.Tensor]]:
@@ -361,10 +398,9 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
398
#
399
# First samples are from speculative scoring, latter samples are non-
400
# speculative samples.
364
- split_sizes = [
365
- num_scoring_tokens,
366
- sampler_output.sampled_token_ids.numel() - num_scoring_tokens
367
- ]
401
+ split_sizes = (num_scoring_tokens,
402
+ sampler_output.sampled_token_ids.numel() -
403
+ num_scoring_tokens)
404
(spec_probs, non_spec_probs
405
) = sampler_output.sampled_token_probs.split(split_sizes)
406
(spec_sampled_tokens, non_spec_sampled_tokens
@@ -382,32 +418,13 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
418
else:
419
spec_hidden_states, non_spec_hidden_states = None, None
420
385
- # Convert scores to tensors.
386
- sampler_output.sampled_token_probs = spec_probs
387
- sampler_output.sampled_token_ids = spec_sampled_tokens
388
- sampler_output.logprobs = spec_logprobs
389
- sampler_output.hidden_states = spec_hidden_states
390
- (target_token_ids, target_probs, target_logprobs,
391
- target_hidden_states) = sampler_output_to_torch([sampler_output],
392
- True)
393
-
394
- # Convert non-speculative output tokens to tensors.
395
- sampler_output.sampled_token_probs = non_spec_probs
396
- sampler_output.sampled_token_ids = non_spec_sampled_tokens
397
- sampler_output.logprobs = non_spec_logprobs
398
- sampler_output.hidden_states = non_spec_hidden_states
399
- (non_spec_target_token_ids, non_spec_target_probs,
400
- non_spec_target_logprobs,
401
- non_spec_target_hidden_states) = sampler_output_to_torch(
402
- [sampler_output], True)
403
-
404
- return (target_token_ids, target_probs, target_logprobs,
405
- target_hidden_states, non_spec_target_token_ids,
406
- non_spec_target_probs, non_spec_target_logprobs,
407
- non_spec_target_hidden_states)
421
+ return (spec_sampled_tokens, spec_probs, spec_logprobs,
422
+ spec_hidden_states, non_spec_sampled_tokens, non_spec_probs,
423
+ non_spec_logprobs, non_spec_hidden_states)
424
425
+ @staticmethod
426
def _create_target_seq_id_iterator(
410
- self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
427
+ seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
428
"""Create an iterator for creating target sequence ids.
429
Target sequence ids are distinct from sequence ids because we create a
430
distinct target sequence id for each proposal token to be scored.
@@ -417,8 +434,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
434
"""
435
return count(start=max(seq_ids) + 1)
436
437
+ @staticmethod
438
def _get_token_ids_to_score(
421
- self,
439
full_spec_token_ids: List[TokenId] # shape: [k]
440
) -> List[List[TokenId]]:
441
"""Given an int tensor of proposal token ids, return a list of
@@ -439,8 +456,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
456
empty_token_ids: List[TokenId] = []
457
458
token_ids_to_score = [empty_token_ids]
442
- token_ids_to_score.extend([
443
- full_spec_token_ids[:i + 1]
444
- for i in range(len(full_spec_token_ids))
445
- ])
459
+ token_ids_to_score.extend(full_spec_token_ids[:i + 1]
460
+ for i in range(len(full_spec_token_ids)))
461
return token_ids_to_score
vllm/spec_decode/spec_decode_worker.py
+9
-16
@@ -365,12 +365,13 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
365
# used during the prefill phase.
366
# 2. Auto-disable enabled: The running queue size exceeds
367
# the specified threshold.
368
- # 3. No request: There are no requests in the batch.
368
+ # 3. No request: There are no requests in the batch, or
369
+ # none of the requests in the batch have spec decoding enabled.
370
# In any of these cases, the proposer and scorer workers
371
# are called normally.
371
- no_spec = num_lookahead_slots == 0 or len(
372
- execute_model_req.seq_group_metadata_list
373
- ) == 0 or disable_all_speculation
372
+ no_spec = num_lookahead_slots == 0 or disable_all_speculation or all(
373
+ sgm.num_speculative_tokens == 0
374
+ for sgm in execute_model_req.seq_group_metadata_list)
375
376
# Broadcast how many lookahead slots are scheduled for this step, and
377
# whether all speculation is disabled, to all non-driver workers.
@@ -415,10 +416,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
416
self, execute_model_req: ExecuteModelRequest) -> bool:
417
# When the batch size is too large, disable speculative decoding
418
# to stop trading off throughput for latency.
418
- disable_all_speculation = (execute_model_req.running_queue_size >=
419
- self.disable_by_batch_size)
420
-
421
- return disable_all_speculation
419
+ return (execute_model_req.running_queue_size >=
420
+ self.disable_by_batch_size)
421
422
def _maybe_disable_speculative_tokens(
423
self, disable_all_speculation: bool,
@@ -621,14 +620,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
620
# proposal len. This adds some complexity (splitting the batch into spec
621
# and non spec sequences) and should be removed in the future. It can be
622
# done by supporting per-sequence proposal lens.
624
- _, spec_indices = split_batch_by_proposal_len(
625
- seq_group_metadata_list,
626
- proposal_lens_list,
627
- select_proposal_len_zero=False)
628
- _, non_spec_indices = split_batch_by_proposal_len(
629
- seq_group_metadata_list,
630
- proposal_lens_list,
631
- select_proposal_len_zero=True)
623
+ (_, spec_indices), (_, non_spec_indices) = split_batch_by_proposal_len(
624
+ seq_group_metadata_list, proposal_lens_list)
625
original_indices = spec_indices + non_spec_indices
626
627
# Get probabilities of target model, excluding bonus token.
vllm/spec_decode/top1_proposer.py
+1
-1
@@ -138,7 +138,7 @@ class Top1Proposer(SpeculativeProposer):
138
139
# Currently only proposal lens of 0 or the global batch proposal len
140
# are supported.
141
- # If max_proposal_len is defined, then we shall no exceed this
141
+ # If max_proposal_len is defined, then we shall not exceed this
142
# quota for nonzero_proposal
143
new_k = 0
144
if (self.max_proposal_len is None
vllm/spec_decode/util.py
+16
-26
@@ -1,6 +1,6 @@
1
import time
2
from contextlib import contextmanager
3
-from typing import Dict, List, Optional, Tuple
3
+from typing import Dict, List, Optional, Sequence, Tuple
4
5
import torch
6
@@ -98,33 +98,26 @@ def create_sequence_group_output(
98
99
def split_batch_by_proposal_len(
100
seq_group_metadata_list: List[SequenceGroupMetadata],
101
- proposal_lens: List[int], select_proposal_len_zero: bool
102
-) -> Tuple[List[SequenceGroupMetadata], List[int]]:
101
+ proposal_lens: List[int],
102
+) -> Tuple[Tuple[List[SequenceGroupMetadata], List[int]], Tuple[
103
+ List[SequenceGroupMetadata], List[int]]]:
104
"""Utility function that splits a batch based on whether the proposal len is
105
zero or not. We should remove this once vLLM supports per-sequence proposal
106
lens in a batch.
107
"""
108
108
- if select_proposal_len_zero:
109
- predicate = lambda proposal_len: proposal_len == 0
110
- else:
111
- predicate = lambda proposal_len: proposal_len != 0
112
-
113
- indices = [
114
- i for i, (_, proposal_len
115
- ) in enumerate(zip(seq_group_metadata_list, proposal_lens))
116
- if predicate(proposal_len)
117
- ]
118
- seq_groups = [
119
- seq_group for seq_group, proposal_len in zip(
120
- seq_group_metadata_list, proposal_lens) if predicate(proposal_len)
121
- ]
122
-
123
- return seq_groups, indices
109
+ nonzero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], [])
110
+ zero_lists: Tuple[List[SequenceGroupMetadata], List[int]] = ([], [])
111
+ for i, (seq_group, proposal_len) in enumerate(
112
+ zip(seq_group_metadata_list, proposal_lens)):
113
+ seq_groups, indices = nonzero_lists if proposal_len else zero_lists
114
+ seq_groups.append(seq_group)
115
+ indices.append(i)
116
+ return nonzero_lists, zero_lists
117
118
119
def sampler_output_to_torch(
127
- sampler_output_list: List[SamplerOutput], sampler_transposed: bool
120
+ sampler_output_list: Sequence[SamplerOutput], sampler_transposed: bool
121
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
122
"""Utility function which converts a list of SamplerOutput to tensors.
123
@@ -148,18 +141,12 @@ def sampler_output_to_torch(
141
dim=0,
142
)
143
151
- if sampler_transposed:
152
- sampled_token_probs = sampled_token_probs.transpose(0, 1)
153
-
144
# shape: [batch_size, num_sampler_output, vocab_size]
145
sampled_token_logprobs = torch.stack(
146
[sampler_output.logprobs for sampler_output in sampler_output_list],
147
dim=0,
148
)
149
160
- if sampler_transposed:
161
- sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
162
-
150
# shape: [batch_size, num_sampler_output]
151
sampled_token_ids = torch.stack(
152
[
@@ -168,7 +155,10 @@ def sampler_output_to_torch(
155
],
156
dim=0,
157
)
158
+
159
if sampler_transposed:
160
+ sampled_token_probs = sampled_token_probs.transpose(0, 1)
161
+ sampled_token_logprobs = sampled_token_logprobs.transpose(0, 1)
162
sampled_token_ids = sampled_token_ids.transpose(0, 1)
163
164
if sampler_output_list[0].hidden_states is not None: