A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding] Improve n-gram efficiency (#4724)
Cody Yu committed
May 13, 2024 at 15:00 UTC
ce532ff45c8008c7157eb448860c13bcdd44823f
4 files changed
+132
-58
tests/spec_decode/test_ngram_worker.py
+9
-8
@@ -34,8 +34,8 @@ def test_ngram_algo_correctness_for_single_no_match():
34
max_proposal_len=20,
35
)
36
37
- # set ngram window (0, 3], which is window=1/2/3
38
- ngram_worker.set_ngram_window_size(0, 3)
37
+ # set ngram window [1, 3], which is window=1/2/3
38
+ ngram_worker.set_ngram_window_size(1, 3)
39
40
prompts = [
41
# shall find no candidate
@@ -90,8 +90,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
90
max_proposal_len=20,
91
)
92
93
- # set ngram window (0, 3], which is window=1/2/3
94
- ngram_worker.set_ngram_window_size(0, 3)
93
+ # set ngram window [1, 3], which is window=1/2/3
94
+ ngram_worker.set_ngram_window_size(1, 3)
95
96
prompts = [
97
# shall find no candidate
@@ -128,11 +128,12 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
128
assert proposals.proposal_probs.shape[:-1] == torch.Size([5, proposal_len])
129
assert proposals.proposal_lens.shape == torch.Size([5])
130
131
+ # the first sequence has no match so proposal_len should be overwritten to 0
132
assert proposals.proposal_lens.tolist(
132
- ) == [proposal_len for _ in range(4)] + [0]
133
+ ) == [0] + [proposal_len for _ in range(3)] + [0]
134
135
for i in range(proposal_len):
135
- assert proposals.proposal_token_ids[0][i] == 0
136
+ assert proposals.proposal_token_ids[0][i] == -1
137
assert proposals.proposal_token_ids[1][i] == prompts[1][i + 1]
138
assert proposals.proposal_token_ids[2][i] == prompts[2][i + 3]
139
assert proposals.proposal_token_ids[3][i] == prompts[3][i + 5]
@@ -167,8 +168,8 @@ def test_ngram_algo_correctness_for_batches_match_all():
168
max_proposal_len=20,
169
)
170
170
- # set ngram window (0, 3], which is window=1/2/3
171
- ngram_worker.set_ngram_window_size(0, 3)
171
+ # set ngram window [0, 3], which is window=1/2/3
172
+ ngram_worker.set_ngram_window_size(1, 3)
173
174
prompts = [
175
# shall find candidate 12,13,14,15,16
vllm/config.py
+8
-5
@@ -784,12 +784,15 @@ class SpeculativeConfig:
784
draft_quantization = None
785
786
if speculative_model == "[ngram]":
787
- assert (ngram_prompt_lookup_max is not None
788
- and ngram_prompt_lookup_max > 0)
787
if ngram_prompt_lookup_min is None:
790
- ngram_prompt_lookup_min = 0
791
- else:
792
- assert ngram_prompt_lookup_max > ngram_prompt_lookup_min
788
+ ngram_prompt_lookup_min = 1
789
+ if ngram_prompt_lookup_max is None or ngram_prompt_lookup_max < 1:
790
+ raise ValueError(f"{ngram_prompt_lookup_max=} must be > 0")
791
+ if ngram_prompt_lookup_min < 1:
792
+ raise ValueError(f"{ngram_prompt_lookup_min=} must be > 0")
793
+ if ngram_prompt_lookup_min > ngram_prompt_lookup_max:
794
+ raise ValueError(f"{ngram_prompt_lookup_min=} cannot be "
795
+ f"larger than {ngram_prompt_lookup_max=}")
796
797
# TODO: current we still need extract vocab_size from target model
798
# config, in future, we may try refactor it out, and set
vllm/spec_decode/ngram_worker.py
+52
-45
@@ -77,9 +77,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
77
"""
78
self._raise_if_unsupported(execute_model_req)
79
80
- arr = []
80
has_spec_out = False
82
- for seq_group_metadata in execute_model_req.seq_group_metadata_list:
81
+ token_id_list = []
82
+ token_prob_list = []
83
+ for idx, seq_group_metadata in enumerate(
84
+ execute_model_req.seq_group_metadata_list):
85
seq_data = next(iter(seq_group_metadata.seq_data.values()))
86
87
input_ids = torch.as_tensor(seq_data.get_token_ids(),
@@ -89,59 +91,64 @@ class NGramWorker(LoraNotSupportedWorkerBase):
91
92
for ngram_size in range(
93
min(self.ngram_prompt_lookup_max, input_length - 1),
92
- self.ngram_prompt_lookup_min,
94
+ self.ngram_prompt_lookup_min - 1,
95
-1,
96
):
95
- ngram_tensor = input_ids[-1 * ngram_size:]
96
- windows = input_ids.unfold(dimension=0,
97
- size=ngram_size,
98
- step=1)
99
- matches = (windows == ngram_tensor).all(dim=1)
100
- match_indices = matches.nonzero(as_tuple=True)[0]
101
- if match_indices.size()[0] > 1:
97
+ ngram_tensor = input_ids[-ngram_size:]
98
+ proposal_start_idx = None
99
+ if ngram_size == 1:
100
+ # Do not match itself and do not use unfold and all
101
+ matches = (input_ids[:-1] == ngram_tensor)
102
+ else:
103
+ windows = input_ids.unfold(dimension=0,
104
+ size=ngram_size,
105
+ step=1)
106
+ # Do not match itself
107
+ matches = (windows[:-1] == ngram_tensor).all(dim=-1)
108
+
109
+ # first_match includes "values" (bool), indicating whether
110
+ # the match is found, and "indices", indicating the index
111
+ # of the first match.
112
+ # Note that "first_match.values.item()" triggers GPU-CPU
113
+ # sync so it is a bit inefficient, but we have not found
114
+ # a better way to do this.
115
+ first_match = matches.max(dim=-1)
116
+ if first_match.values.item():
117
+ proposal_start_idx = first_match.indices.add_(ngram_size)
118
+ spec_indices = (
119
+ proposal_start_idx).repeat(sample_len) + torch.arange(
120
+ sample_len, device=self.device)
121
+ spec_indices.clamp_(max=input_ids.shape[-1] - 1)
122
+ res = input_ids.gather(dim=-1, index=spec_indices)
123
+ token_id_list.append(res)
124
+ token_prob_list.append(
125
+ torch.nn.functional.one_hot(
126
+ res,
127
+ num_classes=self.vocab_size).to(torch.float32))
128
has_spec_out = True
103
- res = seq_data.get_token_ids()
104
- res = res[match_indices[0] + ngram_size:match_indices[0] +
105
- ngram_size + sample_len]
106
- res_len = len(res)
107
- # pad 0 towards output as sample_len tokens required
108
- res += [0] * (sample_len - res_len)
109
-
129
break
130
else:
112
- # if no candidate found, fill with 0
113
- res = [0] * sample_len
114
-
115
- arr.append(res)
131
+ token_id_list.append(None)
132
+ token_prob_list.append(None)
133
134
if not has_spec_out:
135
return None, False
136
120
- outputs = []
121
- token_ids = torch.as_tensor(arr, dtype=torch.long, device=self.device)
122
- indices = token_ids.unsqueeze(2)
137
+ outputs: List[Optional[SamplerOutput]] = []
138
+ for idx in range(len(execute_model_req.seq_group_metadata_list)):
139
+ if token_id_list[idx] is None:
140
+ outputs.append(None)
141
+ else:
142
+ outputs.append(
143
+ SamplerOutput(
144
+ outputs=None,
145
+ sampled_token_probs=token_prob_list[idx],
146
+ logprobs=torch.zeros((sample_len, self.vocab_size),
147
+ dtype=torch.float32,
148
+ device=self.device),
149
+ sampled_token_ids=token_id_list[idx],
150
+ ))
151
124
- token_probs = torch.zeros(
125
- (len(execute_model_req.seq_group_metadata_list), sample_len,
126
- self.vocab_size),
127
- dtype=torch.float32,
128
- device=self.device,
129
- )
130
- token_probs.scatter_(2, indices, 1)
131
- token_logprobs = torch.zeros(
132
- (len(execute_model_req.seq_group_metadata_list), sample_len,
133
- self.vocab_size),
134
- dtype=torch.float32,
135
- device=self.device,
136
- )
137
- for i in range(len(execute_model_req.seq_group_metadata_list)):
138
- outputs.append(
139
- SamplerOutput(
140
- outputs=None,
141
- sampled_token_probs=token_probs[i],
142
- logprobs=token_logprobs[i],
143
- sampled_token_ids=token_ids[i],
144
- ))
152
return outputs, False
153
154
def get_spec_proposals(
vllm/spec_decode/top1_proposer.py
+63
@@ -73,6 +73,14 @@ class Top1Proposer(SpeculativeProposer):
73
execute_model_req=nonzero_execute_model_req,
74
sample_len=proposal_len,
75
)
76
+ (
77
+ proposal_lens,
78
+ maybe_sampler_output,
79
+ nonzero_proposal_len_indices,
80
+ ) = self._remove_no_proposal_seqs(proposal_lens,
81
+ maybe_sampler_output,
82
+ nonzero_proposal_len_indices,
83
+ transposed)
84
else:
85
# If no sequences can be speculated, set sampler output to None.
86
maybe_sampler_output = None
@@ -140,6 +148,61 @@ class Top1Proposer(SpeculativeProposer):
148
nonzero_proposal_len_indices,
149
)
150
151
+ def _remove_no_proposal_seqs(self, proposal_lens, maybe_sampler_output,
152
+ nonzero_proposal_len_indices, transposed):
153
+ """Remove sequences from nonzero_proposal_len_indices and reset
154
+ their proposal_len to 0 the draft worker does not provide a proposal
155
+ (maybe_sampler_output=None). This can avoid scoring overheads.
156
+ """
157
+
158
+ # If maybe_sampler_output is None, then the draft worker did not
159
+ # provide a proposal for any sequence and thus no action needed.
160
+ # Also we do not support transposed maybe_sampler_output for now
161
+ # because it seems not straightforward for draft workers outputting
162
+ # transposed sampler outputs to handle the case of no proposal.
163
+ if maybe_sampler_output is None or transposed:
164
+ return (proposal_lens, maybe_sampler_output,
165
+ nonzero_proposal_len_indices)
166
+
167
+ new_proposal_lens: List[int] = []
168
+ new_nonzero_proposal_len_indices: List[int] = []
169
+ new_maybe_sampler_output: List[SamplerOutput] = []
170
+ nonzero_proposal_len_idx_ptr = 0
171
+ seq_idx = 0
172
+ while seq_idx < len(
173
+ proposal_lens) and nonzero_proposal_len_idx_ptr < len(
174
+ nonzero_proposal_len_indices):
175
+ if seq_idx < nonzero_proposal_len_indices[
176
+ nonzero_proposal_len_idx_ptr]:
177
+ # Sequence is not in the original nonzero_proposal_len_indices,
178
+ # meaning that it has a proposal length of 0 before sending to
179
+ # the draft worker.
180
+ assert proposal_lens[seq_idx] == 0
181
+ new_proposal_lens.append(0)
182
+ else:
183
+ # Sequence is in the original nonzero_proposal_len_indices
184
+ if maybe_sampler_output[nonzero_proposal_len_idx_ptr] is None:
185
+ # but does not have a proposal from the draft worker.
186
+ new_proposal_lens.append(0)
187
+ else:
188
+ # and has a proposal from the draft worker. Add it to the
189
+ # new nonzero proposal list and keep the sampler output.
190
+ new_proposal_lens.append(proposal_lens[seq_idx])
191
+ new_nonzero_proposal_len_indices.append(seq_idx)
192
+ new_maybe_sampler_output.append(
193
+ maybe_sampler_output[nonzero_proposal_len_idx_ptr])
194
+ nonzero_proposal_len_idx_ptr += 1
195
+ seq_idx += 1
196
+
197
+ # The remaining sequences should have proposal length of 0.
198
+ new_proposal_lens.extend(proposal_lens[seq_idx:])
199
+
200
+ # We assume sampler_output will not be a list of all Nones.
201
+ # In this case this function should not be called.
202
+ assert new_maybe_sampler_output
203
+ return (new_proposal_lens, new_maybe_sampler_output,
204
+ new_nonzero_proposal_len_indices)
205
+
206
def _merge_outputs(
207
self,
208
batch_size: int,