@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce532ff45c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding] Improve n-gram efficiency (#4724)

Cody Yu committed May 13, 2024 at 15:00 UTC ce532ff45c8008c7157eb448860c13bcdd44823f
4 files changed +132 -58
tests/spec_decode/test_ngram_worker.py
+9 -8
@@ -34,8 +34,8 @@ def test_ngram_algo_correctness_for_single_no_match():
34 max_proposal_len=20,
35 )
36
37 - # set ngram window (0, 3], which is window=1/2/3
38 - ngram_worker.set_ngram_window_size(0, 3)
37 + # set ngram window [1, 3], which is window=1/2/3
38 + ngram_worker.set_ngram_window_size(1, 3)
39
40 prompts = [
41 # shall find no candidate
@@ -90,8 +90,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
90 max_proposal_len=20,
91 )
92
93 - # set ngram window (0, 3], which is window=1/2/3
94 - ngram_worker.set_ngram_window_size(0, 3)
93 + # set ngram window [1, 3], which is window=1/2/3
94 + ngram_worker.set_ngram_window_size(1, 3)
95
96 prompts = [
97 # shall find no candidate
@@ -128,11 +128,12 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
128 assert proposals.proposal_probs.shape[:-1] == torch.Size([5, proposal_len])
129 assert proposals.proposal_lens.shape == torch.Size([5])
130
131 + # the first sequence has no match so proposal_len should be overwritten to 0
132 assert proposals.proposal_lens.tolist(
132 - ) == [proposal_len for _ in range(4)] + [0]
133 + ) == [0] + [proposal_len for _ in range(3)] + [0]
134
135 for i in range(proposal_len):
135 - assert proposals.proposal_token_ids[0][i] == 0
136 + assert proposals.proposal_token_ids[0][i] == -1
137 assert proposals.proposal_token_ids[1][i] == prompts[1][i + 1]
138 assert proposals.proposal_token_ids[2][i] == prompts[2][i + 3]
139 assert proposals.proposal_token_ids[3][i] == prompts[3][i + 5]
@@ -167,8 +168,8 @@ def test_ngram_algo_correctness_for_batches_match_all():
168 max_proposal_len=20,
169 )
170
170 - # set ngram window (0, 3], which is window=1/2/3
171 - ngram_worker.set_ngram_window_size(0, 3)
171 + # set ngram window [0, 3], which is window=1/2/3
172 + ngram_worker.set_ngram_window_size(1, 3)
173
174 prompts = [
175 # shall find candidate 12,13,14,15,16
vllm/config.py
+8 -5
@@ -784,12 +784,15 @@ class SpeculativeConfig:
784 draft_quantization = None
785
786 if speculative_model == "[ngram]":
787 - assert (ngram_prompt_lookup_max is not None
788 - and ngram_prompt_lookup_max > 0)
787 if ngram_prompt_lookup_min is None:
790 - ngram_prompt_lookup_min = 0
791 - else:
792 - assert ngram_prompt_lookup_max > ngram_prompt_lookup_min
788 + ngram_prompt_lookup_min = 1
789 + if ngram_prompt_lookup_max is None or ngram_prompt_lookup_max < 1:
790 + raise ValueError(f"{ngram_prompt_lookup_max=} must be > 0")
791 + if ngram_prompt_lookup_min < 1:
792 + raise ValueError(f"{ngram_prompt_lookup_min=} must be > 0")
793 + if ngram_prompt_lookup_min > ngram_prompt_lookup_max:
794 + raise ValueError(f"{ngram_prompt_lookup_min=} cannot be "
795 + f"larger than {ngram_prompt_lookup_max=}")
796
797 # TODO: current we still need extract vocab_size from target model
798 # config, in future, we may try refactor it out, and set
vllm/spec_decode/ngram_worker.py
+52 -45
@@ -77,9 +77,11 @@ class NGramWorker(LoraNotSupportedWorkerBase):
77 """
78 self._raise_if_unsupported(execute_model_req)
79
80 - arr = []
80 has_spec_out = False
82 - for seq_group_metadata in execute_model_req.seq_group_metadata_list:
81 + token_id_list = []
82 + token_prob_list = []
83 + for idx, seq_group_metadata in enumerate(
84 + execute_model_req.seq_group_metadata_list):
85 seq_data = next(iter(seq_group_metadata.seq_data.values()))
86
87 input_ids = torch.as_tensor(seq_data.get_token_ids(),
@@ -89,59 +91,64 @@ class NGramWorker(LoraNotSupportedWorkerBase):
91
92 for ngram_size in range(
93 min(self.ngram_prompt_lookup_max, input_length - 1),
92 - self.ngram_prompt_lookup_min,
94 + self.ngram_prompt_lookup_min - 1,
95 -1,
96 ):
95 - ngram_tensor = input_ids[-1 * ngram_size:]
96 - windows = input_ids.unfold(dimension=0,
97 - size=ngram_size,
98 - step=1)
99 - matches = (windows == ngram_tensor).all(dim=1)
100 - match_indices = matches.nonzero(as_tuple=True)[0]
101 - if match_indices.size()[0] > 1:
97 + ngram_tensor = input_ids[-ngram_size:]
98 + proposal_start_idx = None
99 + if ngram_size == 1:
100 + # Do not match itself and do not use unfold and all
101 + matches = (input_ids[:-1] == ngram_tensor)
102 + else:
103 + windows = input_ids.unfold(dimension=0,
104 + size=ngram_size,
105 + step=1)
106 + # Do not match itself
107 + matches = (windows[:-1] == ngram_tensor).all(dim=-1)
108 +
109 + # first_match includes "values" (bool), indicating whether
110 + # the match is found, and "indices", indicating the index
111 + # of the first match.
112 + # Note that "first_match.values.item()" triggers GPU-CPU
113 + # sync so it is a bit inefficient, but we have not found
114 + # a better way to do this.
115 + first_match = matches.max(dim=-1)
116 + if first_match.values.item():
117 + proposal_start_idx = first_match.indices.add_(ngram_size)
118 + spec_indices = (
119 + proposal_start_idx).repeat(sample_len) + torch.arange(
120 + sample_len, device=self.device)
121 + spec_indices.clamp_(max=input_ids.shape[-1] - 1)
122 + res = input_ids.gather(dim=-1, index=spec_indices)
123 + token_id_list.append(res)
124 + token_prob_list.append(
125 + torch.nn.functional.one_hot(
126 + res,
127 + num_classes=self.vocab_size).to(torch.float32))
128 has_spec_out = True
103 - res = seq_data.get_token_ids()
104 - res = res[match_indices[0] + ngram_size:match_indices[0] +
105 - ngram_size + sample_len]
106 - res_len = len(res)
107 - # pad 0 towards output as sample_len tokens required
108 - res += [0] * (sample_len - res_len)
109 -
129 break
130 else:
112 - # if no candidate found, fill with 0
113 - res = [0] * sample_len
114 -
115 - arr.append(res)
131 + token_id_list.append(None)
132 + token_prob_list.append(None)
133
134 if not has_spec_out:
135 return None, False
136
120 - outputs = []
121 - token_ids = torch.as_tensor(arr, dtype=torch.long, device=self.device)
122 - indices = token_ids.unsqueeze(2)
137 + outputs: List[Optional[SamplerOutput]] = []
138 + for idx in range(len(execute_model_req.seq_group_metadata_list)):
139 + if token_id_list[idx] is None:
140 + outputs.append(None)
141 + else:
142 + outputs.append(
143 + SamplerOutput(
144 + outputs=None,
145 + sampled_token_probs=token_prob_list[idx],
146 + logprobs=torch.zeros((sample_len, self.vocab_size),
147 + dtype=torch.float32,
148 + device=self.device),
149 + sampled_token_ids=token_id_list[idx],
150 + ))
151
124 - token_probs = torch.zeros(
125 - (len(execute_model_req.seq_group_metadata_list), sample_len,
126 - self.vocab_size),
127 - dtype=torch.float32,
128 - device=self.device,
129 - )
130 - token_probs.scatter_(2, indices, 1)
131 - token_logprobs = torch.zeros(
132 - (len(execute_model_req.seq_group_metadata_list), sample_len,
133 - self.vocab_size),
134 - dtype=torch.float32,
135 - device=self.device,
136 - )
137 - for i in range(len(execute_model_req.seq_group_metadata_list)):
138 - outputs.append(
139 - SamplerOutput(
140 - outputs=None,
141 - sampled_token_probs=token_probs[i],
142 - logprobs=token_logprobs[i],
143 - sampled_token_ids=token_ids[i],
144 - ))
152 return outputs, False
153
154 def get_spec_proposals(
vllm/spec_decode/top1_proposer.py
+63
@@ -73,6 +73,14 @@ class Top1Proposer(SpeculativeProposer):
73 execute_model_req=nonzero_execute_model_req,
74 sample_len=proposal_len,
75 )
76 + (
77 + proposal_lens,
78 + maybe_sampler_output,
79 + nonzero_proposal_len_indices,
80 + ) = self._remove_no_proposal_seqs(proposal_lens,
81 + maybe_sampler_output,
82 + nonzero_proposal_len_indices,
83 + transposed)
84 else:
85 # If no sequences can be speculated, set sampler output to None.
86 maybe_sampler_output = None
@@ -140,6 +148,61 @@ class Top1Proposer(SpeculativeProposer):
148 nonzero_proposal_len_indices,
149 )
150
151 + def _remove_no_proposal_seqs(self, proposal_lens, maybe_sampler_output,
152 + nonzero_proposal_len_indices, transposed):
153 + """Remove sequences from nonzero_proposal_len_indices and reset
154 + their proposal_len to 0 the draft worker does not provide a proposal
155 + (maybe_sampler_output=None). This can avoid scoring overheads.
156 + """
157 +
158 + # If maybe_sampler_output is None, then the draft worker did not
159 + # provide a proposal for any sequence and thus no action needed.
160 + # Also we do not support transposed maybe_sampler_output for now
161 + # because it seems not straightforward for draft workers outputting
162 + # transposed sampler outputs to handle the case of no proposal.
163 + if maybe_sampler_output is None or transposed:
164 + return (proposal_lens, maybe_sampler_output,
165 + nonzero_proposal_len_indices)
166 +
167 + new_proposal_lens: List[int] = []
168 + new_nonzero_proposal_len_indices: List[int] = []
169 + new_maybe_sampler_output: List[SamplerOutput] = []
170 + nonzero_proposal_len_idx_ptr = 0
171 + seq_idx = 0
172 + while seq_idx < len(
173 + proposal_lens) and nonzero_proposal_len_idx_ptr < len(
174 + nonzero_proposal_len_indices):
175 + if seq_idx < nonzero_proposal_len_indices[
176 + nonzero_proposal_len_idx_ptr]:
177 + # Sequence is not in the original nonzero_proposal_len_indices,
178 + # meaning that it has a proposal length of 0 before sending to
179 + # the draft worker.
180 + assert proposal_lens[seq_idx] == 0
181 + new_proposal_lens.append(0)
182 + else:
183 + # Sequence is in the original nonzero_proposal_len_indices
184 + if maybe_sampler_output[nonzero_proposal_len_idx_ptr] is None:
185 + # but does not have a proposal from the draft worker.
186 + new_proposal_lens.append(0)
187 + else:
188 + # and has a proposal from the draft worker. Add it to the
189 + # new nonzero proposal list and keep the sampler output.
190 + new_proposal_lens.append(proposal_lens[seq_idx])
191 + new_nonzero_proposal_len_indices.append(seq_idx)
192 + new_maybe_sampler_output.append(
193 + maybe_sampler_output[nonzero_proposal_len_idx_ptr])
194 + nonzero_proposal_len_idx_ptr += 1
195 + seq_idx += 1
196 +
197 + # The remaining sequences should have proposal length of 0.
198 + new_proposal_lens.extend(proposal_lens[seq_idx:])
199 +
200 + # We assume sampler_output will not be a list of all Nones.
201 + # In this case this function should not be called.
202 + assert new_maybe_sampler_output
203 + return (new_proposal_lens, new_maybe_sampler_output,
204 + new_nonzero_proposal_len_indices)
205 +
206 def _merge_outputs(
207 self,
208 batch_size: int,