@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f187877945
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[FIX] Simplify sampler logic (#1156)

Zhuohan Li committed Sep 23, 2023 at 17:21 UTC f187877945ea88fff281b0818443b303681eba7d
1 file changed +10 -37
vllm/model_executor/layers/sampler.py
+10 -37
@@ -133,37 +133,22 @@ def _get_penalties(
133 # Collect the presence and frequency penalties.
134 presence_penalties: List[float] = []
135 frequency_penalties: List[float] = []
136 - for i, seq_group in enumerate(input_metadata.seq_groups):
136 + for seq_group in input_metadata.seq_groups:
137 seq_ids, sampling_params = seq_group
138 p = sampling_params.presence_penalty
139 f = sampling_params.frequency_penalty
140 - if i < input_metadata.num_prompts:
141 - # A prompt input.
142 - presence_penalties.append(p)
143 - frequency_penalties.append(f)
144 - else:
145 - # A generation token.
146 - presence_penalties += [p] * len(seq_ids)
147 - frequency_penalties += [f] * len(seq_ids)
140 + presence_penalties += [p] * len(seq_ids)
141 + frequency_penalties += [f] * len(seq_ids)
142 return presence_penalties, frequency_penalties
143
144
145 def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
146 output_tokens: List[List[int]] = []
153 - for i, seq_group in enumerate(input_metadata.seq_groups):
147 + for seq_group in input_metadata.seq_groups:
148 seq_ids, _ = seq_group
155 - if i < input_metadata.num_prompts:
156 - # A prompt input.
157 - # NOTE: While the prompt input usually has no output tokens,
158 - # it may have output tokens in the case of recomputation.
159 - seq_id = seq_ids[0]
149 + for seq_id in seq_ids:
150 seq_data = input_metadata.seq_data[seq_id]
151 output_tokens.append(seq_data.output_token_ids)
162 - else:
163 - # A generation token.
164 - for seq_id in seq_ids:
165 - seq_data = input_metadata.seq_data[seq_id]
166 - output_tokens.append(seq_data.output_token_ids)
152 return output_tokens
153
154
@@ -221,7 +206,7 @@ def _apply_penalties(
206 def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
207 # Collect the temperatures for the logits.
208 temperatures: List[float] = []
224 - for i, seq_group in enumerate(input_metadata.seq_groups):
209 + for seq_group in input_metadata.seq_groups:
210 seq_ids, sampling_params = seq_group
211 temperature = sampling_params.temperature
212 if temperature < _SAMPLING_EPS:
@@ -229,13 +214,7 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
214 # (i.e., greedy sampling or beam search).
215 # Set the temperature to 1 to avoid division by zero.
216 temperature = 1.0
232 -
233 - if i < input_metadata.num_prompts:
234 - # A prompt input.
235 - temperatures.append(temperature)
236 - else:
237 - # A generation token.
238 - temperatures += [temperature] * len(seq_ids)
217 + temperatures += [temperature] * len(seq_ids)
218 return temperatures
219
220
@@ -245,21 +224,15 @@ def _get_top_p_top_k(
224 ) -> Tuple[List[float], List[int]]:
225 top_ps: List[float] = []
226 top_ks: List[int] = []
248 - for i, seq_group in enumerate(input_metadata.seq_groups):
227 + for seq_group in input_metadata.seq_groups:
228 seq_ids, sampling_params = seq_group
229 top_p = sampling_params.top_p
230 # k should not be greater than the vocab size.
231 top_k = min(sampling_params.top_k, vocab_size)
232 # k=-1 means no truncation.
233 top_k = vocab_size if top_k == -1 else top_k
255 - if i < input_metadata.num_prompts:
256 - # A prompt input.
257 - top_ps.append(top_p)
258 - top_ks.append(top_k)
259 - else:
260 - # A generation token.
261 - top_ps += [top_p] * len(seq_ids)
262 - top_ks += [top_k] * len(seq_ids)
234 + top_ps += [top_p] * len(seq_ids)
235 + top_ks += [top_k] * len(seq_ids)
236 return top_ps, top_ks
237
238