A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[FIX] Simplify sampler logic (#1156)
Zhuohan Li committed
Sep 23, 2023 at 17:21 UTC
f187877945ea88fff281b0818443b303681eba7d
1 file changed
+10
-37
vllm/model_executor/layers/sampler.py
+10
-37
@@ -133,37 +133,22 @@ def _get_penalties(
133
# Collect the presence and frequency penalties.
134
presence_penalties: List[float] = []
135
frequency_penalties: List[float] = []
136
- for i, seq_group in enumerate(input_metadata.seq_groups):
136
+ for seq_group in input_metadata.seq_groups:
137
seq_ids, sampling_params = seq_group
138
p = sampling_params.presence_penalty
139
f = sampling_params.frequency_penalty
140
- if i < input_metadata.num_prompts:
141
- # A prompt input.
142
- presence_penalties.append(p)
143
- frequency_penalties.append(f)
144
- else:
145
- # A generation token.
146
- presence_penalties += [p] * len(seq_ids)
147
- frequency_penalties += [f] * len(seq_ids)
140
+ presence_penalties += [p] * len(seq_ids)
141
+ frequency_penalties += [f] * len(seq_ids)
142
return presence_penalties, frequency_penalties
143
144
145
def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
146
output_tokens: List[List[int]] = []
153
- for i, seq_group in enumerate(input_metadata.seq_groups):
147
+ for seq_group in input_metadata.seq_groups:
148
seq_ids, _ = seq_group
155
- if i < input_metadata.num_prompts:
156
- # A prompt input.
157
- # NOTE: While the prompt input usually has no output tokens,
158
- # it may have output tokens in the case of recomputation.
159
- seq_id = seq_ids[0]
149
+ for seq_id in seq_ids:
150
seq_data = input_metadata.seq_data[seq_id]
151
output_tokens.append(seq_data.output_token_ids)
162
- else:
163
- # A generation token.
164
- for seq_id in seq_ids:
165
- seq_data = input_metadata.seq_data[seq_id]
166
- output_tokens.append(seq_data.output_token_ids)
152
return output_tokens
153
154
@@ -221,7 +206,7 @@ def _apply_penalties(
206
def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
207
# Collect the temperatures for the logits.
208
temperatures: List[float] = []
224
- for i, seq_group in enumerate(input_metadata.seq_groups):
209
+ for seq_group in input_metadata.seq_groups:
210
seq_ids, sampling_params = seq_group
211
temperature = sampling_params.temperature
212
if temperature < _SAMPLING_EPS:
@@ -229,13 +214,7 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
214
# (i.e., greedy sampling or beam search).
215
# Set the temperature to 1 to avoid division by zero.
216
temperature = 1.0
232
-
233
- if i < input_metadata.num_prompts:
234
- # A prompt input.
235
- temperatures.append(temperature)
236
- else:
237
- # A generation token.
238
- temperatures += [temperature] * len(seq_ids)
217
+ temperatures += [temperature] * len(seq_ids)
218
return temperatures
219
220
@@ -245,21 +224,15 @@ def _get_top_p_top_k(
224
) -> Tuple[List[float], List[int]]:
225
top_ps: List[float] = []
226
top_ks: List[int] = []
248
- for i, seq_group in enumerate(input_metadata.seq_groups):
227
+ for seq_group in input_metadata.seq_groups:
228
seq_ids, sampling_params = seq_group
229
top_p = sampling_params.top_p
230
# k should not be greater than the vocab size.
231
top_k = min(sampling_params.top_k, vocab_size)
232
# k=-1 means no truncation.
233
top_k = vocab_size if top_k == -1 else top_k
255
- if i < input_metadata.num_prompts:
256
- # A prompt input.
257
- top_ps.append(top_p)
258
- top_ks.append(top_k)
259
- else:
260
- # A generation token.
261
- top_ps += [top_p] * len(seq_ids)
262
- top_ks += [top_k] * len(seq_ids)
234
+ top_ps += [top_p] * len(seq_ids)
235
+ top_ks += [top_k] * len(seq_ids)
236
return top_ps, top_ks
237
238