A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix repetition penalty aligned with huggingface (#1577)
ljss committed
Nov 23, 2023 at 06:41 UTC
de23687d168ebeaa8872c27f05b8292bab0fac71
2 files changed
+50
-32
vllm/model_executor/layers/sampler.py
+47
-29
@@ -21,7 +21,7 @@ class Sampler(nn.Module):
21
1. Discard the hidden states that are not used for sampling (i.e., all
22
tokens except the final one in each prompt).
23
2. Compute the logits for the next tokens.
24
- 3. Apply presence and frequency penalties.
24
+ 3. Apply presence, frequency and repetition penalties.
25
4. Apply temperature scaling.
26
5. Apply top-p and top-k truncation.
27
6. Sample the next tokens.
@@ -50,14 +50,12 @@ class Sampler(nn.Module):
50
# Apply logits processors (if any).
51
logits = _apply_logits_processors(logits, input_metadata)
52
# Apply presence and frequency penalties.
53
- output_tokens = _get_output_tokens(input_metadata)
54
- assert len(output_tokens) == logits.shape[0]
53
presence_penalties, frequency_penalties, repetition_penalties = (
54
_get_penalties(input_metadata))
55
assert len(presence_penalties) == logits.shape[0]
56
assert len(frequency_penalties) == logits.shape[0]
57
assert len(repetition_penalties) == logits.shape[0]
60
- logits = _apply_penalties(logits, output_tokens, presence_penalties,
58
+ logits = _apply_penalties(logits, input_metadata, presence_penalties,
59
frequency_penalties, repetition_penalties)
60
61
# Apply temperature scaling.
@@ -146,7 +144,10 @@ def _get_penalties(
144
return presence_penalties, frequency_penalties, repetition_penalties
145
146
149
-def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
147
+def _get_prompt_and_output_tokens(
148
+ input_metadata: InputMetadata
149
+) -> Tuple[List[List[int]], List[List[int]]]:
150
+ prompt_tokens: List[List[int]] = []
151
output_tokens: List[List[int]] = []
152
for i, seq_group in enumerate(input_metadata.seq_groups):
153
seq_ids, sampling_params = seq_group
@@ -155,11 +156,39 @@ def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
156
# NOTE: prompt token positions do not need output tokens to
157
# compute penalties.
158
prompt_len = input_metadata.prompt_lens[i]
159
+ prompt_tokens.extend([] for _ in range(prompt_len - 1))
160
output_tokens.extend([] for _ in range(prompt_len - 1))
161
for seq_id in seq_ids:
162
seq_data = input_metadata.seq_data[seq_id]
163
+ prompt_tokens.append(seq_data.prompt_token_ids)
164
output_tokens.append(seq_data.output_token_ids)
162
- return output_tokens
165
+ return prompt_tokens, output_tokens
166
+
167
+
168
+def _get_bin_counts_and_mask(
169
+ logits: torch.Tensor,
170
+ tokens: List[List[int]],
171
+ vocab_size: int,
172
+ num_seqs: int,
173
+) -> Tuple[torch.Tensor, torch.Tensor]:
174
+ max_len = max(len(tokens) for tokens in tokens)
175
+ padded_tokens = [
176
+ tokens + [vocab_size] * (max_len - len(tokens)) for tokens in tokens
177
+ ]
178
+ tokens_tensor = torch.tensor(padded_tokens,
179
+ dtype=torch.long,
180
+ device=logits.device)
181
+
182
+ # Compute the bin counts for the tokens.
183
+ # vocab_size + 1 for padding.
184
+ bin_counts = torch.zeros((num_seqs, vocab_size + 1),
185
+ dtype=torch.long,
186
+ device=logits.device)
187
+ bin_counts.scatter_add_(1, tokens_tensor, torch.ones_like(tokens_tensor))
188
+ bin_counts = bin_counts[:, :vocab_size]
189
+ mask = bin_counts > 0
190
+
191
+ return bin_counts, mask
192
193
194
def _apply_logits_processors(logits: torch.Tensor,
@@ -186,15 +215,13 @@ def _apply_logits_processors(logits: torch.Tensor,
215
216
def _apply_penalties(
217
logits: torch.Tensor,
189
- output_tokens: List[List[int]],
218
+ input_metadata: InputMetadata,
219
presence_penalties: List[float],
220
frequency_penalties: List[float],
221
repetition_penalties: List[float],
222
) -> torch.Tensor:
223
num_seqs, vocab_size = logits.shape
224
for i in range(num_seqs):
196
- if not output_tokens[i]:
197
- continue
225
p = presence_penalties[i]
226
f = frequency_penalties[i]
227
r = repetition_penalties[i]
@@ -206,24 +233,15 @@ def _apply_penalties(
233
# Return early if all sequences have zero penalties.
234
return logits
235
209
- max_output_len = max(len(tokens) for tokens in output_tokens)
210
- padded_output_tokens = [
211
- tokens + [vocab_size] * (max_output_len - len(tokens))
212
- for tokens in output_tokens
213
- ]
214
- output_tokens_tensor = torch.tensor(padded_output_tokens,
215
- dtype=torch.long,
216
- device=logits.device)
236
+ prompt_tokens, output_tokens = (
237
+ _get_prompt_and_output_tokens(input_metadata))
238
+ assert len(prompt_tokens) == logits.shape[0]
239
+ assert len(output_tokens) == logits.shape[0]
240
218
- # Compute the bin counts for the output tokens.
219
- # vocab_size + 1 for padding.
220
- bin_counts = torch.zeros((num_seqs, vocab_size + 1),
221
- dtype=torch.long,
222
- device=logits.device)
223
- bin_counts.scatter_add_(1, output_tokens_tensor,
224
- torch.ones_like(output_tokens_tensor))
225
- bin_counts = bin_counts[:, :vocab_size] # Remove the padding bin.
226
- mask = bin_counts > 0
241
+ prompt_bin_counts, prompt_mask = _get_bin_counts_and_mask(
242
+ logits, prompt_tokens, vocab_size, num_seqs)
243
+ output_bin_counts, output_mask = _get_bin_counts_and_mask(
244
+ logits, output_tokens, vocab_size, num_seqs)
245
246
repetition_penalties = torch.tensor(repetition_penalties,
247
dtype=logits.dtype,
@@ -236,14 +254,14 @@ def _apply_penalties(
254
device=logits.device)
255
256
repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
239
- repetition_penalties[~mask] = 1.0
257
+ repetition_penalties[~(prompt_mask | output_mask)] = 1.0
258
logits = torch.where(logits > 0, logits / repetition_penalties,
259
logits * repetition_penalties)
260
261
# We follow the definition in OpenAI API.
262
# Refer to https://platform.openai.com/docs/api-reference/parameter-details
245
- logits -= frequency_penalties.unsqueeze(dim=1) * bin_counts
246
- logits -= presence_penalties.unsqueeze(dim=1) * mask
263
+ logits -= frequency_penalties.unsqueeze(dim=1) * output_bin_counts
264
+ logits -= presence_penalties.unsqueeze(dim=1) * output_mask
265
return logits
266
267
vllm/sampling_params.py
+3
-3
@@ -42,9 +42,9 @@ class SamplingParams:
42
model to use new tokens, while values < 0 encourage the model to
43
repeat tokens.
44
repetition_penalty: Float that penalizes new tokens based on whether
45
- they appear in the generated text so far. Values > 1 encourage the
46
- model to use new tokens, while values < 1 encourage the model to
47
- repeat tokens.
45
+ they appear in the prompt and the generated text so far. Values > 1
46
+ encourage the model to use new tokens, while values < 1 encourage
47
+ the model to repeat tokens.
48
temperature: Float that controls the randomness of the sampling. Lower
49
values make the model more deterministic, while higher values make
50
the model more random. Zero means greedy sampling.