A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support repetition_penalty (#1424)
ljss committed
Oct 30, 2023 at 01:02 UTC
69be658bba3fcd97fbe23246406c64bd3f1cbe93
2 files changed
+35
-7
vllm/model_executor/layers/sampler.py
+25
-7
@@ -50,12 +50,13 @@ class Sampler(nn.Module):
50
# Apply presence and frequency penalties.
51
output_tokens = _get_output_tokens(input_metadata)
52
assert len(output_tokens) == logits.shape[0]
53
- presence_penalties, frequency_penalties = _get_penalties(
54
- input_metadata)
53
+ presence_penalties, frequency_penalties, repetition_penalties = (
54
+ _get_penalties(input_metadata))
55
assert len(presence_penalties) == logits.shape[0]
56
assert len(frequency_penalties) == logits.shape[0]
57
+ assert len(repetition_penalties) == logits.shape[0]
58
logits = _apply_penalties(logits, output_tokens, presence_penalties,
58
- frequency_penalties)
59
+ frequency_penalties, repetition_penalties)
60
61
# Apply temperature scaling.
62
temperatures = _get_temperatures(input_metadata)
@@ -134,14 +135,17 @@ def _prune_hidden_states(
135
136
137
def _get_penalties(
137
- input_metadata: InputMetadata) -> Tuple[List[float], List[float]]:
138
+ input_metadata: InputMetadata
139
+) -> Tuple[List[float], List[float], List[float]]:
140
# Collect the presence and frequency penalties.
141
presence_penalties: List[float] = []
142
frequency_penalties: List[float] = []
143
+ repetition_penalties: List[float] = []
144
for i, seq_group in enumerate(input_metadata.seq_groups):
145
seq_ids, sampling_params = seq_group
146
p = sampling_params.presence_penalty
147
f = sampling_params.frequency_penalty
148
+ r = sampling_params.repetition_penalty
149
if (i < input_metadata.num_prompts
150
and sampling_params.prompt_logprobs is not None):
151
# NOTE: We do not apply presence and frequency penalties for the
@@ -149,9 +153,11 @@ def _get_penalties(
153
prompt_len = input_metadata.prompt_lens[i]
154
presence_penalties += [0] * (prompt_len - 1)
155
frequency_penalties += [0] * (prompt_len - 1)
156
+ repetition_penalties += [1] * (prompt_len - 1)
157
presence_penalties += [p] * len(seq_ids)
158
frequency_penalties += [f] * len(seq_ids)
154
- return presence_penalties, frequency_penalties
159
+ repetition_penalties += [r] * len(seq_ids)
160
+ return presence_penalties, frequency_penalties, repetition_penalties
161
162
163
def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
@@ -175,6 +181,7 @@ def _apply_penalties(
181
output_tokens: List[List[int]],
182
presence_penalties: List[float],
183
frequency_penalties: List[float],
184
+ repetition_penalties: List[float],
185
) -> torch.Tensor:
186
num_seqs, vocab_size = logits.shape
187
for i in range(num_seqs):
@@ -182,7 +189,9 @@ def _apply_penalties(
189
continue
190
p = presence_penalties[i]
191
f = frequency_penalties[i]
185
- if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS:
192
+ r = repetition_penalties[i]
193
+ if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS and abs(
194
+ r - 1.0) < _SAMPLING_EPS:
195
continue
196
break
197
else:
@@ -206,7 +215,11 @@ def _apply_penalties(
215
bin_counts.scatter_add_(1, output_tokens_tensor,
216
torch.ones_like(output_tokens_tensor))
217
bin_counts = bin_counts[:, :vocab_size] # Remove the padding bin.
218
+ mask = bin_counts > 0
219
220
+ repetition_penalties = torch.tensor(repetition_penalties,
221
+ dtype=logits.dtype,
222
+ device=logits.device)
223
frequency_penalties = torch.tensor(frequency_penalties,
224
dtype=logits.dtype,
225
device=logits.device)
@@ -214,10 +227,15 @@ def _apply_penalties(
227
dtype=logits.dtype,
228
device=logits.device)
229
230
+ repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
231
+ repetition_penalties[~mask] = 1.0
232
+ logits = torch.where(logits > 0, logits / repetition_penalties,
233
+ logits * repetition_penalties)
234
+
235
# We follow the definition in OpenAI API.
236
# Refer to https://platform.openai.com/docs/api-reference/parameter-details
237
logits -= frequency_penalties.unsqueeze(dim=1) * bin_counts
220
- logits -= presence_penalties.unsqueeze(dim=1) * (bin_counts > 0)
238
+ logits -= presence_penalties.unsqueeze(dim=1) * mask
239
return logits
240
241
vllm/sampling_params.py
+10
@@ -34,6 +34,10 @@ class SamplingParams:
34
frequency in the generated text so far. Values > 0 encourage the
35
model to use new tokens, while values < 0 encourage the model to
36
repeat tokens.
37
+ repetition_penalty: Float that penalizes new tokens based on whether
38
+ they appear in the generated text so far. Values > 1 encourage the
39
+ model to use new tokens, while values < 1 encourage the model to
40
+ repeat tokens.
41
temperature: Float that controls the randomness of the sampling. Lower
42
values make the model more deterministic, while higher values make
43
the model more random. Zero means greedy sampling.
@@ -75,6 +79,7 @@ class SamplingParams:
79
best_of: Optional[int] = None,
80
presence_penalty: float = 0.0,
81
frequency_penalty: float = 0.0,
82
+ repetition_penalty: float = 1.0,
83
temperature: float = 1.0,
84
top_p: float = 1.0,
85
top_k: int = -1,
@@ -93,6 +98,7 @@ class SamplingParams:
98
self.best_of = best_of if best_of is not None else n
99
self.presence_penalty = presence_penalty
100
self.frequency_penalty = frequency_penalty
101
+ self.repetition_penalty = repetition_penalty
102
self.temperature = temperature
103
self.top_p = top_p
104
self.top_k = top_k
@@ -136,6 +142,9 @@ class SamplingParams:
142
if not -2.0 <= self.frequency_penalty <= 2.0:
143
raise ValueError("frequency_penalty must be in [-2, 2], got "
144
f"{self.frequency_penalty}.")
145
+ if not 0.0 < self.repetition_penalty <= 2.0:
146
+ raise ValueError("repetition_penalty must be in (0, 2], got "
147
+ f"{self.repetition_penalty}.")
148
if self.temperature < 0.0:
149
raise ValueError(
150
f"temperature must be non-negative, got {self.temperature}.")
@@ -201,6 +210,7 @@ class SamplingParams:
210
f"best_of={self.best_of}, "
211
f"presence_penalty={self.presence_penalty}, "
212
f"frequency_penalty={self.frequency_penalty}, "
213
+ f"repetition_penalty={self.repetition_penalty}, "
214
f"temperature={self.temperature}, "
215
f"top_p={self.top_p}, "
216
f"top_k={self.top_k}, "