@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69be658bba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support repetition_penalty (#1424)

ljss committed Oct 30, 2023 at 01:02 UTC 69be658bba3fcd97fbe23246406c64bd3f1cbe93
2 files changed +35 -7
vllm/model_executor/layers/sampler.py
+25 -7
@@ -50,12 +50,13 @@ class Sampler(nn.Module):
50 # Apply presence and frequency penalties.
51 output_tokens = _get_output_tokens(input_metadata)
52 assert len(output_tokens) == logits.shape[0]
53 - presence_penalties, frequency_penalties = _get_penalties(
54 - input_metadata)
53 + presence_penalties, frequency_penalties, repetition_penalties = (
54 + _get_penalties(input_metadata))
55 assert len(presence_penalties) == logits.shape[0]
56 assert len(frequency_penalties) == logits.shape[0]
57 + assert len(repetition_penalties) == logits.shape[0]
58 logits = _apply_penalties(logits, output_tokens, presence_penalties,
58 - frequency_penalties)
59 + frequency_penalties, repetition_penalties)
60
61 # Apply temperature scaling.
62 temperatures = _get_temperatures(input_metadata)
@@ -134,14 +135,17 @@ def _prune_hidden_states(
135
136
137 def _get_penalties(
137 - input_metadata: InputMetadata) -> Tuple[List[float], List[float]]:
138 + input_metadata: InputMetadata
139 +) -> Tuple[List[float], List[float], List[float]]:
140 # Collect the presence and frequency penalties.
141 presence_penalties: List[float] = []
142 frequency_penalties: List[float] = []
143 + repetition_penalties: List[float] = []
144 for i, seq_group in enumerate(input_metadata.seq_groups):
145 seq_ids, sampling_params = seq_group
146 p = sampling_params.presence_penalty
147 f = sampling_params.frequency_penalty
148 + r = sampling_params.repetition_penalty
149 if (i < input_metadata.num_prompts
150 and sampling_params.prompt_logprobs is not None):
151 # NOTE: We do not apply presence and frequency penalties for the
@@ -149,9 +153,11 @@ def _get_penalties(
153 prompt_len = input_metadata.prompt_lens[i]
154 presence_penalties += [0] * (prompt_len - 1)
155 frequency_penalties += [0] * (prompt_len - 1)
156 + repetition_penalties += [1] * (prompt_len - 1)
157 presence_penalties += [p] * len(seq_ids)
158 frequency_penalties += [f] * len(seq_ids)
154 - return presence_penalties, frequency_penalties
159 + repetition_penalties += [r] * len(seq_ids)
160 + return presence_penalties, frequency_penalties, repetition_penalties
161
162
163 def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
@@ -175,6 +181,7 @@ def _apply_penalties(
181 output_tokens: List[List[int]],
182 presence_penalties: List[float],
183 frequency_penalties: List[float],
184 + repetition_penalties: List[float],
185 ) -> torch.Tensor:
186 num_seqs, vocab_size = logits.shape
187 for i in range(num_seqs):
@@ -182,7 +189,9 @@ def _apply_penalties(
189 continue
190 p = presence_penalties[i]
191 f = frequency_penalties[i]
185 - if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS:
192 + r = repetition_penalties[i]
193 + if abs(p) < _SAMPLING_EPS and abs(f) < _SAMPLING_EPS and abs(
194 + r - 1.0) < _SAMPLING_EPS:
195 continue
196 break
197 else:
@@ -206,7 +215,11 @@ def _apply_penalties(
215 bin_counts.scatter_add_(1, output_tokens_tensor,
216 torch.ones_like(output_tokens_tensor))
217 bin_counts = bin_counts[:, :vocab_size] # Remove the padding bin.
218 + mask = bin_counts > 0
219
220 + repetition_penalties = torch.tensor(repetition_penalties,
221 + dtype=logits.dtype,
222 + device=logits.device)
223 frequency_penalties = torch.tensor(frequency_penalties,
224 dtype=logits.dtype,
225 device=logits.device)
@@ -214,10 +227,15 @@ def _apply_penalties(
227 dtype=logits.dtype,
228 device=logits.device)
229
230 + repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
231 + repetition_penalties[~mask] = 1.0
232 + logits = torch.where(logits > 0, logits / repetition_penalties,
233 + logits * repetition_penalties)
234 +
235 # We follow the definition in OpenAI API.
236 # Refer to https://platform.openai.com/docs/api-reference/parameter-details
237 logits -= frequency_penalties.unsqueeze(dim=1) * bin_counts
220 - logits -= presence_penalties.unsqueeze(dim=1) * (bin_counts > 0)
238 + logits -= presence_penalties.unsqueeze(dim=1) * mask
239 return logits
240
241
vllm/sampling_params.py
+10
@@ -34,6 +34,10 @@ class SamplingParams:
34 frequency in the generated text so far. Values > 0 encourage the
35 model to use new tokens, while values < 0 encourage the model to
36 repeat tokens.
37 + repetition_penalty: Float that penalizes new tokens based on whether
38 + they appear in the generated text so far. Values > 1 encourage the
39 + model to use new tokens, while values < 1 encourage the model to
40 + repeat tokens.
41 temperature: Float that controls the randomness of the sampling. Lower
42 values make the model more deterministic, while higher values make
43 the model more random. Zero means greedy sampling.
@@ -75,6 +79,7 @@ class SamplingParams:
79 best_of: Optional[int] = None,
80 presence_penalty: float = 0.0,
81 frequency_penalty: float = 0.0,
82 + repetition_penalty: float = 1.0,
83 temperature: float = 1.0,
84 top_p: float = 1.0,
85 top_k: int = -1,
@@ -93,6 +98,7 @@ class SamplingParams:
98 self.best_of = best_of if best_of is not None else n
99 self.presence_penalty = presence_penalty
100 self.frequency_penalty = frequency_penalty
101 + self.repetition_penalty = repetition_penalty
102 self.temperature = temperature
103 self.top_p = top_p
104 self.top_k = top_k
@@ -136,6 +142,9 @@ class SamplingParams:
142 if not -2.0 <= self.frequency_penalty <= 2.0:
143 raise ValueError("frequency_penalty must be in [-2, 2], got "
144 f"{self.frequency_penalty}.")
145 + if not 0.0 < self.repetition_penalty <= 2.0:
146 + raise ValueError("repetition_penalty must be in (0, 2], got "
147 + f"{self.repetition_penalty}.")
148 if self.temperature < 0.0:
149 raise ValueError(
150 f"temperature must be non-negative, got {self.temperature}.")
@@ -201,6 +210,7 @@ class SamplingParams:
210 f"best_of={self.best_of}, "
211 f"presence_penalty={self.presence_penalty}, "
212 f"frequency_penalty={self.frequency_penalty}, "
213 + f"repetition_penalty={self.repetition_penalty}, "
214 f"temperature={self.temperature}, "
215 f"top_p={self.top_p}, "
216 f"top_k={self.top_k}, "