[Frontend] Support additional sampling parameters for translation API (#45839)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Guan-Ming Chiu committed
Jul 21, 2026 at 13:53 UTC
616c9bd0f4efff9761fd762e4247e18b6bd7eb23
1 file changed
+69
-1
vllm/entrypoints/speech_to_text/translation/protocol.py
+69
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import json
5
import time
6
from typing import TYPE_CHECKING, Literal, TypeAlias
7
@@ -78,7 +79,6 @@ class TranslationRequest(OpenAIBaseModel):
79
`verbose_json`, or `vtt`.
80
"""
81
81
- # TODO support additional sampling parameters
82
# --8<-- [start:translation-sampling-params]
83
use_beam_search: bool = False
84
"""Whether or not beam search should be used."""
@@ -103,6 +103,28 @@ class TranslationRequest(OpenAIBaseModel):
103
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
104
to automatically increase the temperature until certain thresholds are hit.
105
"""
106
+
107
+ top_p: float | None = None
108
+ """Enables nucleus (top-p) sampling, where tokens are selected from the
109
+ smallest possible set whose cumulative probability exceeds `p`.
110
+ """
111
+
112
+ top_k: int | None = None
113
+ """Limits sampling to the `k` most probable tokens at each step."""
114
+
115
+ min_p: float | None = None
116
+ """Filters out tokens with a probability lower than `min_p`, ensuring a
117
+ minimum likelihood threshold during sampling.
118
+ """
119
+
120
+ frequency_penalty: float | None = 0.0
121
+ """The frequency penalty to use for sampling."""
122
+
123
+ repetition_penalty: float | None = None
124
+ """The repetition penalty to use for sampling."""
125
+
126
+ presence_penalty: float | None = 0.0
127
+ """The presence penalty to use for sampling."""
128
# --8<-- [end:translation-sampling-params]
129
130
# --8<-- [start:translation-extra-params]
@@ -139,11 +161,23 @@ class TranslationRequest(OpenAIBaseModel):
161
162
max_completion_tokens: int | None = None
163
"""The maximum number of tokens to generate."""
164
+
165
+ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
166
+ default=None,
167
+ description=(
168
+ "Additional request parameters with (list of) string or "
169
+ "numeric values, used by custom extensions."
170
+ ),
171
+ )
172
# --8<-- [end:translation-extra-params]
173
174
# Default sampling parameters for translation requests.
175
_DEFAULT_SAMPLING_PARAMS: dict = {
176
+ "repetition_penalty": 1.0,
177
"temperature": 0,
178
+ "top_p": 1.0,
179
+ "top_k": 0,
180
+ "min_p": 0.0,
181
}
182
183
def build_stt_params(
@@ -199,14 +233,38 @@ class TranslationRequest(OpenAIBaseModel):
233
temperature = default_sampling_params.get(
234
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
235
)
236
+ if (top_p := self.top_p) is None:
237
+ top_p = default_sampling_params.get(
238
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
239
+ )
240
+ if (top_k := self.top_k) is None:
241
+ top_k = default_sampling_params.get(
242
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
243
+ )
244
+ if (min_p := self.min_p) is None:
245
+ min_p = default_sampling_params.get(
246
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
247
+ )
248
+ if (repetition_penalty := self.repetition_penalty) is None:
249
+ repetition_penalty = default_sampling_params.get(
250
+ "repetition_penalty",
251
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
252
+ )
253
254
return SamplingParams.from_optional(
255
temperature=temperature,
256
max_tokens=max_tokens,
257
seed=self.seed,
258
+ top_p=top_p,
259
+ top_k=top_k,
260
+ min_p=min_p,
261
+ frequency_penalty=self.frequency_penalty,
262
+ repetition_penalty=repetition_penalty,
263
+ presence_penalty=self.presence_penalty,
264
output_kind=RequestOutputKind.DELTA
265
if self.stream
266
else RequestOutputKind.FINAL_ONLY,
267
+ extra_args=self.vllm_xargs,
268
skip_clone=True, # Created fresh per request, safe to skip clone
269
)
270
@@ -226,6 +284,16 @@ class TranslationRequest(OpenAIBaseModel):
284
parameter=invalid_param,
285
)
286
287
+ xargs = data.get("vllm_xargs")
288
+ if isinstance(xargs, str):
289
+ try:
290
+ data["vllm_xargs"] = json.loads(xargs)
291
+ except json.JSONDecodeError as e:
292
+ raise VLLMValidationError(
293
+ f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
294
+ parameter="vllm_xargs",
295
+ ) from e
296
+
297
return data
298
299