A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 08:45 UTC
[Frontend] Add `max-completion-token` option to transcription/translation endpoints (#30769)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Dec 16, 2025 at 20:36 UTC
ca702a14dc2d4c5c077dbb8098e66ca244cea185
4 files changed
+79
-2
tests/entrypoints/openai/test_transcription_validation_whisper.py
+32
@@ -244,3 +244,35 @@ async def test_audio_with_timestamp(mary_had_lamb, whisper_client):
244
)
245
assert transcription.segments is not None
246
assert len(transcription.segments) > 0
247
+
248
+
249
+@pytest.mark.asyncio
250
+async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
251
+ transcription = await whisper_client.audio.transcriptions.create(
252
+ model=MODEL_NAME,
253
+ file=mary_had_lamb,
254
+ language="en",
255
+ response_format="text",
256
+ temperature=0.0,
257
+ extra_body={"max_completion_tokens": 1},
258
+ )
259
+ out = json.loads(transcription)
260
+ out_text = out["text"]
261
+ from transformers import AutoTokenizer
262
+
263
+ tok = AutoTokenizer.from_pretrained(MODEL_NAME)
264
+ out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
265
+ assert len(out_tokens) == 1
266
+ # max_completion_tokens > max_model_len
267
+ transcription = await whisper_client.audio.transcriptions.create(
268
+ model=MODEL_NAME,
269
+ file=mary_had_lamb,
270
+ language="en",
271
+ response_format="text",
272
+ temperature=0.0,
273
+ extra_body={"max_completion_tokens": int(1e6)},
274
+ )
275
+ out = json.loads(transcription)
276
+ out_text = out["text"]
277
+ out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
278
+ assert len(out_tokens) < 450 # ~Whisper max output len
tests/entrypoints/openai/test_translation_validation.py
+33
@@ -227,3 +227,36 @@ async def test_long_audio_request(foscolo, client_and_model):
227
)
228
out = json.loads(translation)["text"].strip().lower()
229
assert out.count("greek sea") == 2
230
+
231
+
232
+@pytest.mark.asyncio
233
+async def test_audio_with_max_tokens(mary_had_lamb, client_and_model):
234
+ client, model_name = client_and_model
235
+ transcription = await client.audio.translations.create(
236
+ model=model_name,
237
+ file=mary_had_lamb,
238
+ response_format="text",
239
+ temperature=0.0,
240
+ extra_body={"max_completion_tokens": 1},
241
+ )
242
+ out = json.loads(transcription)
243
+ out_text = out["text"]
244
+ print(out_text)
245
+ from transformers import AutoTokenizer
246
+
247
+ tok = AutoTokenizer.from_pretrained(model_name)
248
+ out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
249
+ assert len(out_tokens) == 1
250
+ # max_completion_tokens > max_model_len
251
+ transcription = await client.audio.transcriptions.create(
252
+ model=model_name,
253
+ file=mary_had_lamb,
254
+ response_format="text",
255
+ temperature=0.0,
256
+ extra_body={"max_completion_tokens": int(1e6)},
257
+ )
258
+ out = json.loads(transcription)
259
+ out_text = out["text"]
260
+ print(out_text)
261
+ out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
262
+ assert len(out_tokens) < 450 # ~Whisper max output len
vllm/entrypoints/openai/protocol.py
+6
@@ -2054,6 +2054,9 @@ class TranscriptionRequest(OpenAIBaseModel):
2054
2055
presence_penalty: float | None = 0.0
2056
"""The presence penalty to use for sampling."""
2057
+
2058
+ max_completion_tokens: int | None = None
2059
+ """The maximum number of tokens to generate."""
2060
# --8<-- [end:transcription-sampling-params]
2061
2062
# Default sampling parameters for transcription requests.
@@ -2300,6 +2303,9 @@ class TranslationRequest(OpenAIBaseModel):
2303
# Flattened stream option to simplify form data.
2304
stream_include_usage: bool | None = False
2305
stream_continuous_usage_stats: bool | None = False
2306
+
2307
+ max_completion_tokens: int | None = None
2308
+ """The maximum number of tokens to generate."""
2309
# --8<-- [end:translation-extra-params]
2310
2311
# Default sampling parameters for translation requests.
vllm/entrypoints/openai/speech_to_text.py
+8
-2
@@ -293,8 +293,14 @@ class OpenAISpeechToText(OpenAIServing):
293
try:
294
# Unlike most decoder-only models, whisper generation length is not
295
# constrained by the size of the input audio, which is mapped to a
296
- # fixed-size log-mel-spectogram.
297
- default_max_tokens = self.model_config.max_model_len
296
+ # fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
297
+ # generated by respecting the extra completion tokens arg.
298
+ if request.max_completion_tokens is None:
299
+ default_max_tokens = self.model_config.max_model_len
300
+ else:
301
+ default_max_tokens = min(
302
+ self.model_config.max_model_len, request.max_completion_tokens
303
+ )
304
sampling_params = request.to_sampling_params(
305
default_max_tokens, self.default_sampling_params
306
)