A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Misc] Replace TODO in serving transcription (#18895)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
May 29, 2025 at 11:58 UTC
24d0ef89705e0ab8df3d79fcbfd669cf5575772b
1 file changed
+3
-1
vllm/entrypoints/openai/serving_transcription.py
+3
-1
@@ -278,7 +278,9 @@ class OpenAIServingTranscription(OpenAIServing):
278
279
result_generator: Optional[AsyncGenerator[RequestOutput, None]] = None
280
try:
281
- # TODO(rob): subtract len of tokenized prompt.
281
+ # Unlike most decoder-only models, whisper generation length is not
282
+ # constrained by the size of the input audio, which is mapped to a
283
+ # fixed-size log-mel-spectogram.
284
default_max_tokens = self.model_config.max_model_len
285
sampling_params = request.to_sampling_params(
286
default_max_tokens, self.default_sampling_params)