@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4eafc72928
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 20:00 UTC

[Audio] Bundle `get_generation_prompt()` params into `SpeechToTextParams` (#36268)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Ekagra Ranjan committed Apr 22, 2026 at 00:24 UTC 4eafc729285e459a5fc96efd6f7b313b155cad48
18 files changed +214 -161
docs/contributing/model/transcription.md
+19 -20
@@ -66,7 +66,7 @@ This is for controlling general behavior of the API when serving your model:
66
67 See [Audio preprocessing and chunking](#audio-preprocessing-and-chunking) for what each field controls.
68
69 -Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns:
69 +Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server builds a [SpeechToTextParams][vllm.config.speech_to_text.SpeechToTextParams] object that bundles the resampled waveform, task parameters, and request-specific options. Your model receives this single object and returns a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns:
70
71 #### Multimodal LLM with audio embeddings (e.g., Voxtral, Gemma3n)
72
@@ -75,21 +75,20 @@ Return a dict containing `multi_modal_data` with the audio, and either a `prompt
75 ??? code "get_generation_prompt()"
76
77 ```python
78 + from vllm.config.speech_to_text import SpeechToTextParams
79 +
80 class YourASRModel(nn.Module, SupportsTranscription):
81 ...
82
83 @classmethod
84 def get_generation_prompt(
85 cls,
84 - audio: np.ndarray,
85 - stt_config: SpeechToTextConfig,
86 - model_config: ModelConfig,
87 - language: str | None,
88 - task_type: Literal["transcribe", "translate"],
89 - request_prompt: str,
90 - to_language: str | None,
86 + stt_params: SpeechToTextParams,
87 ) -> PromptType:
92 - # Example with a free-form instruction prompt
88 + audio = stt_params.audio
89 + stt_config = stt_params.stt_config
90 + task_type = stt_params.task_type
91 +
92 task_word = "Transcribe" if task_type == "transcribe" else "Translate"
93 prompt = (
94 "<start_of_turn>user\n"
@@ -112,20 +111,22 @@ Return a dict with separate `encoder_prompt` and `decoder_prompt` entries:
111 ??? code "get_generation_prompt()"
112
113 ```python
114 + from vllm.config.speech_to_text import SpeechToTextParams
115 +
116 class YourASRModel(nn.Module, SupportsTranscription):
117 ...
118
119 @classmethod
120 def get_generation_prompt(
121 cls,
121 - audio: np.ndarray,
122 - stt_config: SpeechToTextConfig,
123 - model_config: ModelConfig,
124 - language: str | None,
125 - task_type: Literal["transcribe", "translate"],
126 - request_prompt: str,
127 - to_language: str | None,
122 + stt_params: SpeechToTextParams,
123 ) -> PromptType:
124 + audio = stt_params.audio
125 + stt_config = stt_params.stt_config
126 + language = stt_params.language
127 + task_type = stt_params.task_type
128 + request_prompt = stt_params.request_prompt
129 +
130 if language is None:
131 raise ValueError("Language must be specified")
132
@@ -213,15 +214,13 @@ Relevant server logic:
214 chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))
215 prompts = []
216 for chunk in chunks:
216 - prompt = self.model_cls.get_generation_prompt(
217 + stt_params = request.build_stt_params(
218 audio=chunk,
219 stt_config=self.asr_config,
220 model_config=self.model_config,
220 - language=language,
221 task_type=self.task_type,
222 - request_prompt=request.prompt,
223 - to_language=to_language,
222 )
223 + prompt = self.model_cls.get_generation_prompt(stt_params)
224 prompts.append(prompt)
225 return prompts, duration
226 ```
vllm/config/__init__.py
+2 -1
@@ -37,7 +37,7 @@ from vllm.config.profiler import ProfilerConfig
37 from vllm.config.reasoning import ReasoningConfig
38 from vllm.config.scheduler import SchedulerConfig
39 from vllm.config.speculative import SpeculativeConfig
40 -from vllm.config.speech_to_text import SpeechToTextConfig
40 +from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
41 from vllm.config.structured_outputs import StructuredOutputsConfig
42 from vllm.config.utils import (
43 ConfigType,
@@ -113,6 +113,7 @@ __all__ = [
113 "SpeculativeConfig",
114 # From vllm.config.speech_to_text
115 "SpeechToTextConfig",
116 + "SpeechToTextParams",
117 # From vllm.config.structured_outputs
118 "StructuredOutputsConfig",
119 # From vllm.config.profiler
vllm/config/speech_to_text.py
+40
@@ -1,9 +1,49 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from __future__ import annotations
4
5 +from dataclasses import dataclass
6 +from typing import TYPE_CHECKING
7
8 from vllm.config.utils import config
9
10 +if TYPE_CHECKING:
11 + import numpy as np
12 +
13 + from vllm.config.model import ModelConfig
14 +
15 +
16 +@dataclass
17 +class SpeechToTextParams:
18 + """All parameters consumed by ``get_generation_prompt()``.
19 +
20 + ``TranscriptionRequest.build_stt_params()`` constructs this object,
21 + mapping API-level fields into typed attributes. Models only receive
22 + this object, so new parameters can be added here without changing the
23 + ``get_generation_prompt`` signature.
24 + """
25 +
26 + audio: np.ndarray
27 + """Resampled audio waveform for a single chunk."""
28 +
29 + stt_config: SpeechToTextConfig
30 + """Server-level speech-to-text configuration."""
31 +
32 + model_config: ModelConfig
33 + """Model configuration."""
34 +
35 + language: str | None = None
36 + """ISO 639-1 language code (validated / auto-detected)."""
37 +
38 + task_type: str = "transcribe"
39 + """``"transcribe"`` or ``"translate"``."""
40 +
41 + request_prompt: str = ""
42 + """Optional text prompt to guide the model."""
43 +
44 + to_language: str | None = None
45 + """Target language for translation (model-dependent)."""
46 +
47
48 @config
49 class SpeechToTextConfig:
vllm/entrypoints/openai/speech_to_text/protocol.py
+53 -1
@@ -1,9 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +import json
5 import time
6 from http import HTTPStatus
6 -from typing import Literal, TypeAlias
7 +from typing import TYPE_CHECKING, Literal, TypeAlias
8
9 import torch
10 from fastapi import HTTPException, UploadFile
@@ -12,6 +13,7 @@ from pydantic import (
13 model_validator,
14 )
15
16 +from vllm.config.speech_to_text import SpeechToTextParams
17 from vllm.entrypoints.openai.engine.protocol import (
18 DeltaMessage,
19 OpenAIBaseModel,
@@ -26,6 +28,11 @@ from vllm.sampling_params import (
28 )
29 from vllm.utils import random_uuid
30
31 +if TYPE_CHECKING:
32 + import numpy as np
33 +
34 + from vllm.config import ModelConfig, SpeechToTextConfig
35 +
36 logger = init_logger(__name__)
37 _LONG_INFO = torch.iinfo(torch.long)
38
@@ -183,6 +190,23 @@ class TranscriptionRequest(OpenAIBaseModel):
190 "min_p": 0.0,
191 }
192
193 + def build_stt_params(
194 + self,
195 + audio: "np.ndarray",
196 + stt_config: "SpeechToTextConfig",
197 + model_config: "ModelConfig",
198 + task_type: str,
199 + ) -> SpeechToTextParams:
200 + return SpeechToTextParams(
201 + audio=audio,
202 + stt_config=stt_config,
203 + model_config=model_config,
204 + language=self.language,
205 + task_type=task_type,
206 + request_prompt=self.prompt,
207 + to_language=self.to_language,
208 + )
209 +
210 def to_beam_search_params(
211 self,
212 default_max_tokens: int,
@@ -277,6 +301,17 @@ class TranscriptionRequest(OpenAIBaseModel):
301 parameter=invalid_param,
302 )
303
304 + # Parse vllm_xargs from JSON string (form data sends it as a string)
305 + xargs = data.get("vllm_xargs")
306 + if isinstance(xargs, str):
307 + try:
308 + data["vllm_xargs"] = json.loads(xargs)
309 + except json.JSONDecodeError as e:
310 + raise VLLMValidationError(
311 + f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
312 + parameter="vllm_xargs",
313 + ) from e
314 +
315 return data
316
317
@@ -472,6 +507,23 @@ class TranslationRequest(OpenAIBaseModel):
507 "temperature": 0,
508 }
509
510 + def build_stt_params(
511 + self,
512 + audio: "np.ndarray",
513 + stt_config: "SpeechToTextConfig",
514 + model_config: "ModelConfig",
515 + task_type: str,
516 + ) -> SpeechToTextParams:
517 + return SpeechToTextParams(
518 + audio=audio,
519 + stt_config=stt_config,
520 + model_config=model_config,
521 + language=self.language,
522 + task_type=task_type,
523 + request_prompt=self.prompt,
524 + to_language=self.to_language,
525 + )
526 +
527 def to_beam_search_params(
528 self,
529 default_max_tokens: int,
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+6 -12
@@ -184,9 +184,8 @@ class OpenAISpeechToText(OpenAIServing):
184 request_id: str,
185 ) -> tuple[list[EngineInput], float]:
186 # Validate request
187 - language = self.model_cls.validate_language(request.language)
188 - # Skip to_language validation to avoid extra logging for Whisper.
189 - to_language = (
187 + request.language = self.model_cls.validate_language(request.language)
188 + request.to_language = (
189 self.model_cls.validate_language(request.to_language)
190 if request.to_language
191 else None
@@ -229,28 +228,23 @@ class OpenAISpeechToText(OpenAIServing):
228 min_energy_window_size=self.asr_config.min_energy_split_window_size,
229 )
230
232 - if language is None and getattr(
231 + if request.language is None and getattr(
232 self.model_cls, "supports_explicit_language_detection", False
233 ):
234 # Auto-detect language from the first chunk.
236 - language = await self._detect_language(
235 + request.language = await self._detect_language(
236 chunks[0], f"{request_id}-lang_detect"
237 )
239 - request.language = language
238
239 parsed_prompts: list[DictPrompt] = []
240 for chunk in chunks:
243 - # The model has control over the construction, as long as it
244 - # returns a valid PromptType.
245 - prompt = self.model_cls.get_generation_prompt(
241 + stt_params = request.build_stt_params(
242 audio=chunk,
243 stt_config=self.asr_config,
244 model_config=self.model_config,
249 - language=language,
245 task_type=self.task_type,
251 - request_prompt=request.prompt,
252 - to_language=to_language,
246 )
247 + prompt = self.model_cls.get_generation_prompt(stt_params)
248
249 parsed_prompt: DictPrompt
250 if request.response_format == "verbose_json":
vllm/model_executor/models/cohere_asr.py
+9 -13
@@ -3,9 +3,7 @@
3
4 import math
5 from collections.abc import Iterable, Mapping, Sequence
6 -from typing import Literal
6
8 -import numpy as np
7 import torch
8 import torch.nn.functional as F
9 from torch import nn
@@ -13,6 +11,8 @@ from transformers import PretrainedConfig
11
12 from vllm.compilation.decorators import support_torch_compile
13 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
14 +from vllm.config.multimodal import BaseDummyOptions
15 +from vllm.config.speech_to_text import SpeechToTextParams
16 from vllm.distributed import get_tensor_model_parallel_world_size
17 from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
18 from vllm.logger import init_logger
@@ -1900,7 +1900,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo
1900 self,
1901 seq_len: int,
1902 mm_counts: Mapping[str, int],
1903 - mm_options=None,
1903 + mm_options: Mapping[str, BaseDummyOptions],
1904 mm_processor_kwargs=None,
1905 ) -> MultiModalDataDict:
1906 feature_extractor = self.info.get_feature_extractor()
@@ -2021,16 +2021,12 @@ class CohereAsrForConditionalGeneration(
2021 return super().validate_language(language)
2022
2023 @classmethod
2024 - def get_generation_prompt(
2025 - cls,
2026 - audio: np.ndarray,
2027 - model_config: ModelConfig, # not needed here
2028 - stt_config: SpeechToTextConfig,
2029 - language: str | None,
2030 - task_type: Literal["transcribe", "translate"],
2031 - request_prompt: str,
2032 - to_language: str | None,
2033 - ) -> PromptType:
2024 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
2025 + audio = stt_params.audio
2026 + stt_config = stt_params.stt_config
2027 + language = stt_params.language
2028 + request_prompt = stt_params.request_prompt
2029 +
2030 if language is None:
2031 raise ValueError(
2032 "Language must be specified when creating the CohereASR prompt"
vllm/model_executor/models/fireredasr2.py
+7 -9
@@ -2,9 +2,8 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import math
4 from collections.abc import Iterable, Mapping, Sequence
5 -from typing import Annotated, Literal, cast
5 +from typing import Annotated, cast
6
7 -import numpy as np
7 import torch
8 from torch import nn
9 from transformers import (
@@ -14,6 +13,7 @@ from transformers import (
13
14 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
15 from vllm.config.multimodal import BaseDummyOptions
16 +from vllm.config.speech_to_text import SpeechToTextParams
17 from vllm.inputs import MultiModalDataDict, PromptType
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
@@ -356,14 +356,12 @@ class FireRedASR2ForConditionalGeneration(
356 @classmethod
357 def get_generation_prompt(
358 cls,
359 - audio: np.ndarray,
360 - model_config: ModelConfig, # not needed here
361 - stt_config: SpeechToTextConfig,
362 - language: str | None,
363 - task_type: Literal["transcribe", "translate"],
364 - request_prompt: str,
365 - to_language: str | None,
359 + stt_params: SpeechToTextParams,
360 ) -> PromptType:
361 + audio = stt_params.audio
362 + stt_config = stt_params.stt_config
363 + language = stt_params.language
364 +
365 if language is None:
366 raise ValueError(
367 "Language must be specified when creating the fireredasr2 prompt"
vllm/model_executor/models/funasr.py
+7 -9
@@ -3,9 +3,8 @@
3
4 import math
5 from collections.abc import Iterable, Mapping, Sequence
6 -from typing import Annotated, Literal, cast
6 +from typing import Annotated, cast
7
8 -import numpy as np
8 import torch
9 import torch.nn.functional as F
10 from torch import nn
@@ -16,6 +15,7 @@ from transformers import (
15
16 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
17 from vllm.config.multimodal import BaseDummyOptions
18 +from vllm.config.speech_to_text import SpeechToTextParams
19 from vllm.distributed import get_tensor_model_parallel_world_size
20 from vllm.inputs import MultiModalDataDict, PromptType
21 from vllm.logger import init_logger
@@ -876,14 +876,12 @@ class FunASRForConditionalGeneration(
876 @classmethod
877 def get_generation_prompt(
878 cls,
879 - audio: np.ndarray,
880 - model_config: ModelConfig, # not needed here
881 - stt_config: SpeechToTextConfig,
882 - language: str | None,
883 - task_type: Literal["transcribe", "translate"],
884 - request_prompt: str,
885 - to_language: str | None,
879 + stt_params: SpeechToTextParams,
880 ) -> PromptType:
881 + audio = stt_params.audio
882 + stt_config = stt_params.stt_config
883 + language = stt_params.language
884 +
885 if language is None:
886 raise ValueError(
887 "Language must be specified when creating the funasr prompt"
vllm/model_executor/models/gemma3n_mm.py
+7 -11
@@ -3,7 +3,6 @@
3 from collections.abc import Iterable, Mapping, Sequence
4 from typing import Annotated, Any, Literal
5
6 -import numpy as np
6 import torch
7 from torch import nn
8 from transformers import AutoModel, BatchFeature
@@ -19,6 +18,7 @@ from transformers.models.siglip import SiglipImageProcessorFast
18
19 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
20 from vllm.config.multimodal import BaseDummyOptions
21 +from vllm.config.speech_to_text import SpeechToTextParams
22 from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
23 from vllm.logger import init_logger
24 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -769,21 +769,17 @@ class Gemma3nForConditionalGeneration(
769 raise ValueError(f"Unsupported modality: {modality}")
770
771 @classmethod
772 - def get_generation_prompt(
773 - cls,
774 - audio: np.ndarray,
775 - stt_config: SpeechToTextConfig,
776 - model_config: ModelConfig,
777 - language: str | None,
778 - task_type: Literal["transcribe", "translate"],
779 - request_prompt: str,
780 - to_language: str | None,
781 - ) -> PromptType:
772 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
773 """
774 Gemma3n supports "free-form" transcription.
775 We fix its prompt here to standardize transcriptions/translations
776 requests.
777 """
778 + audio = stt_params.audio
779 + stt_config = stt_params.stt_config
780 + language = stt_params.language
781 + task_type = stt_params.task_type
782 + to_language = stt_params.to_language
783 # Transcribe this audio [into <>] | for transcription
784 # Translate this audio [from <> into <>] | for translation
785 prompt = "<start_of_turn>user\n"
vllm/model_executor/models/glmasr.py
+6 -11
@@ -4,7 +4,6 @@
4 from collections.abc import Iterable, Mapping, Sequence
5 from typing import Annotated, Any, Literal, TypeAlias
6
7 -import numpy as np
7 import torch
8 import torch.nn as nn
9 from transformers import BatchFeature
@@ -13,6 +12,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
12
13 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
14 from vllm.config.multimodal import BaseDummyOptions
15 +from vllm.config.speech_to_text import SpeechToTextParams
16 from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
17 from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
18 from vllm.model_executor.layers.activation import get_act_fn
@@ -1131,17 +1131,12 @@ class GlmAsrForConditionalGeneration(
1131 )
1132
1133 @classmethod
1134 - def get_generation_prompt(
1135 - cls,
1136 - audio: np.ndarray,
1137 - model_config: ModelConfig,
1138 - stt_config: SpeechToTextConfig,
1139 - language: str | None,
1140 - task_type: Literal["transcribe", "translate"],
1141 - request_prompt: str,
1142 - to_language: str | None,
1143 - ) -> PromptType:
1134 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
1135 """Get the generation prompt to be used for transcription requests."""
1136 + audio = stt_params.audio
1137 + model_config = stt_params.model_config
1138 + task_type = stt_params.task_type
1139 + to_language = stt_params.to_language
1140 tokenizer = cached_tokenizer_from_config(model_config)
1141 audio_token = cls._get_audio_token(model_config)
1142
vllm/model_executor/models/granite_speech.py
+8 -9
@@ -26,9 +26,8 @@
26
27 import math
28 from collections.abc import Iterable, Mapping
29 -from typing import Annotated, Literal
29 +from typing import Annotated
30
31 -import numpy as np
31 import torch
32 import torch.nn.functional as F
33 from torch import nn
@@ -36,6 +35,7 @@ from transformers import BatchFeature, PretrainedConfig
35
36 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
37 from vllm.config.multimodal import BaseDummyOptions
38 +from vllm.config.speech_to_text import SpeechToTextParams
39 from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
40 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
41 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -852,15 +852,14 @@ class GraniteSpeechForConditionalGeneration(
852 @classmethod
853 def get_generation_prompt(
854 cls,
855 - audio: np.ndarray,
856 - model_config: ModelConfig,
857 - stt_config: SpeechToTextConfig,
858 - language: str | None,
859 - task_type: Literal["transcribe", "translate"],
860 - request_prompt: str,
861 - to_language: str | None,
855 + stt_params: SpeechToTextParams,
856 ) -> PromptType:
857 """Get the generation prompt to be used for transcription requests."""
858 + audio = stt_params.audio
859 + model_config = stt_params.model_config
860 + task_type = stt_params.task_type
861 + to_language = stt_params.to_language
862 +
863 # Audio placeholders don't use an index, so value doesn't matter
864 audio_tok = cls.get_placeholder_str("audio", 0)
865
vllm/model_executor/models/interfaces.py
+2 -8
@@ -29,7 +29,7 @@ from torch import Tensor
29 from transformers.models.whisper.tokenization_whisper import LANGUAGES
30 from typing_extensions import Self, TypeIs
31
32 -from vllm.config import ModelConfig, SpeechToTextConfig
32 +from vllm.config import ModelConfig, SpeechToTextConfig, SpeechToTextParams
33 from vllm.inputs import PromptType, TokensPrompt
34 from vllm.logger import init_logger
35 from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
@@ -1119,13 +1119,7 @@ class SupportsTranscription(Protocol):
1119 @classmethod
1120 def get_generation_prompt(
1121 cls,
1122 - audio: np.ndarray,
1123 - stt_config: SpeechToTextConfig,
1124 - model_config: ModelConfig,
1125 - language: str | None,
1126 - task_type: Literal["transcribe", "translate"],
1127 - request_prompt: str,
1128 - to_language: str | None,
1122 + stt_params: SpeechToTextParams,
1123 ) -> PromptType:
1124 """Get the prompt for the ASR model.
1125 The model has control over the construction, as long as it
vllm/model_executor/models/kimi_audio.py
+7 -10
@@ -14,6 +14,7 @@ from transformers import WhisperConfig as HFWhisperConfig
14
15 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
16 from vllm.config.multimodal import BaseDummyOptions
17 +from vllm.config.speech_to_text import SpeechToTextParams
18 from vllm.inputs import PromptType, TokensPrompt
19 from vllm.model_executor.model_loader import DefaultModelLoader
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -626,16 +627,12 @@ class KimiAudioForConditionalGeneration(
627 )
628
629 @classmethod
629 - def get_generation_prompt(
630 - cls,
631 - audio: np.ndarray,
632 - model_config: ModelConfig,
633 - stt_config: SpeechToTextConfig,
634 - language: str | None,
635 - task_type: Literal["transcribe", "translate"],
636 - request_prompt: str,
637 - to_language: str | None,
638 - ) -> PromptType:
630 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
631 + audio = stt_params.audio
632 + model_config = stt_params.model_config
633 + task_type = stt_params.task_type
634 + request_prompt = stt_params.request_prompt
635 +
636 tokenizer = cached_get_tokenizer(
637 model_config.tokenizer,
638 tokenizer_cls=KimiAudioTokenizer,
vllm/model_executor/models/qwen3_asr.py
+7 -12
@@ -23,9 +23,8 @@
23 """Inference-only Qwen3-ASR model."""
24
25 from collections.abc import Iterable, Mapping, Sequence
26 -from typing import Any, Literal
26 +from typing import Any
27
28 -import numpy as np
28 import torch
29 import torch.nn as nn
30 from transformers.feature_extraction_utils import BatchFeature
@@ -33,6 +32,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
32
33 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
34 from vllm.config.multimodal import BaseDummyOptions
35 +from vllm.config.speech_to_text import SpeechToTextParams
36 from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
37 from vllm.logger import init_logger
38 from vllm.model_executor.models.interfaces import (
@@ -549,17 +549,12 @@ class Qwen3ASRForConditionalGeneration(
549 )
550
551 @classmethod
552 - def get_generation_prompt(
553 - cls,
554 - audio: np.ndarray,
555 - model_config: ModelConfig,
556 - stt_config: SpeechToTextConfig,
557 - language: str | None,
558 - task_type: Literal["transcribe", "translate"],
559 - request_prompt: str,
560 - to_language: str | None,
561 - ) -> PromptType:
552 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
553 """Get the generation prompt to be used for transcription requests."""
554 + audio = stt_params.audio
555 + model_config = stt_params.model_config
556 + task_type = stt_params.task_type
557 + to_language = stt_params.to_language
558 tokenizer = cached_tokenizer_from_config(model_config)
559 audio_placeholder = cls.get_placeholder_str("audio", 0)
560
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+10 -11
@@ -24,7 +24,7 @@
24
25 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
26 from functools import partial
27 -from typing import Any, Literal, cast
27 +from typing import Any, cast
28
29 import numpy as np
30 import torch
@@ -46,6 +46,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
46
47 from vllm.compilation.decorators import support_torch_compile
48 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
49 +from vllm.config.speech_to_text import SpeechToTextParams
50 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
51 from vllm.inputs import PromptType
52 from vllm.logger import init_logger
@@ -2201,19 +2202,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2202 )
2203
2204 @classmethod
2204 - def get_generation_prompt(
2205 - cls,
2206 - audio: np.ndarray,
2207 - stt_config: SpeechToTextConfig,
2208 - model_config: ModelConfig,
2209 - language: str | None,
2210 - task_type: Literal["transcribe", "translate"],
2211 - request_prompt: str,
2212 - to_language: str | None,
2213 - ) -> PromptType:
2205 + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
2206 """
2207 Construct a transcription/translation prompt for Qwen3-Omni.
2208 """
2209 + audio = stt_params.audio
2210 + stt_config = stt_params.stt_config
2211 + model_config = stt_params.model_config
2212 + language = stt_params.language
2213 + task_type = stt_params.task_type
2214 + to_language = stt_params.to_language
2215 + request_prompt = stt_params.request_prompt
2216 # Transcribe this audio [into <language>] | for transcription
2217 # Translate this audio [from <language> into <to_language>] | for translation
2218 instruction = "Transcribe" if task_type == "transcribe" else "Translate"
vllm/model_executor/models/voxtral.py
+8 -8
@@ -4,7 +4,7 @@
4 import math
5 from collections.abc import Iterable, Mapping, Sequence
6 from functools import partial
7 -from typing import Literal, cast
7 +from typing import cast
8
9 import numpy as np
10 import regex as re
@@ -19,6 +19,7 @@ from transformers import BatchFeature, WhisperConfig
19
20 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
21 from vllm.config.multimodal import BaseDummyOptions
22 +from vllm.config.speech_to_text import SpeechToTextParams
23 from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
24 from vllm.logger import init_logger
25 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -446,14 +447,13 @@ class VoxtralForConditionalGeneration(
447 # for speech-to-text transcription
448 def get_generation_prompt(
449 cls,
449 - audio: np.ndarray,
450 - model_config: ModelConfig,
451 - stt_config: SpeechToTextConfig,
452 - language: str | None,
453 - task_type: Literal["transcribe", "translate"],
454 - request_prompt: str,
455 - to_language: str | None,
450 + stt_params: SpeechToTextParams,
451 ) -> PromptType:
452 + audio = stt_params.audio
453 + model_config = stt_params.model_config
454 + stt_config = stt_params.stt_config
455 + language = stt_params.language
456 +
457 tokenizer = cached_tokenizer_from_config(model_config)
458 audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
459 req = TranscriptionRequest(
vllm/model_executor/models/voxtral_realtime.py
+7 -8
@@ -4,7 +4,6 @@
4 import asyncio
5 import math
6 from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping
7 -from typing import Literal
7
8 import numpy as np
9 import torch
@@ -18,6 +17,7 @@ from mistral_common.tokens.tokenizers.audio import AudioConfig
17
18 from vllm.compilation.decorators import support_torch_compile
19 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
20 +from vllm.config.speech_to_text import SpeechToTextParams
21 from vllm.engine.protocol import StreamingInput
22 from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S
23 from vllm.inputs import PromptType, TokensPrompt
@@ -465,14 +465,13 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
465 # for speech-to-text transcription
466 def get_generation_prompt(
467 cls,
468 - audio: np.ndarray,
469 - model_config: ModelConfig,
470 - stt_config: SpeechToTextConfig,
471 - language: str | None,
472 - task_type: Literal["transcribe", "translate"],
473 - request_prompt: str,
474 - to_language: str | None,
468 + stt_params: SpeechToTextParams,
469 ) -> PromptType:
470 + audio = stt_params.audio
471 + model_config = stt_params.model_config
472 + stt_config = stt_params.stt_config
473 + language = stt_params.language
474 +
475 tokenizer = cached_tokenizer_from_config(model_config)
476 audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
477
vllm/model_executor/models/whisper.py
+9 -8
@@ -5,7 +5,7 @@ import enum
5 import math
6 from collections.abc import Iterable, Mapping, Sequence
7 from contextlib import nullcontext
8 -from typing import Annotated, Literal
8 +from typing import Annotated
9
10 import numpy as np
11 import torch
@@ -20,6 +20,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
20 from vllm.compilation.decorators import support_torch_compile
21 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
22 from vllm.config.multimodal import BaseDummyOptions
23 +from vllm.config.speech_to_text import SpeechToTextParams
24 from vllm.distributed import get_tensor_model_parallel_world_size
25 from vllm.inputs import (
26 ExplicitEncoderDecoderPrompt,
@@ -830,14 +831,14 @@ class WhisperForConditionalGeneration(
831 @classmethod
832 def get_generation_prompt(
833 cls,
833 - audio: np.ndarray,
834 - model_config: ModelConfig, # not needed here
835 - stt_config: SpeechToTextConfig,
836 - language: str | None,
837 - task_type: Literal["transcribe", "translate"],
838 - request_prompt: str,
839 - to_language: str | None,
834 + stt_params: SpeechToTextParams,
835 ) -> PromptType:
836 + audio = stt_params.audio
837 + stt_config = stt_params.stt_config
838 + language = stt_params.language
839 + task_type = stt_params.task_type
840 + request_prompt = stt_params.request_prompt
841 +
842 if language is None:
843 raise ValueError(
844 "Language must be specified when creating the Whisper prompt"