A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 22:45 UTC
[Frontend] Add automatic language detection for Whisper transcription (#34342)
Signed-off-by: space_check <roman.vuskov@rwth-aachen.de> Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
Roman committed
Feb 21, 2026 at 13:49 UTC
98b0205c3c934849d96922e162e65f3178e0886b
5 files changed
+249
-10
tests/entrypoints/openai/test_transcription_validation_whisper.py
+27
@@ -273,3 +273,30 @@ async def test_audio_with_max_tokens(whisper_client, mary_had_lamb):
273
out_text = out["text"]
274
out_tokens = tok(out_text, add_special_tokens=False)["input_ids"]
275
assert len(out_tokens) < 450 # ~Whisper max output len
276
+
277
+
278
+@pytest.mark.asyncio
279
+@pytest.mark.parametrize(
280
+ ("fixture_name", "expected_lang", "expected_text"),
281
+ [
282
+ ("mary_had_lamb", "en", ["Mary had a little lamb"]),
283
+ ("foscolo", "it", ["zacinto", "sacre"]),
284
+ ],
285
+ ids=["english", "italian"],
286
+)
287
+async def test_language_auto_detect(
288
+ whisper_client, fixture_name, expected_lang, expected_text, request
289
+):
290
+ """Auto-detect language when no language param is provided."""
291
+ audio_file = request.getfixturevalue(fixture_name)
292
+ transcription = await whisper_client.audio.transcriptions.create(
293
+ model=MODEL_NAME,
294
+ file=audio_file,
295
+ response_format="verbose_json",
296
+ temperature=0.0,
297
+ )
298
+ assert transcription.language == expected_lang
299
+ text_lower = transcription.text.lower()
300
+ assert any(word.lower() in text_lower for word in expected_text), (
301
+ f"Expected {expected_lang} text but got: {transcription.text}"
302
+ )
tests/models/multimodal/generation/test_whisper.py
+41
@@ -111,6 +111,47 @@ def check_model_available(model: str) -> None:
111
model_info.check_transformers_version(on_fail="skip")
112
113
114
+def test_parse_language_detection_output():
115
+ """Unit test for WhisperForConditionalGeneration.parse_language_detection_output.
116
+
117
+ No GPU or model loading required.
118
+ """
119
+ from unittest.mock import MagicMock
120
+
121
+ from vllm.model_executor.models.whisper import (
122
+ WhisperForConditionalGeneration,
123
+ )
124
+
125
+ cls = WhisperForConditionalGeneration
126
+
127
+ def make_tokenizer(return_value: str) -> MagicMock:
128
+ tok = MagicMock()
129
+ tok.decode = MagicMock(return_value=return_value)
130
+ return tok
131
+
132
+ # English
133
+ assert (
134
+ cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en"
135
+ )
136
+
137
+ # German
138
+ assert (
139
+ cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de"
140
+ )
141
+
142
+ # Unsupported language code
143
+ with pytest.raises(AssertionError):
144
+ cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>"))
145
+
146
+ # No special token format
147
+ with pytest.raises(AssertionError):
148
+ cls.parse_language_detection_output([1], make_tokenizer("hello"))
149
+
150
+ # Empty token_ids
151
+ with pytest.raises((AssertionError, IndexError)):
152
+ cls.parse_language_detection_output([], make_tokenizer("anything"))
153
+
154
+
155
@pytest.mark.core_model
156
@pytest.mark.cpu_model
157
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+61
-1
@@ -41,7 +41,10 @@ from vllm.exceptions import VLLMValidationError
41
from vllm.inputs import ProcessorInputs
42
from vllm.logger import init_logger
43
from vllm.logprobs import FlatLogprobs, Logprob
44
-from vllm.model_executor.models import SupportsTranscription, supports_transcription
44
+from vllm.model_executor.models import (
45
+ SupportsTranscription,
46
+ supports_transcription,
47
+)
48
from vllm.outputs import RequestOutput
49
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
50
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
@@ -242,10 +245,57 @@ class OpenAISpeechToText(OpenAIServing):
245
model_cls = get_model_cls(self.model_config)
246
return cast(type[SupportsTranscription], model_cls)
247
248
+ async def _detect_language(
249
+ self,
250
+ audio_chunk: np.ndarray,
251
+ request_id: str,
252
+ ) -> str:
253
+ """Auto-detect the spoken language from an audio chunk.
254
+
255
+ Delegates prompt construction and output parsing to the model class
256
+ via ``get_language_detection_prompt`` and
257
+ ``parse_language_detection_output``.
258
+ """
259
+ from vllm.sampling_params import SamplingParams
260
+
261
+ prompt = self.model_cls.get_language_detection_prompt(
262
+ audio_chunk,
263
+ self.asr_config,
264
+ )
265
+ allowed_token_ids = self.model_cls.get_language_token_ids(
266
+ self.tokenizer,
267
+ )
268
+ sampling_params = SamplingParams(
269
+ max_tokens=1,
270
+ temperature=0.0,
271
+ allowed_token_ids=allowed_token_ids,
272
+ )
273
+
274
+ result_generator = self.engine_client.generate(
275
+ prompt,
276
+ sampling_params,
277
+ request_id,
278
+ )
279
+
280
+ final_output: RequestOutput
281
+ async for final_output in result_generator:
282
+ if final_output.finished:
283
+ break
284
+
285
+ token_ids = list(final_output.outputs[0].token_ids)
286
+ lang = self.model_cls.parse_language_detection_output(
287
+ token_ids,
288
+ self.tokenizer,
289
+ )
290
+
291
+ logger.info("Auto-detected language: '%s'", lang)
292
+ return lang
293
+
294
async def _preprocess_speech_to_text(
295
self,
296
request: SpeechToTextRequest,
297
audio_data: bytes,
298
+ request_id: str,
299
) -> tuple[list[ProcessorInputs], float]:
300
# Validate request
301
language = self.model_cls.validate_language(request.language)
@@ -274,6 +324,15 @@ class OpenAISpeechToText(OpenAIServing):
324
and duration > self.asr_config.max_audio_clip_s
325
)
326
chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))
327
+
328
+ if language is None and getattr(
329
+ self.model_cls, "supports_explicit_language_detection", False
330
+ ):
331
+ language = await self._detect_language(
332
+ chunks[0], f"{request_id}-lang_detect"
333
+ )
334
+ request.language = language
335
+
336
parsed_prompts: list[DictPrompt] = []
337
for chunk in chunks:
338
# The model has control over the construction, as long as it
@@ -435,6 +494,7 @@ class OpenAISpeechToText(OpenAIServing):
494
engine_prompts, duration_s = await self._preprocess_speech_to_text(
495
request=request,
496
audio_data=audio_data,
497
+ request_id=request_id,
498
)
499
500
except ValueError as e:
vllm/model_executor/models/interfaces.py
+50
@@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol):
1111
Enables the segment timestamp option for supported models by setting this to `True`.
1112
"""
1113
1114
+ supports_explicit_language_detection: ClassVar[bool] = False
1115
+ """
1116
+ Transcription models that require an explicit language detection step
1117
+ (e.g. Whisper needs a separate forward pass to predict the language
1118
+ token) should set this to ``True`` and implement
1119
+ :meth:`get_language_detection_prompt` and
1120
+ :meth:`parse_language_detection_output` and
1121
+ :meth:`get_language_token_ids`.
1122
+ """
1123
+
1124
def __init_subclass__(cls, **kwargs):
1125
super().__init_subclass__(**kwargs)
1126
# language codes in supported_languages
@@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol):
1216
"""
1217
return text
1218
1219
+ @classmethod
1220
+ def get_language_detection_prompt(
1221
+ cls,
1222
+ audio: np.ndarray,
1223
+ stt_config: SpeechToTextConfig,
1224
+ ) -> PromptType:
1225
+ """Return a prompt that triggers language detection.
1226
+
1227
+ Only needs to be implemented when
1228
+ ``supports_explicit_language_detection`` is ``True``.
1229
+ """
1230
+ raise NotImplementedError
1231
+
1232
+ @classmethod
1233
+ def parse_language_detection_output(
1234
+ cls,
1235
+ token_ids: list[int],
1236
+ tokenizer: object,
1237
+ ) -> str:
1238
+ """Parse the detected language from model output token IDs.
1239
+
1240
+ Only needs to be implemented when
1241
+ ``supports_explicit_language_detection`` is ``True``.
1242
+ """
1243
+ raise NotImplementedError
1244
+
1245
+ @classmethod
1246
+ def get_language_token_ids(
1247
+ cls,
1248
+ tokenizer: object,
1249
+ ) -> list[int] | None:
1250
+ """Return token IDs that represent valid language tokens.
1251
+
1252
+ Used to constrain language detection to only produce valid language tokens.
1253
+
1254
+ Only needs to be implemented when
1255
+ ``supports_explicit_language_detection`` is ``True``.
1256
+ """
1257
+ raise NotImplementedError
1258
+
1259
1260
@overload
1261
def supports_transcription(
vllm/model_executor/models/whisper.py
+70
-9
@@ -64,7 +64,11 @@ from vllm.v1.attention.backend import (
64
AttentionType,
65
)
66
67
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
67
+from .interfaces import (
68
+ MultiModalEmbeddings,
69
+ SupportsMultiModal,
70
+ SupportsTranscription,
71
+)
72
from .utils import (
73
AutoWeightsLoader,
74
WeightsMapper,
@@ -784,7 +788,9 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
788
dummy_inputs=WhisperDummyInputsBuilder,
789
)
790
class WhisperForConditionalGeneration(
787
- nn.Module, SupportsTranscription, SupportsMultiModal
791
+ nn.Module,
792
+ SupportsTranscription,
793
+ SupportsMultiModal,
794
):
795
packed_modules_mapping = {
796
"self_attn.qkv_proj": [
@@ -802,20 +808,18 @@ class WhisperForConditionalGeneration(
808
# Whisper only supports audio-conditioned generation.
809
supports_transcription_only = True
810
supports_segment_timestamp = True
811
+ supports_explicit_language_detection = True
812
supported_languages = ISO639_1_SUPPORTED_LANGS
813
814
@classmethod
815
def validate_language(cls, language: str | None) -> str | None:
816
if language is None:
810
- # TODO language should be optional and can be guessed.
811
- # For now we default to en. See
812
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
813
- logger.warning(
814
- "Defaulting to language='en'. If you wish to transcribe "
815
- "audio in a different language, pass the `language` field "
817
+ logger.debug(
818
+ "No language specified. Language will be auto-detected "
819
+ "from audio. To skip detection, pass the `language` field "
820
"in the TranscriptionRequest."
821
)
818
- language = "en"
822
+ return None
823
return super().validate_language(language)
824
825
@classmethod
@@ -846,6 +850,63 @@ class WhisperForConditionalGeneration(
850
decoder_prompt=TextPrompt(prompt=decoder_text),
851
)
852
853
+ @classmethod
854
+ def get_language_token_ids(
855
+ cls,
856
+ tokenizer: object,
857
+ ) -> list[int]:
858
+ """Return token IDs for all supported language tokens.
859
+
860
+ Used with ``SamplingParams.allowed_token_ids`` to constrain
861
+ language detection to only produce valid language tokens.
862
+ """
863
+ token_ids = [
864
+ tokenizer.convert_tokens_to_ids(f"<|{lang_code}|>")
865
+ for lang_code in cls.supported_languages
866
+ ]
867
+ return token_ids
868
+
869
+ @classmethod
870
+ def get_language_detection_prompt(
871
+ cls,
872
+ audio: np.ndarray,
873
+ stt_config: SpeechToTextConfig,
874
+ ) -> PromptType:
875
+ """Return a prompt that elicits a single language token from Whisper.
876
+
877
+ Feed only ``<|startoftranscript|>`` as the decoder input so the model
878
+ predicts the most likely language token (e.g. ``<|de|>``).
879
+ """
880
+ return ExplicitEncoderDecoderPrompt(
881
+ encoder_prompt=TextPrompt(
882
+ prompt="",
883
+ multi_modal_data={"audio": (audio, stt_config.sample_rate)},
884
+ ),
885
+ decoder_prompt=TextPrompt(prompt="<|startoftranscript|>"),
886
+ )
887
+
888
+ @classmethod
889
+ def parse_language_detection_output(
890
+ cls,
891
+ token_ids: list[int],
892
+ tokenizer: object,
893
+ ) -> str | None:
894
+ """Parse the language token predicted by Whisper.
895
+
896
+ Decodes the first token ID and extracts the language code from the
897
+ ``<|xx|>`` format. Expects a valid language token from constrained generation.
898
+ """
899
+
900
+ decoded = tokenizer.decode(
901
+ [token_ids[0]],
902
+ skip_special_tokens=False,
903
+ )
904
+ # Whisper language tokens have the form <|xx|>
905
+ assert decoded.startswith("<|") and decoded.endswith("|>")
906
+ lang_code = decoded[2:-2]
907
+ assert lang_code in cls.supported_languages
908
+ return lang_code
909
+
910
@classmethod
911
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
912
if modality.startswith("audio"):