A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Bugfix] Relax lang pin for voxtral (#21833)
Signed-off-by: Sanchit Gandhi <sgandhi3141@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Sanchit Gandhi committed
Jul 31, 2025 at 04:38 UTC
ec02e536dfa46c7e8785cb5aaf5dd4eaad88f405
4 files changed
+80
-80
vllm/entrypoints/openai/speech_to_text.py
+2
-6
@@ -86,11 +86,7 @@ class OpenAISpeechToText(OpenAIServing):
86
audio_data: bytes,
87
) -> tuple[list[PromptType], float]:
88
# Validate request
89
- # TODO language should be optional and can be guessed.
90
- # For now we default to en. See
91
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
92
- lang = request.language or "en"
93
- self.model_cls.validate_language(lang)
89
+ language = self.model_cls.validate_language(request.language)
90
91
if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
92
raise ValueError("Maximum file size exceeded.")
@@ -112,7 +108,7 @@ class OpenAISpeechToText(OpenAIServing):
108
audio=chunk,
109
stt_config=self.asr_config,
110
model_config=self.model_config,
115
- language=lang,
111
+ language=language,
112
task_type=self.task_type,
113
request_prompt=request.prompt)
114
prompts.append(prompt)
vllm/model_executor/models/interfaces.py
+47
-6
@@ -1,13 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Iterable, MutableSequence
4
+from collections.abc import Iterable, Mapping, MutableSequence
5
from typing import (TYPE_CHECKING, ClassVar, Literal, Optional, Protocol,
6
Union, overload, runtime_checkable)
7
8
import numpy as np
9
import torch
10
from torch import Tensor
11
+from transformers.models.whisper.tokenization_whisper import LANGUAGES
12
from typing_extensions import Self, TypeIs
13
14
from vllm.config import ModelConfig, SpeechToTextConfig
@@ -685,6 +686,8 @@ class SupportsQuant:
686
@runtime_checkable
687
class SupportsTranscription(Protocol):
688
"""The interface required for all models that support transcription."""
689
+ # Mapping from ISO639_1 language codes: language names
690
+ supported_languages: ClassVar[Mapping[str, str]]
691
692
supports_transcription: ClassVar[Literal[True]] = True
693
@@ -694,11 +697,22 @@ class SupportsTranscription(Protocol):
697
`True`.
698
"""
699
700
+ def __init_subclass__(cls, **kwargs):
701
+ super().__init_subclass__(**kwargs)
702
+ # language codes in supported_languages
703
+ # that don't exist in the full language map
704
+ invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
705
+ if invalid:
706
+ raise ValueError(
707
+ f"{cls.__name__}.supported_languages contains invalid "
708
+ f"language codes: {sorted(invalid)}\n. "
709
+ f"Valid choices are: {sorted(LANGUAGES.keys())}")
710
+
711
@classmethod
712
def get_generation_prompt(cls, audio: np.ndarray,
713
stt_config: SpeechToTextConfig,
700
- model_config: ModelConfig, language: str,
701
- task_type: str,
714
+ model_config: ModelConfig,
715
+ language: Optional[str], task_type: str,
716
request_prompt: str) -> PromptType:
717
"""Get the prompt for the ASR model.
718
The model has control over the construction, as long as it
@@ -706,9 +720,36 @@ class SupportsTranscription(Protocol):
720
...
721
722
@classmethod
709
- def validate_language(cls, language: str) -> bool:
710
- """Check if the model supports a specific ISO639_1 language."""
711
- ...
723
+ def get_other_languages(cls) -> Mapping[str, str]:
724
+ # other possible language codes from the whisper map
725
+ return {
726
+ k: v
727
+ for k, v in LANGUAGES.items() if k not in cls.supported_languages
728
+ }
729
+
730
+ @classmethod
731
+ def validate_language(cls, language: Optional[str]) -> Optional[str]:
732
+ """
733
+ Ensure the language specified in the transcription request
734
+ is a valid ISO 639-1 language code. If the request language is
735
+ valid, but not natively supported by the model, trigger a
736
+ warning (but not an exception).
737
+ """
738
+ if language is None or language in cls.supported_languages:
739
+ return language
740
+ elif language in cls.get_other_languages():
741
+ logger.warning(
742
+ "Language %r is not natively supported by %s; "
743
+ "results may be less accurate. Supported languages: %r",
744
+ language,
745
+ cls.__name__,
746
+ list(cls.supported_languages.keys()),
747
+ )
748
+ return language
749
+ else:
750
+ raise ValueError(
751
+ f"Unsupported language: {language!r}. Must be one of "
752
+ f"{list(cls.supported_languages.keys())}.")
753
754
@classmethod
755
def get_speech_to_text_config(
vllm/model_executor/models/voxtral.py
+16
-9
@@ -26,8 +26,7 @@ from vllm.logger import init_logger
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.models import SupportsPP
28
# yapf: disable
29
-from vllm.model_executor.models.whisper import (
30
- WhisperEncoder, WhisperForConditionalGeneration)
29
+from vllm.model_executor.models.whisper import WhisperEncoder
30
# yapf: enable
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -50,6 +49,18 @@ from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,
49
50
logger = init_logger(__name__)
51
52
+ISO639_1_SUPPORTED_LANGS = {
53
+ "ar": "Arabic",
54
+ "nl": "Dutch",
55
+ "en": "English",
56
+ "fr": "French",
57
+ "de": "German",
58
+ "hi": "Hindi",
59
+ "it": "Italian",
60
+ "pt": "Portuguese",
61
+ "es": "Spanish",
62
+}
63
+
64
65
class VoxtralProcessorAdapter:
66
"""
@@ -301,6 +312,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
312
dummy_inputs=VoxtralDummyInputsBuilder)
313
class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
314
SupportsPP, SupportsTranscription):
315
+ supported_languages = ISO639_1_SUPPORTED_LANGS
316
317
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
318
super().__init__()
@@ -441,8 +453,8 @@ class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
453
# for speech-to-text transcription
454
def get_generation_prompt(cls, audio: np.ndarray,
455
model_config: ModelConfig,
444
- stt_config: SpeechToTextConfig, language: str,
445
- task_type: str,
456
+ stt_config: SpeechToTextConfig,
457
+ language: Optional[str], task_type: str,
458
request_prompt: str) -> PromptType:
459
tokenizer = cached_tokenizer_from_config(model_config)
460
audio = Audio(audio, int(stt_config.sample_rate),
@@ -457,11 +469,6 @@ class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
469
prompts_dict["prompt_token_ids"] = tokenized.tokens
470
return cast(PromptType, prompts_dict)
471
460
- @classmethod
461
- def validate_language(cls, language: str) -> bool:
462
- # same as whisper
463
- return WhisperForConditionalGeneration.validate_language(language)
464
-
472
@classmethod
473
def get_num_audio_tokens(cls, audio_duration_s: float,
474
stt_config: SpeechToTextConfig,
vllm/model_executor/models/whisper.py
+15
-59
@@ -109,51 +109,6 @@ ISO639_1_SUPPORTED_LANGS = {
109
"vi": "Vietnamese",
110
"cy": "Welsh"
111
}
112
-ISO639_1_OTHER_LANGS = {
113
- "lo": "Lao",
114
- "jw": "Javanese",
115
- "tk": "Turkmen",
116
- "yi": "Yiddish",
117
- "so": "Somali",
118
- "bn": "Bengali",
119
- "nn": "Norwegian Nynorsk",
120
- "si": "Sinhala",
121
- "yo": "Yoruba",
122
- "sa": "Sanskrit",
123
- "mi": "Māori",
124
- "fo": "Faroese", # codespell:ignore
125
- "mt": "Maltese",
126
- "tg": "Tajik",
127
- "mg": "Malagasy",
128
- "haw": "Hawaiian",
129
- "km": "Khmer",
130
- "br": "Breton",
131
- "ps": "Pashto",
132
- "ln": "Lingala",
133
- "la": "Latin",
134
- "ml": "Malayalam",
135
- "sq": "Albanian",
136
- "su": "Sundanese",
137
- "eu": "Basque",
138
- "ka": "Georgian",
139
- "uz": "Uzbek",
140
- "sn": "Shona",
141
- "ht": "Haitian",
142
- "as": "Assamese",
143
- "mn": "Mongolian",
144
- "te": "Telugu",
145
- "pa": "Panjabi",
146
- "tt": "Tatar",
147
- "gu": "Gujarati",
148
- "oc": "Occitan",
149
- "ha": "Hausa",
150
- "ba": "Bashkir",
151
- "my": "Burmese",
152
- "sd": "Sindhi",
153
- "am": "Amharic",
154
- "lb": "Luxembourgish",
155
- "bo": "Tibetan"
156
-}
112
113
114
class WhisperAudioInputs(TypedDict):
@@ -807,22 +762,20 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
762
763
# Whisper only supports audio-conditioned generation.
764
supports_transcription_only = True
765
+ supported_languages = ISO639_1_SUPPORTED_LANGS
766
767
@classmethod
812
- def validate_language(cls, language: str) -> bool:
813
- if language in ISO639_1_SUPPORTED_LANGS:
814
- return True
815
- elif language in ISO639_1_OTHER_LANGS:
768
+ def validate_language(cls, language: Optional[str]) -> Optional[str]:
769
+ if language is None:
770
+ # TODO language should be optional and can be guessed.
771
+ # For now we default to en. See
772
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520
773
logger.warning(
817
- "The selected language %s has limited accuracy with"
818
- " reported WER>=0.5. Results may be less accurate "
819
- "for this choice.", language)
820
- return True
821
- else:
822
- raise ValueError(f"Unsupported language: {language}."
823
- "Language should be one of:" +
824
- f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +
825
- f"or {list(ISO639_1_OTHER_LANGS.values())}")
774
+ "Defaulting to language='en'. If you wish to transcribe "
775
+ "audio in a different language, pass the `language` field "
776
+ "in the TranscriptionRequest.")
777
+ language = "en"
778
+ return super().validate_language(language)
779
780
@classmethod
781
def get_generation_prompt(
@@ -830,9 +783,12 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
783
audio: np.ndarray,
784
model_config: ModelConfig, # not needed here
785
stt_config: SpeechToTextConfig,
833
- language: str,
786
+ language: Optional[str],
787
task_type: str,
788
request_prompt: str) -> PromptType:
789
+ if language is None:
790
+ raise ValueError(
791
+ "Language must be specified when creating the Whisper prompt")
792
prompt = {
793
"encoder_prompt": {
794
# Whisper does not support encoder prompt.