A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[LoRA]: Add LoRA support to Mistral's Voxtral models (#24517)
Signed-off-by: Yash Pratap Singh <yashsingh20001@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Yash Pratap Singh committed
Sep 10, 2025 at 18:42 UTC
9e3c3a7df286b0dffffe6309ce035d2a74affb6c
2 files changed
+19
-4
docs/models/supported_models.md
+1
-1
@@ -765,7 +765,7 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
765
| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | [V1](gh-issue:8779) |
766
|--------------|--------|-------------------|----------------------|---------------------------|---------------------|
767
| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | | |
768
-| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | | ✅︎ | ✅︎ |
768
+| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ | ✅︎ |
769
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | ✅︎ |
770
771
### Pooling Models
vllm/model_executor/models/voxtral.py
+18
-3
@@ -25,6 +25,7 @@ from vllm.inputs.data import PromptType
25
from vllm.logger import init_logger
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.models import SupportsPP
28
+from vllm.model_executor.models.module_mapping import MultiModelKeys
29
# yapf: disable
30
from vllm.model_executor.models.whisper import WhisperEncoder
31
# yapf: enable
@@ -44,8 +45,8 @@ from vllm.sequence import IntermediateTensors
45
from vllm.transformers_utils.tokenizer import (MistralTokenizer,
46
cached_tokenizer_from_config)
47
47
-from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
48
- SupportsTranscription)
48
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
49
+ SupportsMultiModal, SupportsTranscription)
50
from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,
51
merge_multimodal_embeddings)
52
@@ -313,9 +314,15 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
314
info=VoxtralProcessingInfo,
315
dummy_inputs=VoxtralDummyInputsBuilder)
316
class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
316
- SupportsPP, SupportsTranscription):
317
+ SupportsPP, SupportsLoRA,
318
+ SupportsTranscription):
319
supported_languages = ISO639_1_SUPPORTED_LANGS
320
321
+ packed_modules_mapping = {
322
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
323
+ "gate_up_proj": ["gate_proj", "up_proj"]
324
+ }
325
+
326
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
327
super().__init__()
328
self.tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
@@ -341,6 +348,14 @@ class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,
348
def get_language_model(self) -> torch.nn.Module:
349
return self.language_model
350
351
+ def get_mm_mapping(self) -> MultiModelKeys:
352
+ """Get module prefix for multimodal models to filter LoRA modules."""
353
+ return MultiModelKeys.from_string_field(
354
+ language_model="language_model",
355
+ connector="audio_language_adapter",
356
+ tower_model=["whisper_encoder"],
357
+ )
358
+
359
def forward(
360
self,
361
input_ids: torch.Tensor,