A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[LoRA]: Add lora support to qwen-2.5-omni (#24231)
Yash Pratap Singh committed
Sep 4, 2025 at 18:20 UTC
c9f7081f9c848d83ecbf42b57591451d6ff5a7a9
2 files changed
+14
-3
docs/models/supported_models.md
+1
-1
@@ -662,7 +662,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
662
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | ✅︎ |
663
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
664
| `Qwen2_5_VLForConditionalGeneration` | Qwen2.5-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
665
-| `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen2.5-Omni-7B` | | ✅︎ | ✅︎ |
665
+| `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen2.5-Omni-3B`, `Qwen/Qwen2.5-Omni-7B` | ✅︎ | ✅︎ | ✅︎ |
666
| `RForConditionalGeneration` | R-VL-4B | T + I<sup>E+</sup> | `YannQi/R-4B` | | ✅︎ | ✅︎ |
667
| `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ | ✅︎ |
668
| `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | | ✅︎ |
vllm/model_executor/models/qwen2_5_omni_thinker.py
+13
-2
@@ -41,6 +41,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
41
from vllm.config import VllmConfig
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
44
+from vllm.model_executor.models.module_mapping import MultiModelKeys
45
from vllm.model_executor.models.qwen2_5_vl import (
46
Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs,
47
Qwen2_5_VLImageInputs, Qwen2_5_VLImagePixelInputs,
@@ -66,7 +67,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
67
from vllm.sequence import IntermediateTensors
68
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
69
69
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
70
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
71
+ SupportsMultiModal, SupportsPP)
72
from .utils import (AutoWeightsLoader, WeightsMapper,
73
init_vllm_registered_model, maybe_prefix,
74
merge_multimodal_embeddings)
@@ -705,7 +707,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
707
dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
708
)
709
class Qwen2_5OmniThinkerForConditionalGeneration(
708
- nn.Module, SupportsMultiModal, SupportsPP,
710
+ nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
711
Qwen2_5OmniConditionalGenerationMixin):
712
hf_to_vllm_mapper = WeightsMapper(
713
orig_to_new_prefix={
@@ -798,6 +800,15 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
800
def get_language_model(self) -> torch.nn.Module:
801
return self.language_model
802
803
+ def get_mm_mapping(self) -> MultiModelKeys:
804
+ """Get module prefix for multimodal models to filter LoRA modules."""
805
+ return MultiModelKeys.from_string_field(
806
+ language_model="language_model",
807
+ connector=[], # No explicit connector in this model
808
+ tower_model=["visual",
809
+ "audio_tower"], # Exclude vision and audio towers
810
+ )
811
+
812
def get_multimodal_embeddings(self,
813
**kwargs: object) -> MultiModalEmbeddings:
814