A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Enable BNB support for qwen2_5_omni_thinker (#24420)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Sep 9, 2025 at 00:37 UTC
6f4a82f8b5a13a62dac0b84c7c817a33c602af52
1 file changed
+29
-2
vllm/model_executor/models/qwen2_5_omni_thinker.py
+29
-2
@@ -41,6 +41,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
41
from vllm.config import VllmConfig
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
44
+from vllm.model_executor.models.module_mapping import MultiModelKeys
45
from vllm.model_executor.models.qwen2_5_vl import (
46
Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs,
47
Qwen2_5_VLImageInputs, Qwen2_5_VLImagePixelInputs,
@@ -66,7 +67,8 @@ from vllm.sequence import IntermediateTensors
67
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
68
from vllm.utils.tensor_schema import TensorSchema, TensorShape
69
69
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
70
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
71
+ SupportsMultiModal, SupportsPP)
72
from .utils import (AutoWeightsLoader, WeightsMapper,
73
init_vllm_registered_model, maybe_prefix,
74
merge_multimodal_embeddings)
@@ -726,7 +728,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
728
dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
729
)
730
class Qwen2_5OmniThinkerForConditionalGeneration(
729
- nn.Module, SupportsMultiModal, SupportsPP,
731
+ nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
732
Qwen2_5OmniConditionalGenerationMixin):
733
hf_to_vllm_mapper = WeightsMapper(
734
orig_to_new_prefix={
@@ -734,6 +736,22 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
736
"thinker.model.": "language_model.model.",
737
"thinker.": "",
738
})
739
+ packed_modules_mapping = {
740
+ "qkv_proj": [
741
+ "q_proj",
742
+ "k_proj",
743
+ "v_proj",
744
+ ],
745
+ "attn.qkv": [
746
+ "attn.q",
747
+ "attn.k",
748
+ "attn.v",
749
+ ],
750
+ "gate_up_proj": [
751
+ "gate_proj",
752
+ "up_proj",
753
+ ],
754
+ }
755
756
@classmethod
757
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
@@ -956,3 +974,12 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
974
mapper=self.hf_to_vllm_mapper)
975
976
return loaded_weights
977
+
978
+ def get_mm_mapping(self) -> MultiModelKeys:
979
+ """
980
+ Get the module prefix in multimodal models
981
+ """
982
+ return MultiModelKeys.from_string_field(
983
+ language_model="language_model",
984
+ connector="merger.",
985
+ tower_model=["visual.", "audio_tower."])