A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
feat(model): Add BitsAndBytes quantization support for Qwen3-Omni-MoE (#29896)
Signed-off-by: navanit-git <navanitdubey@gmail.com>
Navanit Dubey committed
Dec 3, 2025 at 00:58 UTC
a2b053dc858db461d8d98cff37ee7c67ba21126b
1 file changed
+23
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+23
@@ -62,6 +62,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
62
from vllm.model_executor.layers.rotary_embedding import get_rope
63
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
64
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
65
+from vllm.model_executor.models.module_mapping import MultiModelKeys
66
from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
67
from vllm.multimodal import MULTIMODAL_REGISTRY
68
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems
@@ -1137,6 +1138,18 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1138
}
1139
)
1140
1141
+ packed_modules_mapping = {
1142
+ "qkv_proj": [
1143
+ "q_proj",
1144
+ "k_proj",
1145
+ "v_proj",
1146
+ ],
1147
+ "gate_up_proj": [
1148
+ "gate_proj",
1149
+ "up_proj",
1150
+ ],
1151
+ }
1152
+
1153
@classmethod
1154
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1155
if modality.startswith("image"):
@@ -1763,3 +1776,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1776
1777
mrope_position_delta = llm_positions.max() + 1 - seq_len
1778
return llm_positions, mrope_position_delta
1779
+
1780
+ def get_mm_mapping(self) -> MultiModelKeys:
1781
+ """
1782
+ Get the module prefix in multimodal models
1783
+ """
1784
+ return MultiModelKeys.from_string_field(
1785
+ language_model="language_model",
1786
+ connector="visual.merger",
1787
+ tower_model=["visual.", "audio_tower."],
1788
+ )