A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 11:15 UTC
Make Qwen3VL compatible with Transformers v5 (#34262)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Harry Mellor committed
Feb 11, 2026 at 13:13 UTC
1e9204bff31f021dce8290d894c7aaf26bb4642e
2 files changed
+25
-37
vllm/model_executor/models/qwen3_vl.py
+13
-13
@@ -1112,17 +1112,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1112
}
1113
)
1114
class Qwen3LLMModel(Qwen3Model):
1115
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1116
- super().__init__(vllm_config=vllm_config, prefix=prefix)
1117
- vision_config = vllm_config.model_config.hf_config.vision_config
1118
- if not get_pp_group().is_first_rank and hasattr(
1119
- vision_config, "deepstack_visual_indexes"
1120
- ):
1121
- assert self.start_layer >= len(vision_config.deepstack_visual_indexes), (
1122
- "start_layer should be greater than or equal to "
1123
- "len(deepstack_visual_indexes)"
1124
- )
1125
-
1115
def forward(
1116
self,
1117
input_ids: torch.Tensor | None,
@@ -1178,7 +1167,7 @@ class Qwen3LLMModel(Qwen3Model):
1167
class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
1168
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1169
super(Qwen3ForCausalLM, self).__init__()
1181
- config = vllm_config.model_config.hf_config.text_config
1170
+ config = vllm_config.model_config.hf_config
1171
quant_config = vllm_config.quant_config
1172
1173
self.config = config
@@ -1298,7 +1287,18 @@ class Qwen3VLForConditionalGeneration(
1287
1288
with self._mark_language_model(vllm_config):
1289
self.language_model = Qwen3LLMForCausalLM(
1301
- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
1290
+ vllm_config=vllm_config.with_hf_config(config.text_config),
1291
+ prefix=maybe_prefix(prefix, "language_model"),
1292
+ )
1293
+
1294
+ if not get_pp_group().is_first_rank and hasattr(
1295
+ config.vision_config, "deepstack_visual_indexes"
1296
+ ):
1297
+ assert self.language_model.start_layer >= len(
1298
+ config.vision_config.deepstack_visual_indexes
1299
+ ), (
1300
+ "start_layer should be greater than or equal to "
1301
+ "len(deepstack_visual_indexes)"
1302
)
1303
1304
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/qwen3_vl_moe.py
+12
-24
@@ -48,7 +48,6 @@ from vllm.sequence import IntermediateTensors
48
49
from .interfaces import MixtureOfExperts
50
from .qwen3_moe import (
51
- Qwen3MoeDecoderLayer,
51
Qwen3MoeForCausalLM,
52
Qwen3MoeModel,
53
Qwen3MoeSparseMoeBlock,
@@ -83,27 +82,6 @@ class Qwen3VLMoeProcessingInfo(Qwen3VLProcessingInfo):
82
}
83
)
84
class Qwen3MoeLLMModel(Qwen3MoeModel):
86
- def __init__(
87
- self,
88
- *,
89
- vllm_config: VllmConfig,
90
- prefix: str = "",
91
- decoder_layer_type: type[torch.nn.Module] = Qwen3MoeDecoderLayer,
92
- ):
93
- super().__init__(
94
- vllm_config=vllm_config,
95
- prefix=prefix,
96
- decoder_layer_type=decoder_layer_type,
97
- )
98
- vision_config = vllm_config.model_config.hf_config.vision_config
99
- if not get_pp_group().is_first_rank and hasattr(
100
- vision_config, "deepstack_visual_indexes"
101
- ):
102
- assert self.start_layer >= len(vision_config.deepstack_visual_indexes), (
103
- "start_layer should be greater than or equal to "
104
- "len(deepstack_visual_indexes)"
105
- )
106
-
85
def forward(
86
self,
87
input_ids: torch.Tensor | None,
@@ -352,7 +330,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
330
class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
331
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
332
super(Qwen3MoeForCausalLM, self).__init__()
355
- self.config = vllm_config.model_config.hf_config.text_config
333
+ self.config = vllm_config.model_config.hf_config
334
self.quant_config = vllm_config.quant_config
335
self.model = Qwen3MoeLLMModel(
336
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
@@ -473,10 +451,20 @@ class Qwen3VLMoeForConditionalGeneration(
451
452
with self._mark_language_model(vllm_config):
453
self.language_model = Qwen3MoeLLMForCausalLM(
476
- vllm_config=vllm_config,
454
+ vllm_config=vllm_config.with_hf_config(config.text_config),
455
prefix=maybe_prefix(prefix, "language_model"),
456
)
457
458
+ if not get_pp_group().is_first_rank and hasattr(
459
+ config.vision_config, "deepstack_visual_indexes"
460
+ ):
461
+ assert self.language_model.start_layer >= len(
462
+ config.vision_config.deepstack_visual_indexes
463
+ ), (
464
+ "start_layer should be greater than or equal to "
465
+ "len(deepstack_visual_indexes)"
466
+ )
467
+
468
# Whether to include the gate_up_proj mapping is determined by
469
# the language model.
470
self.packed_modules_mapping = (