A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Use merge_by_field_config for MM models (Qwen series) (#27546)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 27, 2025 at 13:38 UTC
cbd5e07a513759892554a7af5a49432bc8dd3386
7 files changed
+36
-305
vllm/model_executor/models/qwen2_5_omni_thinker.py
+15
-80
@@ -126,12 +126,12 @@ class Qwen2_5OmniAudioFeatureInputs(TensorSchema):
126
type: Literal["audio_features"]
127
input_features: Annotated[
128
torch.Tensor | list[torch.Tensor],
129
- TensorShape("nmb", "tsl"),
129
+ TensorShape("nmb", "tsl", dynamic_dims={"tsl"}),
130
]
131
132
feature_attention_mask: Annotated[
133
- torch.Tensor,
134
- TensorShape("na", "msl"),
133
+ torch.Tensor | list[torch.Tensor],
134
+ TensorShape("na", "msl", dynamic_dims={"msl"}),
135
]
136
137
@@ -651,18 +651,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
651
652
653
class Qwen2_5OmniConditionalGenerationMixin:
654
- def _validate_and_reshape_mm_tensor(
655
- self, mm_input: object, name: str, dim: int = 0
656
- ) -> torch.Tensor:
657
- if not isinstance(mm_input, (torch.Tensor, list)):
658
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
659
- if isinstance(mm_input, torch.Tensor):
660
- if dim == 0:
661
- return mm_input.reshape(-1, *mm_input.shape[2:])
662
- return torch.concat(list(mm_input), dim=dim)
663
- else:
664
- return torch.concat(mm_input, dim=dim)
665
-
654
def _parse_and_validate_audio_input(
655
self, **kwargs: object
656
) -> Qwen2_5OmniAudioFeatureInputs | None:
@@ -671,18 +659,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
659
feature_attention_mask = kwargs.pop("feature_attention_mask", None)
660
if input_audio_features is None:
661
return None
674
- input_audio_features = self._validate_and_reshape_mm_tensor(
675
- input_audio_features, "input_audio_features", dim=1
676
- )
677
- if feature_attention_mask is not None:
678
- feature_attention_mask = self._validate_and_reshape_mm_tensor(
679
- feature_attention_mask, "feature_attention_mask"
680
- )
681
- if not isinstance(input_audio_features, (torch.Tensor, list)):
682
- raise ValueError(
683
- "Incorrect type of audio input features. "
684
- f"Got type: {type(input_audio_features)}"
685
- )
662
+
663
return Qwen2_5OmniAudioFeatureInputs(
664
type="audio_features",
665
input_features=input_audio_features,
@@ -702,19 +679,6 @@ class Qwen2_5OmniConditionalGenerationMixin:
679
return None
680
681
if pixel_values is not None:
705
- pixel_values = self._validate_and_reshape_mm_tensor(
706
- pixel_values, "image pixel values"
707
- )
708
- image_grid_thw = self._validate_and_reshape_mm_tensor(
709
- image_grid_thw, "image grid_thw"
710
- )
711
-
712
- if not isinstance(pixel_values, (torch.Tensor, list)):
713
- raise ValueError(
714
- "Incorrect type of image pixel values. "
715
- f"Got type: {type(pixel_values)}"
716
- )
717
-
682
return Qwen2_5_VLImagePixelInputs(
683
type="pixel_values",
684
pixel_values=pixel_values,
@@ -722,18 +686,6 @@ class Qwen2_5OmniConditionalGenerationMixin:
686
)
687
688
if image_embeds is not None:
725
- image_embeds = self._validate_and_reshape_mm_tensor(
726
- image_embeds, "image embeds"
727
- )
728
- image_grid_thw = self._validate_and_reshape_mm_tensor(
729
- image_grid_thw, "image grid_thw"
730
- )
731
-
732
- if not isinstance(image_embeds, torch.Tensor):
733
- raise ValueError(
734
- "Incorrect type of image embeddings. "
735
- f"Got type: {type(image_embeds)}"
736
- )
689
return Qwen2_5_VLImageEmbeddingInputs(
690
type="image_embeds",
691
image_embeds=image_embeds,
@@ -752,13 +704,6 @@ class Qwen2_5OmniConditionalGenerationMixin:
704
return None
705
706
if pixel_values_videos is not None:
755
- pixel_values_videos = self._validate_and_reshape_mm_tensor(
756
- pixel_values_videos, "video pixel values"
757
- )
758
- video_grid_thw = self._validate_and_reshape_mm_tensor(
759
- video_grid_thw, "video grid_thw"
760
- )
761
-
707
return Qwen2_5_VLVideoPixelInputs(
708
type="pixel_values_videos",
709
pixel_values_videos=pixel_values_videos,
@@ -766,13 +711,6 @@ class Qwen2_5OmniConditionalGenerationMixin:
711
)
712
713
if video_embeds is not None:
769
- video_embeds = self._validate_and_reshape_mm_tensor(
770
- video_embeds, "video embeds"
771
- )
772
- video_grid_thw = self._validate_and_reshape_mm_tensor(
773
- video_grid_thw, "video grid_thw"
774
- )
775
-
714
if not isinstance(video_embeds, torch.Tensor):
715
raise ValueError(
716
"Incorrect type of video embeddings. "
@@ -787,23 +725,18 @@ class Qwen2_5OmniConditionalGenerationMixin:
725
def _process_audio_input(
726
self,
727
audio_input: Qwen2_5OmniAudioFeatureInputs,
790
- audio_hashes: list[str] = None,
791
- cached_audio_features: torch.Tensor = None,
728
+ audio_hashes: list[str] | None = None,
729
+ cached_audio_features: torch.Tensor | None = None,
730
) -> torch.Tensor:
731
input_features = audio_input["input_features"]
732
audio_feature_lengths = audio_input["audio_feature_lengths"]
795
- if input_features.ndim == 3:
796
- assert input_features.shape[0] == 1
797
- input_features = input_features.squeeze(0)
798
- if audio_feature_lengths.ndim == 2:
799
- assert (
800
- audio_feature_lengths.shape[0] == 1
801
- or audio_feature_lengths.shape[1] == 1
802
- )
803
- if audio_feature_lengths.shape[0] == 1:
804
- audio_feature_lengths = audio_feature_lengths.squeeze(0)
805
- else:
806
- audio_feature_lengths = audio_feature_lengths.squeeze(1)
733
+
734
+ if audio_feature_lengths.shape[0] == 1:
735
+ audio_feature_lengths = audio_feature_lengths.squeeze(0)
736
+ elif audio_feature_lengths.shape[1] == 1:
737
+ audio_feature_lengths = audio_feature_lengths.squeeze(1)
738
+ else:
739
+ raise AssertionError(audio_feature_lengths.shape)
740
741
audio_feat_lengths, audio_output_lengths = (
742
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
@@ -867,6 +800,8 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
800
SupportsMRoPE,
801
Qwen2_5OmniConditionalGenerationMixin,
802
):
803
+ merge_by_field_config = True
804
+
805
hf_to_vllm_mapper = WeightsMapper(
806
orig_to_new_prefix={
807
"thinker.lm_head.": "language_model.lm_head.",
vllm/model_executor/models/qwen2_5_vl.py
+2
-47
@@ -1071,6 +1071,8 @@ class Qwen2_5_VLForConditionalGeneration(
1071
SupportsMultiModalPruning,
1072
SupportsMRoPE,
1073
):
1074
+ merge_by_field_config = True
1075
+
1076
packed_modules_mapping = {
1077
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
1078
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -1273,24 +1275,6 @@ class Qwen2_5_VLForConditionalGeneration(
1275
num_layers = len(self.language_model.model.layers)
1276
return (2, num_layers // 2, num_layers - 3)
1277
1276
- def _validate_and_reshape_mm_tensor(
1277
- self, mm_input: object, name: str
1278
- ) -> torch.Tensor:
1279
- if not isinstance(mm_input, (torch.Tensor, list)):
1280
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
1281
- if isinstance(mm_input, torch.Tensor):
1282
- if mm_input.ndim == 2:
1283
- return mm_input
1284
- if mm_input.ndim != 3:
1285
- raise ValueError(
1286
- f"{name} should be 2D or batched 3D tensor. "
1287
- f"Got ndim: {mm_input.ndim} "
1288
- f"(shape={mm_input.shape})"
1289
- )
1290
- return mm_input.reshape(-1, mm_input.shape[-1])
1291
- else:
1292
- return torch.concat(mm_input)
1293
-
1278
def _parse_and_validate_image_input(
1279
self, **kwargs: object
1280
) -> Qwen2_5_VLImageInputs | None:
@@ -1302,13 +1286,6 @@ class Qwen2_5_VLForConditionalGeneration(
1286
return None
1287
1288
if pixel_values is not None:
1305
- pixel_values = self._validate_and_reshape_mm_tensor(
1306
- pixel_values, "image pixel values"
1307
- )
1308
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1309
- image_grid_thw, "image grid_thw"
1310
- )
1311
-
1289
return Qwen2_5_VLImagePixelInputs(
1290
type="pixel_values",
1291
pixel_values=pixel_values,
@@ -1316,13 +1293,6 @@ class Qwen2_5_VLForConditionalGeneration(
1293
)
1294
1295
if image_embeds is not None:
1319
- image_embeds = self._validate_and_reshape_mm_tensor(
1320
- image_embeds, "image embeds"
1321
- )
1322
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1323
- image_grid_thw, "image grid_thw"
1324
- )
1325
-
1296
return Qwen2_5_VLImageEmbeddingInputs(
1297
type="image_embeds",
1298
image_embeds=image_embeds,
@@ -1341,14 +1311,6 @@ class Qwen2_5_VLForConditionalGeneration(
1311
return None
1312
1313
if pixel_values_videos is not None:
1344
- pixel_values_videos = self._validate_and_reshape_mm_tensor(
1345
- pixel_values_videos, "video pixel values"
1346
- )
1347
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1348
- video_grid_thw, "video grid_thw"
1349
- )
1350
- if second_per_grid_ts is not None and second_per_grid_ts.ndim == 2:
1351
- second_per_grid_ts = second_per_grid_ts.squeeze(-1)
1314
return Qwen2_5_VLVideoPixelInputs(
1315
type="pixel_values_videos",
1316
pixel_values_videos=pixel_values_videos,
@@ -1357,13 +1319,6 @@ class Qwen2_5_VLForConditionalGeneration(
1319
)
1320
1321
if video_embeds is not None:
1360
- video_embeds = self._validate_and_reshape_mm_tensor(
1361
- video_embeds, "video embeds"
1362
- )
1363
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1364
- video_grid_thw, "video grid_thw"
1365
- )
1366
-
1322
return Qwen2_5_VLVideoEmbeddingInputs(
1323
type="video_embeds",
1324
video_embeds=video_embeds,
vllm/model_executor/models/qwen2_audio.py
+2
-23
@@ -313,6 +313,8 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
313
dummy_inputs=Qwen2AudioDummyInputsBuilder,
314
)
315
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
316
+ merge_by_field_config = True
317
+
318
@classmethod
319
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
320
if modality.startswith("audio"):
@@ -346,16 +348,6 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports
348
self.language_model.make_empty_intermediate_tensors
349
)
350
349
- def _validate_and_reshape_mm_tensor(
350
- self, mm_input: object, name: str
351
- ) -> torch.Tensor:
352
- if not isinstance(mm_input, (torch.Tensor, list)):
353
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
354
- if isinstance(mm_input, torch.Tensor):
355
- return mm_input.reshape(-1, *mm_input.shape[2:])
356
- else:
357
- return torch.concat(mm_input)
358
-
351
def _parse_and_validate_audio_input(
352
self, **kwargs: object
353
) -> Qwen2AudioInputs | None:
@@ -367,24 +359,11 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports
359
return None
360
361
if audio_embeds is not None:
370
- if not isinstance(audio_embeds, (torch.Tensor, list)):
371
- raise ValueError(
372
- f"Incorrect type of audio embeds. Got type: {type(audio_embeds)}"
373
- )
374
- audio_embeds = self._validate_and_reshape_mm_tensor(
375
- audio_embeds, "audio_embeds"
376
- )
362
return Qwen2AudioEmbeddingInputs(
363
type="audio_embeds", audio_embeds=audio_embeds
364
)
365
366
if input_features is not None:
382
- input_features = self._validate_and_reshape_mm_tensor(
383
- input_features, "input_features"
384
- )
385
- feature_attention_mask = self._validate_and_reshape_mm_tensor(
386
- feature_attention_mask, "feature_attention_mask"
387
- )
367
return Qwen2AudioFeatureInputs(
368
type="audio_features",
369
input_features=input_features,
vllm/model_executor/models/qwen2_vl.py
+2
-46
@@ -1213,6 +1213,8 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
1213
class Qwen2VLForConditionalGeneration(
1214
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1215
):
1216
+ merge_by_field_config = True
1217
+
1218
# To ensure correct weight loading and mapping.
1219
hf_to_vllm_mapper = WeightsMapper(
1220
orig_to_new_prefix={
@@ -1406,24 +1408,6 @@ class Qwen2VLForConditionalGeneration(
1408
self.language_model.make_empty_intermediate_tensors
1409
)
1410
1409
- def _validate_and_reshape_mm_tensor(
1410
- self, mm_input: object, name: str
1411
- ) -> torch.Tensor:
1412
- if not isinstance(mm_input, (torch.Tensor, list)):
1413
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
1414
- if isinstance(mm_input, torch.Tensor):
1415
- if mm_input.ndim == 2:
1416
- return mm_input
1417
- if mm_input.ndim != 3:
1418
- raise ValueError(
1419
- f"{name} should be 2D or batched 3D tensor. "
1420
- f"Got ndim: {mm_input.ndim} "
1421
- f"(shape={mm_input.shape})"
1422
- )
1423
- return mm_input.reshape(-1, mm_input.shape[-1])
1424
- else:
1425
- return torch.concat(mm_input)
1426
-
1411
def _parse_and_validate_image_input(
1412
self, **kwargs: object
1413
) -> Qwen2VLImageInputs | None:
@@ -1435,13 +1419,6 @@ class Qwen2VLForConditionalGeneration(
1419
return None
1420
1421
if pixel_values is not None:
1438
- pixel_values = self._validate_and_reshape_mm_tensor(
1439
- pixel_values, "image pixel values"
1440
- )
1441
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1442
- image_grid_thw, "image grid_thw"
1443
- )
1444
-
1422
return Qwen2VLImagePixelInputs(
1423
type="pixel_values",
1424
pixel_values=pixel_values,
@@ -1449,13 +1426,6 @@ class Qwen2VLForConditionalGeneration(
1426
)
1427
1428
if image_embeds is not None:
1452
- image_embeds = self._validate_and_reshape_mm_tensor(
1453
- image_embeds, "image embeds"
1454
- )
1455
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1456
- image_grid_thw, "image grid_thw"
1457
- )
1458
-
1429
return Qwen2VLImageEmbeddingInputs(
1430
type="image_embeds",
1431
image_embeds=image_embeds,
@@ -1473,13 +1443,6 @@ class Qwen2VLForConditionalGeneration(
1443
return None
1444
1445
if pixel_values_videos is not None:
1476
- pixel_values_videos = self._validate_and_reshape_mm_tensor(
1477
- pixel_values_videos, "video pixel values"
1478
- )
1479
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1480
- video_grid_thw, "video grid_thw"
1481
- )
1482
-
1446
return Qwen2VLVideoPixelInputs(
1447
type="pixel_values_videos",
1448
pixel_values_videos=pixel_values_videos,
@@ -1487,13 +1450,6 @@ class Qwen2VLForConditionalGeneration(
1450
)
1451
1452
if video_embeds is not None:
1490
- video_embeds = self._validate_and_reshape_mm_tensor(
1491
- video_embeds, "video embeds"
1492
- )
1493
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1494
- video_grid_thw, "video grid_thw"
1495
- )
1496
-
1453
return Qwen2VLVideoEmbeddingInputs(
1454
type="video_embeds",
1455
video_embeds=video_embeds,
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+9
-33
@@ -63,10 +63,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
63
from vllm.model_executor.layers.quantization import QuantizationConfig
64
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
65
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
66
-from vllm.model_executor.models.qwen2_audio import (
67
- Qwen2AudioFeatureInputs,
68
- Qwen2AudioProcessingInfo,
69
-)
66
+from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
67
from vllm.multimodal import MULTIMODAL_REGISTRY
68
from vllm.multimodal.inputs import MultiModalKwargsItems
69
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
@@ -86,6 +83,7 @@ from .interfaces import (
83
SupportsPP,
84
)
85
from .qwen2_5_omni_thinker import (
86
+ Qwen2_5OmniAudioFeatureInputs,
87
Qwen2_5OmniConditionalGenerationMixin,
88
Qwen2_5OmniThinkerDummyInputsBuilder,
89
Qwen2_5OmniThinkerMultiModalProcessor,
@@ -101,6 +99,7 @@ from .utils import (
99
AutoWeightsLoader,
100
WeightsMapper,
101
_merge_multimodal_embeddings,
102
+ flatten_bn,
103
maybe_prefix,
104
)
105
from .vision import (
@@ -1056,41 +1055,16 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1055
1056
1057
class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin):
1059
- def _validate_and_reshape_mm_tensor(
1060
- self, mm_input: object, name: str, dim: int = 0
1061
- ) -> torch.Tensor:
1062
- if not isinstance(mm_input, (torch.Tensor, list)):
1063
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
1064
- if name == "feature_attention_mask":
1065
- dim = -1
1066
- if isinstance(mm_input, torch.Tensor):
1067
- return torch.concat(list(mm_input), dim=dim)
1068
- else:
1069
- if isinstance(mm_input[0], list):
1070
- return torch.concat(
1071
- [torch.concat(mm_input[i], dim=dim) for i in range(len(mm_input))],
1072
- dim=dim,
1073
- )
1074
- else:
1075
- return torch.concat(mm_input, dim=dim)
1076
-
1058
def _process_audio_input(
1059
self,
1079
- audio_input: Qwen2AudioFeatureInputs,
1080
- audio_hashes: list[str] = None,
1081
- cached_audio_features: torch.Tensor = None,
1060
+ audio_input: Qwen2_5OmniAudioFeatureInputs,
1061
+ audio_hashes: list[str] | None = None,
1062
+ cached_audio_features: torch.Tensor | None = None,
1063
) -> torch.Tensor:
1064
input_features = audio_input["input_features"]
1065
audio_feature_lengths = audio_input["audio_feature_lengths"]
1066
1086
- if input_features.ndim == 3:
1087
- assert input_features.shape[0] == 1
1088
- input_features = input_features.squeeze(0)
1089
-
1090
- if not isinstance(audio_feature_lengths, torch.Tensor):
1091
- audio_feature_lengths = torch.cat(audio_feature_lengths)
1092
- if audio_feature_lengths.ndim == 2:
1093
- audio_feature_lengths = audio_feature_lengths.reshape(-1)
1067
+ audio_feature_lengths = flatten_bn(audio_feature_lengths, concat=True)
1068
1069
audio_feat_lengths, audio_output_lengths = _get_feat_extract_output_lengths(
1070
audio_feature_lengths
@@ -1117,6 +1091,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1091
SupportsMRoPE,
1092
Qwen3OmniMoeConditionalGenerationMixin,
1093
):
1094
+ merge_by_field_config = True
1095
+
1096
hf_to_vllm_mapper = WeightsMapper(
1097
orig_to_new_prefix={
1098
"thinker.lm_head.": "language_model.lm_head.",
vllm/model_executor/models/qwen3_vl.py
+2
-62
@@ -1175,6 +1175,8 @@ class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
1175
class Qwen3VLForConditionalGeneration(
1176
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1177
):
1178
+ merge_by_field_config = True
1179
+
1180
packed_modules_mapping = {
1181
"qkv_proj": [
1182
"q_proj",
@@ -1298,24 +1300,6 @@ class Qwen3VLForConditionalGeneration(
1300
for idx in range(self.deepstack_num_level):
1301
self.deepstack_input_embeds[idx][:num_tokens].zero_()
1302
1301
- def _validate_and_reshape_mm_tensor(
1302
- self, mm_input: object, name: str
1303
- ) -> torch.Tensor:
1304
- if not isinstance(mm_input, (torch.Tensor, list)):
1305
- raise ValueError(f"Incorrect type of {name}. Got type: {type(mm_input)}")
1306
- if isinstance(mm_input, torch.Tensor):
1307
- if mm_input.ndim == 2:
1308
- return mm_input
1309
- if mm_input.ndim != 3:
1310
- raise ValueError(
1311
- f"{name} should be 2D or batched 3D tensor. "
1312
- f"Got ndim: {mm_input.ndim} "
1313
- f"(shape={mm_input.shape})"
1314
- )
1315
- return mm_input.reshape(-1, mm_input.shape[-1])
1316
- else:
1317
- return torch.concat(mm_input)
1318
-
1303
def _parse_and_validate_image_input(
1304
self, **kwargs: object
1305
) -> Qwen2_5_VLImageInputs | None:
@@ -1327,19 +1311,6 @@ class Qwen3VLForConditionalGeneration(
1311
return None
1312
1313
if pixel_values is not None:
1330
- pixel_values = self._validate_and_reshape_mm_tensor(
1331
- pixel_values, "image pixel values"
1332
- )
1333
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1334
- image_grid_thw, "image grid_thw"
1335
- )
1336
-
1337
- if not isinstance(pixel_values, (torch.Tensor, list)):
1338
- raise ValueError(
1339
- "Incorrect type of image pixel values. "
1340
- f"Got type: {type(pixel_values)}"
1341
- )
1342
-
1314
return Qwen2_5_VLImagePixelInputs(
1315
type="pixel_values",
1316
pixel_values=pixel_values,
@@ -1347,18 +1318,6 @@ class Qwen3VLForConditionalGeneration(
1318
)
1319
1320
if image_embeds is not None:
1350
- image_embeds = self._validate_and_reshape_mm_tensor(
1351
- image_embeds, "image embeds"
1352
- )
1353
- image_grid_thw = self._validate_and_reshape_mm_tensor(
1354
- image_grid_thw, "image grid_thw"
1355
- )
1356
-
1357
- if not isinstance(image_embeds, torch.Tensor):
1358
- raise ValueError(
1359
- "Incorrect type of image embeddings. "
1360
- f"Got type: {type(image_embeds)}"
1361
- )
1321
return Qwen2_5_VLImageEmbeddingInputs(
1322
type="image_embeds",
1323
image_embeds=image_embeds,
@@ -1377,13 +1336,6 @@ class Qwen3VLForConditionalGeneration(
1336
return None
1337
1338
if pixel_values_videos is not None:
1380
- pixel_values_videos = self._validate_and_reshape_mm_tensor(
1381
- pixel_values_videos, "video pixel values"
1382
- )
1383
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1384
- video_grid_thw, "video grid_thw"
1385
- )
1386
-
1339
return Qwen2_5_VLVideoPixelInputs(
1340
type="pixel_values_videos",
1341
pixel_values_videos=pixel_values_videos,
@@ -1392,18 +1344,6 @@ class Qwen3VLForConditionalGeneration(
1344
)
1345
1346
if video_embeds is not None:
1395
- video_embeds = self._validate_and_reshape_mm_tensor(
1396
- video_embeds, "video embeds"
1397
- )
1398
- video_grid_thw = self._validate_and_reshape_mm_tensor(
1399
- video_grid_thw, "video grid_thw"
1400
- )
1401
-
1402
- if not isinstance(video_embeds, torch.Tensor):
1403
- raise ValueError(
1404
- "Incorrect type of video embeddings. "
1405
- f"Got type: {type(video_embeds)}"
1406
- )
1347
return Qwen2_5_VLVideoEmbeddingInputs(
1348
type="video_embeds",
1349
video_embeds=video_embeds,
vllm/model_executor/models/qwen_vl.py
+4
-14
@@ -58,7 +58,6 @@ from .interfaces import (
58
SupportsPP,
59
)
60
from .qwen import QWenBaseModel, QWenModel
61
-from .utils import flatten_bn
61
62
63
class QwenImagePixelInputs(TensorSchema):
@@ -703,6 +702,8 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
702
class QwenVLForConditionalGeneration(
703
QWenBaseModel, SupportsPP, SupportsLoRA, SupportsMultiModal
704
):
705
+ merge_by_field_config = True
706
+
707
packed_modules_mapping = {
708
"c_attn": ["c_attn"],
709
"gate_up_proj": [
@@ -750,30 +751,19 @@ class QwenVLForConditionalGeneration(
751
image_embeds = kwargs.pop("image_embeds", None)
752
753
if pixel_values is not None:
753
- if not isinstance(pixel_values, (torch.Tensor, list)):
754
- raise ValueError(
755
- f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
756
- )
757
-
754
expected_h = expected_w = self.config.visual["image_size"]
755
resolve_bindings = {"h": expected_h, "w": expected_w}
756
757
return QwenImagePixelInputs(
758
type="pixel_values",
763
- data=flatten_bn(pixel_values, concat=True),
759
+ data=pixel_values,
760
resolve_bindings=resolve_bindings,
761
)
762
763
if image_embeds is not None:
768
- if not isinstance(image_embeds, (torch.Tensor, list)):
769
- raise ValueError(
770
- "Incorrect type of image embeddings. "
771
- f"Got type: {type(image_embeds)}"
772
- )
773
-
764
return QwenImageEmbeddingInputs(
765
type="image_embeds",
776
- data=flatten_bn(image_embeds, concat=True),
766
+ data=image_embeds,
767
)
768
769
return None