A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Fix the issue where image embeddings were incorrectly split.… (#23366)
Signed-off-by: bppps <bpppsaka@gmail.com> Co-authored-by: zouyu.zzx <zouyu.zzx@alibaba-inc.com> Co-authored-by: bppps <bpppsaka@gmail.com>
bppps committed
Aug 23, 2025 at 00:56 UTC
424fb7a5d22c013ec0ef6244c62cd75ed076375a
3 files changed
+104
-57
vllm/model_executor/models/glm4_1v.py
+5
-2
@@ -74,7 +74,8 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
74
from ..layers.activation import SiluAndMul
75
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
76
SupportsMultiModal, SupportsPP)
77
-from .qwen2_vl import _qwen2vl_field_config, apply_rotary_pos_emb_vision
77
+from .qwen2_vl import (_create_qwen2vl_field_factory,
78
+ apply_rotary_pos_emb_vision)
79
from .utils import (AutoWeightsLoader, WeightsMapper,
80
init_vllm_registered_model, maybe_prefix,
81
merge_multimodal_embeddings)
@@ -1153,7 +1154,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1154
hf_inputs: BatchFeature,
1155
hf_processor_mm_kwargs: Mapping[str, object],
1156
) -> Mapping[str, MultiModalFieldConfig]:
1156
- return _qwen2vl_field_config(hf_inputs)
1157
+ return _create_qwen2vl_field_factory(
1158
+ self.info.get_hf_config().vision_config.spatial_merge_size)(
1159
+ hf_inputs)
1160
1161
def _get_prompt_updates(
1162
self,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+54
-32
@@ -25,7 +25,7 @@
25
from collections.abc import Iterable, Mapping, Sequence
26
from copy import copy
27
from functools import partial
28
-from typing import Any, Optional, Union
28
+from typing import Any, Callable, Optional, Union
29
30
import torch
31
import torch.nn as nn
@@ -79,40 +79,57 @@ except (ImportError, ModuleNotFoundError):
79
logger = init_logger(__name__)
80
81
82
-def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
83
- audio_feature_lengths = hf_inputs.get("audio_feature_lengths",
84
- torch.empty((0, )))
85
-
86
- image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
87
- image_grid_sizes = image_grid_thw.prod(-1)
88
-
89
- video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
90
- video_grid_sizes = video_grid_thw.prod(-1)
91
-
92
- num_videos = len(video_grid_sizes)
82
+def create_qwen2_5_omni_thinker_field_factory(
83
+ spatial_merge_size: int
84
+) -> Callable[[Mapping[str, torch.Tensor]], Mapping[str,
85
+ MultiModalFieldConfig]]:
86
+
87
+ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str,
88
+ torch.Tensor]):
89
+ audio_feature_lengths = hf_inputs.get("audio_feature_lengths",
90
+ torch.empty((0, )))
91
+
92
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
93
+ image_pixel_grid_sizes = image_grid_thw.prod(-1)
94
+ image_embed_grid_sizes = (image_pixel_grid_sizes //
95
+ spatial_merge_size // spatial_merge_size)
96
+
97
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
98
+ video_grid_sizes = video_grid_thw.prod(-1)
99
+ video_embed_grid_sizes = (video_grid_sizes // spatial_merge_size //
100
+ spatial_merge_size)
101
+
102
+ num_videos = len(video_grid_sizes)
103
+
104
+ return dict(
105
+ input_audio_features=MultiModalFieldConfig.flat_from_sizes(
106
+ "audio", audio_feature_lengths, dim=1),
107
+ feature_attention_mask=MultiModalFieldConfig.batched("audio"),
108
+ audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
109
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
110
+ "image", image_pixel_grid_sizes),
111
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
112
+ "image", image_embed_grid_sizes),
113
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
114
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
115
+ "video", video_grid_sizes),
116
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
117
+ "video", video_embed_grid_sizes),
118
+ video_grid_thw=MultiModalFieldConfig.batched("video"),
119
+ second_per_grid_ts=MultiModalFieldConfig.batched("video"),
120
+ use_audio_in_video=MultiModalFieldConfig.shared(
121
+ "video", num_videos),
122
+ )
123
94
- return dict(
95
- input_audio_features=MultiModalFieldConfig.flat_from_sizes(
96
- "audio", audio_feature_lengths, dim=1),
97
- feature_attention_mask=MultiModalFieldConfig.batched("audio"),
98
- audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
99
- pixel_values=MultiModalFieldConfig.flat_from_sizes(
100
- "image", image_grid_sizes),
101
- image_embeds=MultiModalFieldConfig.flat_from_sizes(
102
- "image", image_grid_sizes),
103
- image_grid_thw=MultiModalFieldConfig.batched("image"),
104
- pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
105
- "video", video_grid_sizes),
106
- video_embeds=MultiModalFieldConfig.flat_from_sizes(
107
- "video", video_grid_sizes),
108
- video_grid_thw=MultiModalFieldConfig.batched("video"),
109
- second_per_grid_ts=MultiModalFieldConfig.batched("video"),
110
- use_audio_in_video=MultiModalFieldConfig.shared("video", num_videos),
111
- )
124
+ return _qwen2_5_omni_thinker_field_config
125
126
127
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
128
129
+ def __init__(self, spatial_merge_size: int, *args, **kwargs):
130
+ self._spatial_merge_size = spatial_merge_size
131
+ super().__init__(self._spatial_merge_size, *args, **kwargs)
132
+
133
def _parse_audio_data(
134
self,
135
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
@@ -124,7 +141,8 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
141
required_fields={
142
"input_audio_features", "audio_feature_lengths"
143
},
127
- fields_factory=_qwen2_5_omni_thinker_field_config,
144
+ fields_factory=create_qwen2_5_omni_thinker_field_factory(
145
+ self._spatial_merge_size),
146
)
147
148
return super()._parse_audio_data(data)
@@ -214,6 +232,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
232
def _get_data_parser(self) -> MultiModalDataParser:
233
feature_extractor = self.info.get_feature_extractor()
234
return Qwen2_5OmniThinkerMultiModalDataParser(
235
+ spatial_merge_size=self.info.get_hf_config(
236
+ ).vision_config.spatial_merge_size,
237
target_sr=feature_extractor.sampling_rate)
238
239
def _call_hf_processor(
@@ -265,7 +285,9 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
285
hf_inputs: BatchFeature,
286
hf_processor_mm_kwargs: Mapping[str, object],
287
) -> Mapping[str, MultiModalFieldConfig]:
268
- return _qwen2_5_omni_thinker_field_config(hf_inputs)
288
+ return create_qwen2_5_omni_thinker_field_factory(
289
+ self.info.get_hf_config().vision_config.spatial_merge_size)(
290
+ hf_inputs)
291
292
def _maybe_apply_prompt_updates(
293
self,
vllm/model_executor/models/qwen2_vl.py
+45
-23
@@ -699,29 +699,46 @@ class Qwen2VisionTransformer(nn.Module):
699
return loaded_params
700
701
702
-def _qwen2vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
703
- image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
704
- image_grid_sizes = image_grid_thw.prod(-1)
705
-
706
- video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
707
- video_grid_sizes = video_grid_thw.prod(-1)
708
-
709
- return dict(
710
- pixel_values=MultiModalFieldConfig.flat_from_sizes(
711
- "image", image_grid_sizes),
712
- image_embeds=MultiModalFieldConfig.flat_from_sizes(
713
- "image", image_grid_sizes),
714
- image_grid_thw=MultiModalFieldConfig.batched("image"),
715
- pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
716
- "video", video_grid_sizes),
717
- video_embeds=MultiModalFieldConfig.flat_from_sizes(
718
- "video", video_grid_sizes),
719
- video_grid_thw=MultiModalFieldConfig.batched("video"),
720
- )
702
+def _create_qwen2vl_field_factory(
703
+ spatial_merge_size: int
704
+) -> Callable[
705
+ [Mapping[str, torch.Tensor]],
706
+ Mapping[str, MultiModalFieldConfig],
707
+]:
708
+
709
+ def _qwen2vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
710
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
711
+ image_pixel_grid_sizes = image_grid_thw.prod(-1)
712
+ image_embed_grid_sizes = (image_pixel_grid_sizes //
713
+ spatial_merge_size // spatial_merge_size)
714
+
715
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
716
+ video_grid_sizes = video_grid_thw.prod(-1)
717
+ video_embed_grid_sizes = (video_grid_sizes // spatial_merge_size //
718
+ spatial_merge_size)
719
+
720
+ return dict(
721
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
722
+ "image", image_pixel_grid_sizes),
723
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
724
+ "image", image_embed_grid_sizes),
725
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
726
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
727
+ "video", video_grid_sizes),
728
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
729
+ "video", video_embed_grid_sizes),
730
+ video_grid_thw=MultiModalFieldConfig.batched("video"),
731
+ )
732
+
733
+ return _qwen2vl_field_config
734
735
736
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
737
738
+ def __init__(self, spatial_merge_size: int, *args, **kwargs):
739
+ self._spatial_merge_size = spatial_merge_size
740
+ super().__init__(*args, **kwargs)
741
+
742
def _parse_image_data(
743
self,
744
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
@@ -731,7 +748,8 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
748
data,
749
modality="image",
750
required_fields={"image_embeds", "image_grid_thw"},
734
- fields_factory=_qwen2vl_field_config,
751
+ fields_factory=_create_qwen2vl_field_factory(
752
+ self._spatial_merge_size),
753
)
754
755
return super()._parse_image_data(data)
@@ -745,7 +763,8 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
763
data,
764
modality="video",
765
required_fields={"video_embeds", "video_grid_thw"},
748
- fields_factory=_qwen2vl_field_config,
766
+ fields_factory=_create_qwen2vl_field_factory(
767
+ self._spatial_merge_size),
768
)
769
770
return super()._parse_video_data(data)
@@ -967,7 +986,8 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
986
):
987
988
def _get_data_parser(self) -> MultiModalDataParser:
970
- return Qwen2VLMultiModalDataParser()
989
+ return Qwen2VLMultiModalDataParser(
990
+ self.info.get_hf_config().vision_config.spatial_merge_size)
991
992
def _get_prompt_updates(
993
self,
@@ -1010,7 +1030,9 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
1030
hf_inputs: BatchFeature,
1031
hf_processor_mm_kwargs: Mapping[str, object],
1032
) -> Mapping[str, MultiModalFieldConfig]:
1013
- return _qwen2vl_field_config(hf_inputs)
1033
+ return _create_qwen2vl_field_factory(
1034
+ self.info.get_hf_config().vision_config.spatial_merge_size)(
1035
+ hf_inputs)
1036
1037
1038
@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,