A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[Model] Use mm_position to compute mrope positions for Qwen2-VL/2.5-VL (#32126)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
YunzhuLu committed
Jan 13, 2026 at 17:04 UTC
542a4059b2bb0f790e82822c8b9cbcf8cde91adb
2 files changed
+113
-190
vllm/model_executor/models/qwen2_5_vl.py
+57
-95
@@ -26,11 +26,12 @@
26
# limitations under the License.
27
"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
28
29
-from collections.abc import Callable, Iterable, Mapping, Sequence
29
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
30
from functools import lru_cache, partial
31
from typing import Annotated, Any, Literal, TypeAlias
32
33
import einops
34
+import numpy as np
35
import torch
36
import torch.nn as nn
37
import torch.nn.functional as F
@@ -1044,121 +1045,82 @@ class Qwen2_5_VLForConditionalGeneration(
1045
1046
supports_encoder_tp_data = True
1047
1048
+ def iter_mm_grid_thw(
1049
+ self, mm_features: list[MultiModalFeatureSpec]
1050
+ ) -> Iterator[tuple[int, int, int, int, float]]:
1051
+ """
1052
+ Iterate over multimodal features and yield grid information.
1053
+
1054
+ Args:
1055
+ mm_features: List of multimodal feature specifications
1056
+
1057
+ Yields:
1058
+ Tuple of (offset, grid_t, grid_h, grid_w, t_factor) for each frame/image
1059
+ """
1060
+ spatial_merge_size = self.config.vision_config.spatial_merge_size
1061
+ tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
1062
+ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
1063
+ offset = mm_feature.mm_position.offset
1064
+ if mm_feature.modality == "image":
1065
+ t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1066
+ assert t == 1, f"Image must have 1 frame, got {t}"
1067
+ yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
1068
+ elif mm_feature.modality == "video":
1069
+ t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1070
+ second_per_grid_ts = 1.0
1071
+ if mm_feature.data.get("second_per_grid_ts", None):
1072
+ second_per_grid_ts = mm_feature.data[
1073
+ "second_per_grid_ts"
1074
+ ].data.item()
1075
+ t_factor = second_per_grid_ts * tokens_per_second
1076
+ yield (
1077
+ offset,
1078
+ t,
1079
+ h // spatial_merge_size,
1080
+ w // spatial_merge_size,
1081
+ t_factor,
1082
+ )
1083
+ else:
1084
+ raise ValueError(f"Unsupported modality: {mm_feature.modality}")
1085
+
1086
def get_mrope_input_positions(
1087
self,
1088
input_tokens: list[int],
1089
mm_features: list[MultiModalFeatureSpec],
1090
) -> tuple[torch.Tensor, int]:
1052
- kwargs = MultiModalFeatureSpec.gather_kwargs(
1053
- mm_features,
1054
- {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"},
1055
- )
1056
- image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
1057
- video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]
1058
- second_per_grid_ts = kwargs.get("second_per_grid_ts", [])
1059
-
1060
- hf_config = self.config
1061
- image_token_id = hf_config.image_token_id
1062
- video_token_id = hf_config.video_token_id
1063
- vision_start_token_id = hf_config.vision_start_token_id
1064
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
1065
- tokens_per_second = getattr(hf_config.vision_config, "tokens_per_second", 1.0)
1066
-
1067
- input_tokens_tensor = torch.tensor(input_tokens)
1068
- vision_start_indices = torch.argwhere(
1069
- input_tokens_tensor == vision_start_token_id
1070
- ).squeeze(1)
1071
- vision_tokens = input_tokens_tensor[vision_start_indices + 1]
1072
- image_nums = (vision_tokens == image_token_id).sum()
1073
- video_nums = (vision_tokens == video_token_id).sum()
1091
llm_pos_ids_list: list = []
1075
-
1092
st = 0
1077
- remain_images, remain_videos = image_nums, video_nums
1078
-
1079
- image_index, video_index = 0, 0
1080
- for _ in range(image_nums + video_nums):
1081
- video_second_per_grid_t = 0.0
1082
- if remain_images > 0:
1083
- try:
1084
- ed_image = input_tokens.index(image_token_id, st)
1085
- except ValueError:
1086
- ed_image = len(input_tokens) + 1
1087
- else:
1088
- ed_image = len(input_tokens) + 1
1089
- if remain_videos > 0:
1090
- try:
1091
- ed_video = input_tokens.index(video_token_id, st)
1092
- except ValueError:
1093
- ed_video = len(input_tokens) + 1
1094
- else:
1095
- ed_video = len(input_tokens) + 1
1096
- if ed_image < ed_video:
1097
- t, h, w = image_grid_thw[image_index]
1098
- image_index += 1
1099
- remain_images -= 1
1100
- ed = ed_image
1101
- else:
1102
- t, h, w = video_grid_thw[video_index]
1103
- video_second_per_grid_t = 1.0
1104
- if second_per_grid_ts:
1105
- video_second_per_grid_t = second_per_grid_ts[video_index]
1106
- video_index += 1
1107
- remain_videos -= 1
1108
- ed = ed_video
1109
-
1110
- llm_grid_t, llm_grid_h, llm_grid_w = (
1111
- t,
1112
- h // spatial_merge_size,
1113
- w // spatial_merge_size,
1114
- )
1115
- text_len = ed - st
1093
1094
+ for (
1095
+ offset,
1096
+ llm_grid_t,
1097
+ llm_grid_h,
1098
+ llm_grid_w,
1099
+ t_factor,
1100
+ ) in self.iter_mm_grid_thw(mm_features):
1101
+ text_len = offset - st
1102
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1103
llm_pos_ids_list.append(
1119
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1104
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1105
)
1106
1122
- t_index = (
1123
- (
1124
- torch.arange(llm_grid_t)
1125
- .view(-1, 1)
1126
- .expand(-1, llm_grid_h * llm_grid_w)
1127
- * video_second_per_grid_t
1128
- * tokens_per_second
1129
- )
1130
- .long()
1131
- .flatten()
1132
- )
1133
-
1134
- h_index = (
1135
- torch.arange(llm_grid_h)
1136
- .view(1, -1, 1)
1137
- .expand(llm_grid_t, -1, llm_grid_w)
1138
- .flatten()
1139
- )
1140
- w_index = (
1141
- torch.arange(llm_grid_w)
1142
- .view(1, 1, -1)
1143
- .expand(llm_grid_t, llm_grid_h, -1)
1144
- .flatten()
1145
- )
1146
- llm_pos_ids_list.append(
1147
- torch.stack([t_index, h_index, w_index]) + text_len + st_idx
1148
- )
1149
- st = ed + llm_grid_t * llm_grid_h * llm_grid_w
1107
+ grid_indices = np.indices((llm_grid_t, llm_grid_h, llm_grid_w))
1108
+ if t_factor != 1.0:
1109
+ grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
1110
+ llm_pos_ids_list.append(grid_indices.reshape(3, -1) + text_len + st_idx)
1111
+ st = offset + llm_grid_t * llm_grid_h * llm_grid_w
1112
1113
if st < len(input_tokens):
1114
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1115
text_len = len(input_tokens) - st
1116
llm_pos_ids_list.append(
1155
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1117
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1118
)
1119
1158
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1120
+ llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
1121
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1122
1161
- return llm_positions, mrope_position_delta
1123
+ return torch.from_numpy(llm_positions), mrope_position_delta
1124
1125
@classmethod
1126
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
vllm/model_executor/models/qwen2_vl.py
+56
-95
@@ -26,7 +26,7 @@
26
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
27
28
import math
29
-from collections.abc import Callable, Iterable, Mapping, Sequence
29
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
30
from functools import partial
31
from typing import Annotated, Any, Literal, TypeAlias
32
@@ -1137,121 +1137,82 @@ class Qwen2VLForConditionalGeneration(
1137
1138
supports_encoder_tp_data = True
1139
1140
+ def iter_mm_grid_thw(
1141
+ self, mm_features: list[MultiModalFeatureSpec]
1142
+ ) -> Iterator[tuple[int, int, int, int, float]]:
1143
+ """
1144
+ Iterate over multimodal features and yield grid information.
1145
+
1146
+ Args:
1147
+ mm_features: List of multimodal feature specifications
1148
+
1149
+ Yields:
1150
+ Tuple of (offset, grid_t, grid_h, grid_w, t_factor) for each frame/image
1151
+ """
1152
+ spatial_merge_size = self.config.vision_config.spatial_merge_size
1153
+ tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
1154
+ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
1155
+ offset = mm_feature.mm_position.offset
1156
+ if mm_feature.modality == "image":
1157
+ t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1158
+ assert t == 1, f"Image must have 1 frame, got {t}"
1159
+ yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
1160
+ elif mm_feature.modality == "video":
1161
+ t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1162
+ second_per_grid_ts = 1.0
1163
+ if mm_feature.data.get("second_per_grid_ts", None):
1164
+ second_per_grid_ts = mm_feature.data[
1165
+ "second_per_grid_ts"
1166
+ ].data.item()
1167
+ t_factor = second_per_grid_ts * tokens_per_second
1168
+ yield (
1169
+ offset,
1170
+ t,
1171
+ h // spatial_merge_size,
1172
+ w // spatial_merge_size,
1173
+ t_factor,
1174
+ )
1175
+ else:
1176
+ raise ValueError(f"Unsupported modality: {mm_feature.modality}")
1177
+
1178
def get_mrope_input_positions(
1179
self,
1180
input_tokens: list[int],
1181
mm_features: list[MultiModalFeatureSpec],
1182
) -> tuple[torch.Tensor, int]:
1145
- kwargs = MultiModalFeatureSpec.gather_kwargs(
1146
- mm_features,
1147
- {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"},
1148
- )
1149
- image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]
1150
- video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]
1151
- second_per_grid_ts = kwargs.get("second_per_grid_ts", [])
1152
-
1153
- hf_config = self.config
1154
- image_token_id = hf_config.image_token_id
1155
- video_token_id = hf_config.video_token_id
1156
- vision_start_token_id = hf_config.vision_start_token_id
1157
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
1158
- tokens_per_second = getattr(hf_config.vision_config, "tokens_per_second", 1.0)
1159
-
1160
- input_tokens_tensor = torch.tensor(input_tokens)
1161
- vision_start_indices = torch.argwhere(
1162
- input_tokens_tensor == vision_start_token_id
1163
- ).squeeze(1)
1164
- vision_tokens = input_tokens_tensor[vision_start_indices + 1]
1165
- image_nums = (vision_tokens == image_token_id).sum()
1166
- video_nums = (vision_tokens == video_token_id).sum()
1183
llm_pos_ids_list: list = []
1168
-
1184
st = 0
1170
- remain_images, remain_videos = image_nums, video_nums
1171
-
1172
- image_index, video_index = 0, 0
1173
- for _ in range(image_nums + video_nums):
1174
- video_second_per_grid_t = 0.0
1175
- if remain_images > 0:
1176
- try:
1177
- ed_image = input_tokens.index(image_token_id, st)
1178
- except ValueError:
1179
- ed_image = len(input_tokens) + 1
1180
- else:
1181
- ed_image = len(input_tokens) + 1
1182
- if remain_videos > 0:
1183
- try:
1184
- ed_video = input_tokens.index(video_token_id, st)
1185
- except ValueError:
1186
- ed_video = len(input_tokens) + 1
1187
- else:
1188
- ed_video = len(input_tokens) + 1
1189
- if ed_image < ed_video:
1190
- t, h, w = image_grid_thw[image_index]
1191
- image_index += 1
1192
- remain_images -= 1
1193
- ed = ed_image
1194
- else:
1195
- t, h, w = video_grid_thw[video_index]
1196
- video_second_per_grid_t = 1.0
1197
- if second_per_grid_ts:
1198
- video_second_per_grid_t = second_per_grid_ts[video_index]
1199
- video_index += 1
1200
- remain_videos -= 1
1201
- ed = ed_video
1202
-
1203
- llm_grid_t, llm_grid_h, llm_grid_w = (
1204
- t,
1205
- h // spatial_merge_size,
1206
- w // spatial_merge_size,
1207
- )
1208
- text_len = ed - st
1185
1186
+ for (
1187
+ offset,
1188
+ llm_grid_t,
1189
+ llm_grid_h,
1190
+ llm_grid_w,
1191
+ t_factor,
1192
+ ) in self.iter_mm_grid_thw(mm_features):
1193
+ text_len = offset - st
1194
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1195
llm_pos_ids_list.append(
1212
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1213
- )
1214
-
1215
- t_index = (
1216
- (
1217
- torch.arange(llm_grid_t)
1218
- .view(-1, 1)
1219
- .expand(-1, llm_grid_h * llm_grid_w)
1220
- * video_second_per_grid_t
1221
- * tokens_per_second
1222
- )
1223
- .long()
1224
- .flatten()
1196
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1197
)
1198
1227
- h_index = (
1228
- torch.arange(llm_grid_h)
1229
- .view(1, -1, 1)
1230
- .expand(llm_grid_t, -1, llm_grid_w)
1231
- .flatten()
1232
- )
1233
- w_index = (
1234
- torch.arange(llm_grid_w)
1235
- .view(1, 1, -1)
1236
- .expand(llm_grid_t, llm_grid_h, -1)
1237
- .flatten()
1238
- )
1239
- llm_pos_ids_list.append(
1240
- torch.stack([t_index, h_index, w_index]) + text_len + st_idx
1241
- )
1242
- st = ed + llm_grid_t * llm_grid_h * llm_grid_w
1199
+ grid_indices = np.indices((llm_grid_t, llm_grid_h, llm_grid_w))
1200
+ if t_factor != 1.0:
1201
+ grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
1202
+ llm_pos_ids_list.append(grid_indices.reshape(3, -1) + text_len + st_idx)
1203
+ st = offset + llm_grid_t * llm_grid_h * llm_grid_w
1204
1205
if st < len(input_tokens):
1206
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1207
text_len = len(input_tokens) - st
1208
llm_pos_ids_list.append(
1248
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1209
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1210
)
1211
1251
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1212
+ llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
1213
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1214
1254
- return llm_positions, mrope_position_delta
1215
+ return torch.from_numpy(llm_positions), mrope_position_delta
1216
1217
@classmethod
1218
def get_placeholder_str(cls, modality: str, i: int) -> str | None: