[Bugfix][Qwen2.5-VL] Honor video fps for temporal M-RoPE (#47736)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Ting SUN committed
Sep 22, 2026 at 13:56 UTC
a33b3bac5dad5654065adbdadc87f35118e482e3
7 files changed
+117
-8
examples/generate/multimodal/vision_language_offline.py
+1
@@ -2252,6 +2252,7 @@ MODELS_NEED_VIDEO_METADATA = [
2252
"glm4_5v",
2253
"glm4_5v_fp8",
2254
"molmo2",
2255
+ "qwen2_5_vl",
2256
"qwen3_vl",
2257
"qwen3_vl_moe",
2258
"qwen3_5",
tests/models/multimodal/generation/test_common.py
+2
@@ -151,10 +151,12 @@ VLM_TEST_SETTINGS = {
151
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
152
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
153
enforce_eager=False,
154
+ needs_video_metadata=True,
155
max_model_len=4096,
156
max_num_seqs=2,
157
auto_cls=AutoModelForImageTextToText,
158
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
159
+ patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
160
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
161
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
162
),
tests/models/multimodal/generation/test_qwen2_5_vl.py
+7
-3
@@ -4,10 +4,10 @@
4
import pytest
5
6
from vllm.assets.image import ImageAsset
7
-from vllm.multimodal.video import sample_frames_from_video
7
from vllm.platforms import current_platform
8
9
from ....conftest import VIDEO_ASSETS
10
+from .vlm_utils.builders import sample_frames_with_video_metadata
11
12
models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
13
target_dtype = "bfloat16"
@@ -81,7 +81,9 @@ def test_qwen2_5_vl_evs_functionality(
81
82
# Sample frames from video assets
83
sampled_vids = [
84
- sample_frames_from_video(asset.np_ndarrays, num_frames)
84
+ sample_frames_with_video_metadata(
85
+ (asset.np_ndarrays, asset.metadata), num_frames
86
+ )
87
for asset in video_assets
88
]
89
@@ -145,7 +147,9 @@ def test_qwen2_5_vl_evs_batched_videos(
147
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
148
# Sample frames from video assets
149
sampled_vids = [
148
- sample_frames_from_video(asset.np_ndarrays, num_frames)
150
+ sample_frames_with_video_metadata(
151
+ (asset.np_ndarrays, asset.metadata), num_frames
152
+ )
153
for asset in video_assets
154
]
155
tests/models/multimodal/generation/test_vit_cudagraph.py
+1
-1
@@ -144,7 +144,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
144
"<|vision_start|><|video_pad|><|vision_end|>"
145
"Describe this video in one sentence."
146
),
147
- needs_video_metadata=False,
147
+ needs_video_metadata=True,
148
marks=[pytest.mark.core_model],
149
),
150
"kimi_vl": VitCudagraphTestConfig(
tests/models/multimodal/processing/test_qwen2_vl.py
+32
@@ -1,6 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from typing import Any
5
+
6
+import numpy as np
7
import pytest
8
import torch
9
from PIL import Image
@@ -208,6 +211,35 @@ def test_get_image_size_with_most_features(
211
assert tokens < max_tokens
212
213
214
+def _build_qwen2_5_vl_video_mm_data(num_frames: int, fps: float) -> dict[str, Any]:
215
+ video = np.zeros((num_frames, 56, 56, 3), dtype=np.uint8)
216
+ metadata = {
217
+ "fps": fps,
218
+ "duration": num_frames / fps,
219
+ "total_num_frames": num_frames,
220
+ "frames_indices": list(range(num_frames)),
221
+ "video_backend": "opencv",
222
+ "do_sample_frames": False,
223
+ }
224
+ return {"video": [(video, metadata)]}
225
+
226
+
227
+@pytest.mark.parametrize("model_id", ["Qwen/Qwen2.5-VL-3B-Instruct"])
228
+def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
229
+ ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 0, "video": 1})
230
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
231
+ temporal_patch_size = (
232
+ processor.info.get_hf_processor().video_processor.temporal_patch_size
233
+ )
234
+
235
+ prompt = "<|vision_start|><|video_pad|><|vision_end|>"
236
+ for fps in (1.0, 30.0):
237
+ mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
238
+ processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
239
+ spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
240
+ assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
241
+
242
+
243
@pytest.mark.parametrize(
244
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
245
)
tests/renderers/test_process_multi_modal_uuids.py
+7
-3
@@ -15,7 +15,11 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
15
16
cherry_pil_image = ImageAsset("cherry_blossom").pil_image
17
stop_pil_image = ImageAsset("stop_sign").pil_image
18
-baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays
18
+baby_reading_video_asset = VideoAsset("baby_reading")
19
+baby_reading_video = (
20
+ baby_reading_video_asset.np_ndarrays,
21
+ baby_reading_video_asset.metadata,
22
+)
23
24
25
def _build_renderer(
@@ -127,7 +131,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
131
132
mm_data = {
133
"image": [cherry_pil_image, stop_pil_image],
130
- "video": baby_reading_np_ndarrays,
134
+ "video": baby_reading_video,
135
}
136
137
# Use a consistent two-image scenario across all configurations
@@ -207,7 +211,7 @@ def test_multi_modal_uuids_preserved_when_caching_disabled(mm_uuids, expected):
211
request_id = "req-42"
212
mm_data = {
213
"image": [cherry_pil_image, stop_pil_image],
210
- "video": baby_reading_np_ndarrays,
214
+ "video": baby_reading_video,
215
}
216
217
mm_processor = renderer.get_mm_processor()
vllm/model_executor/models/qwen2_5_vl.py
+67
-1
@@ -41,12 +41,14 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
41
Qwen2_5_VLConfig,
42
Qwen2_5_VLVisionConfig,
43
)
44
+from transformers.video_utils import VideoMetadata
45
46
from vllm.compilation.decorators import (
47
should_torch_compile_mm_encoder,
48
support_torch_compile,
49
)
50
from vllm.config import VllmConfig
51
+from vllm.config.multimodal import VideoDummyOptions
52
from vllm.distributed import parallel_state
53
from vllm.distributed import utils as dist_utils
54
from vllm.logger import init_logger
@@ -72,9 +74,11 @@ from vllm.multimodal.inputs import (
74
MultiModalFieldConfig,
75
MultiModalKwargsItem,
76
MultiModalKwargsItems,
77
+ VideoItem,
78
)
79
from vllm.multimodal.parse import MultiModalDataItems
80
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
81
+from vllm.multimodal.processing.processor import HFMultiModalInputs
82
from vllm.multimodal.video_prune.evs import (
83
compute_mrope_for_media,
84
compute_retained_tokens_count,
@@ -102,8 +106,9 @@ from .interfaces import (
106
SupportsPP,
107
SupportsQuant,
108
)
105
-from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
109
from .qwen2_vl import (
110
+ Qwen2VLDummyInputsBuilder,
111
+ Qwen2VLMultiModalDataParser,
112
Qwen2VLMultiModalProcessor,
113
Qwen2VLProcessingInfo,
114
)
@@ -1137,6 +1142,40 @@ class Qwen2_5_VisionTransformer(nn.Module):
1142
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1143
1144
1145
+class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
1146
+ def _get_dummy_videos(
1147
+ self,
1148
+ *,
1149
+ width: int,
1150
+ height: int,
1151
+ num_frames: int,
1152
+ num_videos: int,
1153
+ overrides: VideoDummyOptions | None = None,
1154
+ ) -> list[VideoItem]:
1155
+ videos = super()._get_dummy_videos(
1156
+ width=width,
1157
+ height=height,
1158
+ num_frames=num_frames,
1159
+ num_videos=num_videos,
1160
+ overrides=overrides,
1161
+ )
1162
+ videos = [v.copy() for v in videos]
1163
+
1164
+ video_items = []
1165
+ for video in videos:
1166
+ video_num_frames = video.shape[0]
1167
+ video_metadata = {
1168
+ "fps": 2.0,
1169
+ "duration": video_num_frames / 2.0,
1170
+ "total_num_frames": video_num_frames,
1171
+ "frames_indices": list(range(video_num_frames)),
1172
+ "video_backend": "opencv",
1173
+ "do_sample_frames": False,
1174
+ }
1175
+ video_items.append((video, video_metadata))
1176
+ return video_items
1177
+
1178
+
1179
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
1180
def get_hf_config(self):
1181
return self.ctx.get_hf_config(Qwen2_5_VLConfig)
@@ -1148,6 +1187,14 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
1187
**kwargs,
1188
)
1189
1190
+ def get_data_parser(self):
1191
+ return Qwen2VLMultiModalDataParser(
1192
+ self.get_hf_config().vision_config.spatial_merge_size,
1193
+ video_needs_metadata=True,
1194
+ expected_hidden_size=self._get_expected_hidden_size(),
1195
+ allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
1196
+ )
1197
+
1198
1199
class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
1200
def _get_mm_fields_config(
@@ -1160,6 +1207,25 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
1207
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
1208
)
1209
1210
+ def _get_hf_mm_inputs(
1211
+ self,
1212
+ mm_items: MultiModalDataItems,
1213
+ hf_kwargs: Mapping[str, object],
1214
+ ) -> HFMultiModalInputs:
1215
+ hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
1216
+ hf_data = hf_inputs.hf_data
1217
+
1218
+ if videos := hf_data.get("videos"):
1219
+ # HF expects video metadata as a separate argument.
1220
+ hf_data["videos"] = [video for video, _ in videos]
1221
+ hf_data["video_metadata"] = [
1222
+ VideoMetadata(
1223
+ **{k: metadata[k] for k in metadata if k != "do_sample_frames"}
1224
+ )
1225
+ for _, metadata in videos
1226
+ ]
1227
+ return hf_inputs
1228
+
1229
def _get_prompt_updates(
1230
self,
1231
mm_items: MultiModalDataItems,