[Multimodal] Add Qwen2-VL/Qwen2.5-VL processor-mapped video loader (#45555)
Signed-off-by: Ranran <hzz5361@psu.edu> Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Ranran committed
Jun 21, 2026 at 13:56 UTC
635c38338afe132f9555ebf4a7c8ac7dfb05b2a0
4 files changed
+117
-4
tests/entrypoints/pooling/classify/test_online_vision.py
+2
-2
@@ -25,7 +25,7 @@ def server():
25
"--runner",
26
"pooling",
27
"--max-model-len",
28
- "5000",
28
+ "16384",
29
"--enforce-eager",
30
"--limit-mm-per-prompt",
31
json.dumps({"video": MAXIMUM_VIDEOS}),
@@ -143,4 +143,4 @@ def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
143
assert output.model == model_name
144
assert len(output.data) == 1
145
assert len(output.data[0].probs) == 2
146
- assert output.usage.prompt_tokens == 4807
146
+ assert output.usage.prompt_tokens == 8993
tests/multimodal/test_video.py
+25
-2
@@ -15,6 +15,7 @@ from vllm.multimodal.video import (
15
DynamicVideoBackend,
16
GLM46VVideoBackend,
17
Molmo2VideoBackend,
18
+ Qwen2VLVideoBackend,
19
Qwen3VLVideoBackend,
20
VideoLoader,
21
VideoSourceMetadata,
@@ -70,11 +71,12 @@ def test_video_loader_type_doesnt_exist():
71
72
73
@pytest.mark.parametrize(
73
- "model_repo, expected_loader_cls",
74
+ "model_repo, expected_loader_cls, hf_sample_kwargs",
75
[
76
pytest.param(
77
"allenai/Molmo2-4B",
78
Molmo2VideoBackend,
79
+ None,
80
marks=pytest.mark.skip(
81
reason="Video processor not aligned, investigate later.",
82
),
@@ -83,23 +85,44 @@ def test_video_loader_type_doesnt_exist():
85
pytest.param(
86
"zai-org/GLM-4.1V-9B-Thinking",
87
DynamicVideoBackend,
88
+ None,
89
id="glm4v",
90
),
91
pytest.param(
92
"zai-org/GLM-4.6V-Flash",
93
GLM46VVideoBackend,
94
+ None,
95
id="glm46v",
96
),
97
pytest.param(
98
"Qwen/Qwen3-VL-4B-Instruct",
99
Qwen3VLVideoBackend,
100
+ None,
101
id="qwen3vl",
102
),
103
+ # Qwen2-VL/Qwen2.5-VL ship no ``video_processor_type`` in their
104
+ # preprocessor config, so resolution relies on the model_type ->
105
+ # video processor fallback in get_video_processor_cls_name_from_config.
106
+ # They also ship no default fps/num_frames, so the HF sampler needs an
107
+ # explicit target rate; pass fps=2 to match the loader default.
108
+ pytest.param(
109
+ "Qwen/Qwen2-VL-7B-Instruct",
110
+ Qwen2VLVideoBackend,
111
+ {"fps": 2},
112
+ id="qwen2vl",
113
+ ),
114
+ pytest.param(
115
+ "Qwen/Qwen2.5-VL-7B-Instruct",
116
+ Qwen2VLVideoBackend,
117
+ {"fps": 2},
118
+ id="qwen2_5_vl",
119
+ ),
120
],
121
)
122
def test_video_processor_from_model_repo(
123
model_repo: str,
124
expected_loader_cls: type,
125
+ hf_sample_kwargs: dict[str, int | float] | None,
126
):
127
"""Test that a model repo resolves to the correct video loader backend.
128
@@ -143,7 +166,7 @@ def test_video_processor_from_model_repo(
166
fps=vllm_meta["fps"],
167
duration=vllm_meta["duration"],
168
)
146
- hf_indices = processor.sample_frames(hf_metadata)
169
+ hf_indices = processor.sample_frames(hf_metadata, **(hf_sample_kwargs or {}))
170
vllm_indices = np.array(vllm_meta["frames_indices"])
171
np.testing.assert_array_equal(
172
hf_indices,
vllm/multimodal/video.py
+80
@@ -7,6 +7,7 @@ from typing import Any, ClassVar, Literal, NamedTuple, cast
7
8
import numpy as np
9
import numpy.typing as npt
10
+import torch
11
12
from vllm.logger import init_logger
13
from vllm.utils.import_utils import PlaceholderModule
@@ -653,6 +654,85 @@ class Qwen3VLVideoBackend(VideoBackend):
654
)
655
656
657
+@VIDEO_LOADER_REGISTRY.register(
658
+ "qwen2_vl",
659
+ video_processor="Qwen2VLVideoProcessor",
660
+)
661
+class Qwen2VLVideoBackend(VideoBackend):
662
+ """Qwen2-VL / Qwen2.5-VL fps-based video backend.
663
+
664
+ Ports transformers' ``Qwen2VLVideoProcessor.sample_frames`` (fps mode),
665
+ shared by Qwen2-VL and Qwen2.5-VL (the latter has no video processor of its
666
+ own): sample ``total / original_fps * fps`` frames, clamp to
667
+ ``[min_frames, max_frames]`` (4 and 768), floor to a multiple of
668
+ ``temporal_patch_size`` (2), and take indices with the exact
669
+ ``torch.arange(0, total, total / n)`` call so they match HF byte-for-byte.
670
+
671
+ ``num_frames`` is ignored (fps-driven, like the Qwen3-VL loader). The
672
+ float32 step can emit an out-of-range tail index (e.g. 451 for a 451-frame
673
+ clip); it is clamped to the last valid frame.
674
+ """
675
+
676
+ @classmethod
677
+ def compute_frames_index_to_sample(
678
+ cls,
679
+ source: VideoSourceMetadata,
680
+ target: VideoTargetMetadata,
681
+ **kwargs,
682
+ ) -> list[int]:
683
+ # Refer to:
684
+ # https://github.com/huggingface/transformers/blob/v5.7.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L122-L190
685
+ total_frames_num = source.total_frames_num
686
+ original_fps = source.original_fps
687
+ temporal_patch_size = kwargs.get("temporal_patch_size", 2)
688
+ min_frames = kwargs.get("min_frames", 4)
689
+ max_frames = kwargs.get("max_frames", 768)
690
+
691
+ # vLLM reports original_fps == 0 for clips with unknown/variable fps
692
+ # (VFR, malformed, streaming); fail loudly instead of dividing by zero.
693
+ if original_fps <= 0:
694
+ raise ValueError(
695
+ "Qwen2-VL video sampling needs a known source fps, but the "
696
+ "container reported 0 (variable or unknown frame rate)."
697
+ )
698
+
699
+ max_frames = (
700
+ math.floor(min(max_frames, total_frames_num) / temporal_patch_size)
701
+ * temporal_patch_size
702
+ )
703
+ n = total_frames_num / original_fps * target.fps
704
+ n = min(max(n, min_frames), max_frames, total_frames_num)
705
+ n = math.floor(n / temporal_patch_size) * temporal_patch_size
706
+
707
+ # ``torch.arange`` matches transformers' float32 index math exactly
708
+ # (numpy's float64 diverges by a frame on some inputs); clamp the tail
709
+ # because that step can emit an index == total_frames_num.
710
+ indices = torch.arange(0, total_frames_num, total_frames_num / n).int()
711
+ return torch.clamp(indices, max=total_frames_num - 1).tolist()
712
+
713
+ @classmethod
714
+ def load_bytes(
715
+ cls,
716
+ data: bytes,
717
+ num_frames: int = -1,
718
+ fps: int = 2,
719
+ max_duration: int = 300,
720
+ frame_recovery: bool = False,
721
+ *,
722
+ backend: Literal["opencv", "pyav"] = "opencv",
723
+ **kwargs,
724
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
725
+ return super().load_bytes(
726
+ data,
727
+ num_frames=num_frames,
728
+ fps=fps,
729
+ max_duration=max_duration,
730
+ frame_recovery=frame_recovery,
731
+ backend=backend,
732
+ **kwargs,
733
+ )
734
+
735
+
736
@VIDEO_LOADER_REGISTRY.register(
737
"opencv_dynamic",
738
video_processor="Glm4vVideoProcessor",
vllm/transformers_utils/processor.py
+10
@@ -18,6 +18,7 @@ from transformers.audio_utils import AudioInput
18
from transformers.feature_extraction_utils import FeatureExtractionMixin
19
from transformers.image_processing_utils import BaseImageProcessor
20
from transformers.image_utils import ImageInput
21
+from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
22
from transformers.processing_utils import ProcessorMixin
23
from transformers.video_processing_utils import BaseVideoProcessor
24
from transformers.video_utils import VideoInput
@@ -169,6 +170,15 @@ def get_video_processor_cls_name_from_config(
170
config = get_hf_file_to_dict(file, processor_name, revision=revision)
171
if config and "video_processor_type" in config:
172
return config["video_processor_type"]
173
+
174
+ # Some models ship no explicit ``video_processor_type`` in their
175
+ # preprocessor config. Fall back to transformers' ``model_type`` -> video
176
+ # processor mapping so these still resolve to their registered loader
177
+ # instead of the generic opencv fallback. The mapping is ``None`` for a
178
+ # given type when torchvision is unavailable; callers then use opencv.
179
+ model_config = get_hf_file_to_dict("config.json", processor_name, revision=revision)
180
+ if model_config and "model_type" in model_config:
181
+ return VIDEO_PROCESSOR_MAPPING_NAMES.get(model_config["model_type"])
182
return None
183
184