A high-throughput and memory-efficient inference and serving engine for LLMs
[Multimodal] Add Qwen3-VL video loader (#44412)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jun 16, 2026 at 11:45 UTC
e3cfea2e1ba048744025cdc664e96b954370cb51
3 files changed
+100
-3
tests/multimodal/test_video.py
+50
-2
@@ -1,11 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
+import itertools
4
from pathlib import Path
5
6
import numpy as np
7
import numpy.typing as npt
8
import pytest
9
+from transformers import AutoVideoProcessor
10
+from transformers.video_utils import VideoMetadata
11
12
from vllm.assets.base import get_vllm_public_assets
13
from vllm.multimodal.video import (
@@ -13,6 +15,7 @@ from vllm.multimodal.video import (
15
DynamicVideoBackend,
16
GLM46VVideoBackend,
17
Molmo2VideoBackend,
18
+ Qwen3VLVideoBackend,
19
VideoLoader,
20
VideoSourceMetadata,
21
VideoTargetMetadata,
@@ -72,6 +75,9 @@ def test_video_loader_type_doesnt_exist():
75
pytest.param(
76
"allenai/Molmo2-4B",
77
Molmo2VideoBackend,
78
+ marks=pytest.mark.skip(
79
+ reason="Video processor not aligned, investigate later.",
80
+ ),
81
id="molmo2",
82
),
83
pytest.param(
@@ -84,6 +90,11 @@ def test_video_loader_type_doesnt_exist():
90
GLM46VVideoBackend,
91
id="glm46v",
92
),
93
+ pytest.param(
94
+ "Qwen/Qwen3-VL-4B-Instruct",
95
+ Qwen3VLVideoBackend,
96
+ id="qwen3vl",
97
+ ),
98
],
99
)
100
def test_video_processor_from_model_repo(
@@ -94,7 +105,9 @@ def test_video_processor_from_model_repo(
105
106
The test downloads the preprocessor config from HuggingFace Hub,
107
extracts the ``video_processor_type`` field, and verifies it maps
97
- to the expected backend and loader class.
108
+ to the expected backend and loader class. When a corresponding HF
109
+ ``VideoProcessor.sample_frames`` implementation exists, the test
110
+ also verifies that the vLLM backend produces identical frame indices.
111
"""
112
video_processor = get_video_processor_cls_name_from_config(model_repo)
113
assert video_processor is not None, (
@@ -109,6 +122,41 @@ def test_video_processor_from_model_repo(
122
f"{type(loader)}, expected {expected_loader_cls}"
123
)
124
125
+ # --- Alignment check with HF VideoProcessor.sample_frames ---
126
+ processor = AutoVideoProcessor.from_pretrained(model_repo, trust_remote_code=True)
127
+
128
+ fps_list = [1, 2, 30, 60]
129
+ duration_list = [10, 60, 600]
130
+ for fps, duration_secs in itertools.product(fps_list, duration_list):
131
+ num_frames = fps * duration_secs
132
+ video_bytes = create_long_gop_video(
133
+ num_frames=num_frames,
134
+ fps=fps,
135
+ width=8,
136
+ height=8,
137
+ )
138
+
139
+ _, vllm_meta = loader.load_bytes(video_bytes) # type: ignore[attr-defined]
140
+
141
+ hf_metadata = VideoMetadata(
142
+ total_num_frames=vllm_meta["total_num_frames"],
143
+ fps=vllm_meta["fps"],
144
+ duration=vllm_meta["duration"],
145
+ )
146
+ hf_indices = processor.sample_frames(hf_metadata)
147
+ vllm_indices = np.array(vllm_meta["frames_indices"])
148
+ np.testing.assert_array_equal(
149
+ hf_indices,
150
+ vllm_indices,
151
+ err_msg=(
152
+ f"{model_repo!r} fps={fps} duration={duration_secs}s: "
153
+ f"HF has {len(hf_indices)} indices "
154
+ f"{hf_indices[:5].tolist()}..{hf_indices[-5:].tolist()}, "
155
+ f"vLLM has {len(vllm_indices)} indices "
156
+ f"{vllm_indices[:5].tolist()}..{vllm_indices[-5:].tolist()}"
157
+ ),
158
+ )
159
+
160
161
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
162
"""
tests/multimodal/utils.py
+1
-1
@@ -94,7 +94,7 @@ def create_long_gop_video(
94
}
95
for i in range(num_frames):
96
img = np.zeros((height, width, 3), dtype=np.uint8)
97
- img[:, :, 1] = i
97
+ img[:, :, 1] = i % 256
98
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
99
for packet in stream.encode(frame):
100
container.mux(packet)
vllm/multimodal/video.py
+49
@@ -604,6 +604,55 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
604
)
605
606
607
+@VIDEO_LOADER_REGISTRY.register(
608
+ "qwen3_vl",
609
+ video_processor="Qwen3VLVideoProcessor",
610
+)
611
+class Qwen3VLVideoBackend(VideoBackend):
612
+ @classmethod
613
+ def compute_frames_index_to_sample(
614
+ cls,
615
+ source: VideoSourceMetadata,
616
+ target: VideoTargetMetadata,
617
+ **kwargs,
618
+ ) -> list[int]:
619
+ total_frames_num = source.total_frames_num
620
+ original_fps = source.original_fps
621
+ fps = target.fps
622
+ max_frame_idx = source.total_frames_num - 1
623
+ min_frames = kwargs.get("min_frames", 4)
624
+ max_frames = kwargs.get("max_frames", 768)
625
+
626
+ # Refer to:
627
+ # https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/qwen3_vl/video_processing_qwen3_vl.py#L119-L125
628
+ num_frames = int(total_frames_num / original_fps * fps)
629
+ num_frames = min(max(num_frames, min_frames), max_frames, total_frames_num)
630
+ indices = np.linspace(0, max_frame_idx, num_frames).round().astype(int).tolist()
631
+ return indices
632
+
633
+ @classmethod
634
+ def load_bytes(
635
+ cls,
636
+ data: bytes,
637
+ num_frames: int = -1,
638
+ fps: int = 2,
639
+ max_duration: int = 300,
640
+ frame_recovery: bool = False,
641
+ *,
642
+ backend: Literal["opencv", "pyav"] = "opencv",
643
+ **kwargs,
644
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
645
+ return super().load_bytes(
646
+ data,
647
+ num_frames=num_frames,
648
+ fps=fps,
649
+ max_duration=max_duration,
650
+ frame_recovery=frame_recovery,
651
+ backend=backend,
652
+ **kwargs,
653
+ )
654
+
655
+
656
@VIDEO_LOADER_REGISTRY.register(
657
"opencv_dynamic",
658
video_processor="Glm4vVideoProcessor",