A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[VLM] Add Qwen3-VL generation test (#25185)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Isotr0py committed
Oct 29, 2025 at 20:19 UTC
ad3ec89532b7e0ef676fd319b061adde360f5c68
7 files changed
+108
-5
tests/models/multimodal/generation/test_common.py
+22
@@ -159,6 +159,28 @@ VLM_TEST_SETTINGS = {
159
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
160
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
161
),
162
+ "qwen3_vl": VLMTestInfo(
163
+ models=["Qwen/Qwen3-VL-4B-Instruct"],
164
+ test_type=(
165
+ VLMTestType.IMAGE,
166
+ VLMTestType.MULTI_IMAGE,
167
+ VLMTestType.VIDEO,
168
+ ),
169
+ needs_video_metadata=True,
170
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
171
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
172
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
173
+ max_model_len=4096,
174
+ max_num_seqs=2,
175
+ num_logprobs=20,
176
+ auto_cls=AutoModelForImageTextToText,
177
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
178
+ patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
179
+ image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
180
+ marks=[
181
+ pytest.mark.core_model,
182
+ ],
183
+ ),
184
"ultravox": VLMTestInfo(
185
models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
186
test_type=VLMTestType.AUDIO,
tests/models/multimodal/generation/vlm_utils/builders.py
+30
-3
@@ -4,7 +4,9 @@
4
5
from collections.abc import Callable, Iterable
6
from pathlib import PosixPath
7
+from typing import Any
8
9
+import numpy.typing as npt
10
import torch
11
12
from vllm.multimodal.audio import AudioResampler
@@ -236,6 +238,7 @@ def build_video_inputs_from_test_info(
238
video_assets: VideoTestAssets,
239
size_wrapper: ImageSizeWrapper,
240
num_frames: int,
241
+ needs_video_metadata: bool,
242
) -> list[PromptWithMultiModalInput]:
243
if test_info.prompt_formatter is None:
244
raise ValueError("Prompt formatter must be set to build video inputs")
@@ -248,7 +251,10 @@ def build_video_inputs_from_test_info(
251
)
252
253
sampled_vids = [
251
- sample_frames_from_video(asset.np_ndarrays, num_frames)
254
+ sample_frames_with_video_metadata(
255
+ (asset.np_ndarrays, asset.metadata),
256
+ num_frames,
257
+ )
258
for asset in video_assets
259
]
260
@@ -259,12 +265,33 @@ def build_video_inputs_from_test_info(
265
return [
266
PromptWithMultiModalInput(
267
prompts=[prompt for _ in size_wrapper.data],
262
- video_data=[video_scaler(video, size) for size in size_wrapper.data],
268
+ video_data=[
269
+ (
270
+ video_scaler(video, size)
271
+ if not needs_video_metadata
272
+ else (video_scaler(video, size), meta)
273
+ )
274
+ for size in size_wrapper.data
275
+ ],
276
)
264
- for video, prompt in zip(sampled_vids, model_prompts)
277
+ for (video, meta), prompt in zip(sampled_vids, model_prompts)
278
]
279
280
281
+def sample_frames_with_video_metadata(
282
+ video_with_meta: tuple[npt.NDArray, dict[str, Any]],
283
+ num_frames: int,
284
+) -> tuple[npt.NDArray, dict[str, Any]]:
285
+ video, meta = video_with_meta
286
+ video = sample_frames_from_video(video, num_frames)
287
+
288
+ meta["do_sample_frames"] = meta["total_num_frames"] == num_frames
289
+ meta["total_num_frames"] = num_frames
290
+ meta["fps"] = meta["duration"] / num_frames
291
+ meta["frames_indices"] = list(range(num_frames))
292
+ return video, meta
293
+
294
+
295
def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType):
296
"""Applies a size scaler to one image; this can be an image size factor,
297
which scales the image while maintaining the aspect ratio"""
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+3
@@ -100,6 +100,9 @@ def get_parametrized_options(
100
# num_frames is video only
101
if test_type == VLMTestType.VIDEO:
102
iter_kwargs["num_video_frames"] = ensure_wrapped(test_info.num_video_frames)
103
+ iter_kwargs["needs_video_metadata"] = ensure_wrapped(
104
+ test_info.needs_video_metadata
105
+ )
106
107
# No sizes passed for custom inputs, since inputs are directly provided
108
if test_type not in (VLMTestType.CUSTOM_INPUTS, VLMTestType.AUDIO):
tests/models/multimodal/generation/vlm_utils/model_utils.py
+48
@@ -905,6 +905,54 @@ def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
905
return hf_model
906
907
908
+def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
909
+ """Patches and returns an instance of the HfRunner to use for GLM4.1V."""
910
+ hf_processor = hf_model.processor
911
+
912
+ def processor(*args, videos=None, **kwargs):
913
+ if videos is not None and is_list_of(videos, tuple):
914
+ # batched multi videos
915
+ do_sample_frames = {video[1]["do_sample_frames"] for video in videos}
916
+ assert len(do_sample_frames) == 1
917
+ if kwargs.get("do_sample_frames") is None:
918
+ kwargs["do_sample_frames"] = do_sample_frames
919
+ video_metadata = [
920
+ [
921
+ VideoMetadata(
922
+ **{k: v for k, v in video[1].items() if k != "do_sample_frames"}
923
+ )
924
+ ]
925
+ for video in videos
926
+ ]
927
+ videos = [[video[0]] for video in videos]
928
+ elif videos is not None and isinstance(videos, tuple):
929
+ # single video
930
+ do_sample_frames = videos[1]["do_sample_frames"]
931
+ if kwargs.get("do_sample_frames") is None:
932
+ kwargs["do_sample_frames"] = do_sample_frames
933
+ video_metadata = [
934
+ [
935
+ VideoMetadata(
936
+ **{
937
+ k: v
938
+ for k, v in videos[1].items()
939
+ if k != "do_sample_frames"
940
+ }
941
+ )
942
+ ]
943
+ ]
944
+ videos = [[videos[0]]]
945
+ else:
946
+ video_metadata = None
947
+
948
+ return hf_processor(
949
+ *args, videos=videos, video_metadata=video_metadata, **kwargs
950
+ )
951
+
952
+ hf_model.processor = processor
953
+ return hf_model
954
+
955
+
956
def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
957
from vllm.model_executor.models.tarsier import get_vision_encoder_info
958
tests/models/multimodal/generation/vlm_utils/runners.py
+1
@@ -117,6 +117,7 @@ def run_video_test(
117
video_assets,
118
test_case.size_wrapper,
119
test_case.num_video_frames,
120
+ test_case.needs_video_metadata,
121
)
122
123
core.run_test(
tests/models/multimodal/generation/vlm_utils/types.py
+3
-1
@@ -154,7 +154,8 @@ class VLMTestInfo(NamedTuple):
154
dtype: str = "auto"
155
distributed_executor_backend: str | None = None
156
# Only expanded in video tests
157
- num_video_frames: int = 16
157
+ num_video_frames: int | tuple[int] = 16
158
+ needs_video_metadata: bool = False
159
160
# Fixed image sizes / image size factors; most tests use image_size_factors
161
# The values provided for these two fields will be stacked and expanded
@@ -212,5 +213,6 @@ class ExpandableVLMTestArgs(NamedTuple):
213
size_wrapper: ImageSizeWrapper | None = None
214
# Video only
215
num_video_frames: int | None = None
216
+ needs_video_metadata: bool = False
217
# Custom inputs only
218
custom_test_opts: CustomTestOptions | None = None
vllm/assets/video.py
+1
-1
@@ -94,7 +94,7 @@ def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]:
94
95
metadata = {
96
"total_num_frames": num_frames,
97
- "fps": fps,
97
+ "fps": duration / num_frames,
98
"duration": duration,
99
"video_backend": "opencv",
100
"frames_indices": list(range(num_frames)),