A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 10:15 UTC
[CI/Build] Automatically patch video metadata for multimodal processor test (#35822)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 3, 2026 at 12:27 UTC
7d8bbe6f42d371b7e570529a9239e6ff0913dc76
1 file changed
+5
-36
tests/models/multimodal/processing/test_common.py
+5
-36
@@ -33,32 +33,9 @@ from ...registry import (
33
)
34
35
36
-def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
36
+def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
37
"""
38
- Patch the multimodal data for GLM4.1V model.
39
- """
40
- # Ensure video metadata is included
41
- if "video" in mm_data:
42
- # GLM4.1V doesn't support multiple videos
43
- video = mm_data["video"]
44
- num_frames = len(video)
45
- mm_data["video"] = (
46
- video,
47
- {
48
- "total_num_frames": num_frames,
49
- "fps": num_frames,
50
- "duration": 1,
51
- "frames_indices": [i for i in range(num_frames)],
52
- "video_backend": "opencv",
53
- "do_sample_frames": True,
54
- },
55
- )
56
- return mm_data
57
-
58
-
59
-def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
60
- """
61
- Patch the multimodal data for Qwen3-VL model.
38
+ Add metadata to video mm_data
39
"""
40
41
def create_metadata(frames: np.ndarray):
@@ -119,18 +96,7 @@ _IGNORE_MM_KEYS = {
96
}
97
98
MM_DATA_PATCHES = {
122
- # Ernie4.5-VL, GLM4.1V and Qwen3-VL requires video metadata
123
- "ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
124
- "glm4v": glm4_1v_patch_mm_data,
125
- "glm4v_moe": glm4_1v_patch_mm_data,
126
- "glm_ocr": glm4_1v_patch_mm_data,
99
"glmasr": glmasr_patch_mm_data,
128
- "interns1_pro": qwen3_vl_patch_mm_data,
129
- "molmo2": qwen3_vl_patch_mm_data,
130
- "qwen3_5": qwen3_vl_patch_mm_data,
131
- "qwen3_5_moe": qwen3_vl_patch_mm_data,
132
- "qwen3_vl": qwen3_vl_patch_mm_data,
133
- "qwen3_vl_moe": qwen3_vl_patch_mm_data,
100
}
101
102
@@ -176,6 +142,9 @@ def get_text_token_prompts(
142
tokenizer: TokenizerLike = processor.info.get_tokenizer()
143
model_config = processor.info.ctx.model_config
144
145
+ if processor.info.data_parser.video_needs_metadata:
146
+ mm_data = add_video_metadata(mm_data)
147
+
148
model_type = model_config.hf_config.model_type
149
if model_type in MM_DATA_PATCHES:
150
mm_data = MM_DATA_PATCHES[model_type](mm_data)