@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66a168a197
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[CI/Build] Refactor processing tests (#27470)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Cyrus Leung committed Oct 26, 2025 at 00:14 UTC 66a168a197ba214a5b70a74fa2e713c9eeb3251a
4 files changed +174 -230
tests/models/multimodal/processing/test_common.py
+128 -149
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from collections.abc import Set as AbstractSet
5 from functools import partial
6
7 import numpy as np
@@ -22,14 +23,17 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache
23 from vllm.multimodal.inputs import MultiModalInputs
24 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
25 from vllm.transformers_utils.tokenizer import (
25 - AnyTokenizer,
26 MistralTokenizer,
27 cached_tokenizer_from_config,
28 encode_tokens,
29 )
30
31 from ....multimodal.utils import random_audio, random_image, random_video
32 -from ...registry import HF_EXAMPLE_MODELS
32 +from ...registry import (
33 + _MULTIMODAL_EXAMPLE_MODELS,
34 + _TRANSFORMERS_BACKEND_MODELS,
35 + HF_EXAMPLE_MODELS,
36 +)
37
38
39 def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
@@ -83,6 +87,119 @@ def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
87 return mm_data
88
89
90 +# For some multimodal models, tokenizer will always add bos_token
91 +# at the beginning of prompt by default, causing hf_processor outputs
92 +# incorrect token ids. So we need use `add_special_tokens=False` here
93 +# to leave bos_token to be added by the processor.
94 +_ADD_SPECIAL_TOKENS_OVERRIDES = {
95 + "ovis": False,
96 + "ovis2_5": False,
97 + "paligemma": False,
98 + "ultravox": False,
99 + "whisper": False,
100 +}
101 +
102 +_IGNORE_MM_KEYS = {
103 + # In Ultravox, the audio_features can be different depending on padding
104 + # The slight difference should not be a problem though, since
105 + # attention_mask lets us ignore the difference.
106 + "ultravox": {"audio_features"},
107 +}
108 +
109 +MM_DATA_PATCHES = {
110 + # GLM4.1V and Qwen3-VL requires video metadata to be included in the input
111 + "glm4v": glm4_1v_patch_mm_data,
112 + "glm4v_moe": glm4_1v_patch_mm_data,
113 + "qwen3_vl": qwen3_vl_patch_mm_data,
114 + "qwen3_vl_moe": qwen3_vl_patch_mm_data,
115 +}
116 +
117 +
118 +def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
119 + for model_arch in model_arch_list:
120 + model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
121 + yield model_info.default
122 +
123 + for extra_type, extra_model_id in model_info.extras.items():
124 + if "fp" in extra_type:
125 + continue # Redundant to test quantized models
126 +
127 + yield extra_model_id
128 +
129 +
130 +def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
131 + return list(_iter_model_ids_to_test(model_arch_list))
132 +
133 +
134 +def get_model_ids_to_test():
135 + transformers_arch_ids = {
136 + model_id
137 + for info in _TRANSFORMERS_BACKEND_MODELS.values()
138 + for model_id in (info.default, *info.extras.values())
139 + }
140 + vllm_only_archs = {
141 + arch
142 + for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
143 + if not any(
144 + model_id in transformers_arch_ids
145 + for model_id in (info.default, *info.extras.values())
146 + )
147 + }
148 +
149 + return _get_model_ids_to_test(vllm_only_archs)
150 +
151 +
152 +def get_text_token_prompts(
153 + processor: BaseMultiModalProcessor,
154 + mm_data: MultiModalDataDict,
155 +):
156 + dummy_inputs = processor.dummy_inputs
157 + tokenizer = processor.info.get_tokenizer()
158 + model_config = processor.info.ctx.model_config
159 +
160 + model_type = model_config.hf_config.model_type
161 + if model_type in MM_DATA_PATCHES:
162 + mm_data = MM_DATA_PATCHES[model_type](mm_data)
163 +
164 + parsed_data = processor.data_parser.parse_mm_data(mm_data)
165 + mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
166 +
167 + text_prompt: str | None
168 + token_prompt: list[int]
169 + if isinstance(tokenizer, MistralTokenizer):
170 + images = parsed_data.get("image", [])
171 + request = ChatCompletionRequest(
172 + messages=[
173 + UserMessage(
174 + content=[
175 + TextChunk(text=""),
176 + *(ImageChunk(image=image) for image in images),
177 + ]
178 + ),
179 + ]
180 + )
181 + res = tokenizer.mistral.encode_chat_completion(request)
182 +
183 + # Mistral does not support decode_tokens with skip_special_tokens=False
184 + text_prompt = None
185 + token_prompt = res.tokens
186 + else:
187 + inputs = dummy_inputs.get_dummy_processor_inputs(
188 + model_config.max_model_len,
189 + mm_counts,
190 + )
191 + assert isinstance(inputs.prompt, str)
192 +
193 + text_prompt = inputs.prompt
194 + token_prompt = encode_tokens(
195 + tokenizer,
196 + text_prompt,
197 + add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type),
198 + )
199 +
200 + return text_prompt, token_prompt
201 +
202 +
203 def _test_processing_correctness(
204 model_id_or_arch: str,
205 hit_rate: float,
@@ -148,8 +265,6 @@ def _test_processing_correctness(
265
266 baseline_processor = factories.build_processor(ctx, cache=None)
267 cached_processor = factories.build_processor(ctx, cache=cache)
151 - dummy_inputs = baseline_processor.dummy_inputs
152 - tokenizer = baseline_processor.info.get_tokenizer()
268
269 rng = np.random.RandomState(0)
270
@@ -175,29 +290,6 @@ def _test_processing_correctness(
290 for k, limit in limit_mm_per_prompt_ints.items()
291 }
292
178 - mm_counts = {k: len(vs) for k, vs in mm_data.items()}
179 -
180 - # Mistral chat outputs tokens directly, rather than text prompts
181 - if isinstance(tokenizer, MistralTokenizer):
182 - images = mm_data.get("image", [])
183 - request = ChatCompletionRequest(
184 - messages=[
185 - UserMessage(
186 - content=[
187 - TextChunk(text=""),
188 - *(ImageChunk(image=image) for image in images),
189 - ]
190 - ),
191 - ]
192 - )
193 - res = tokenizer.mistral.encode_chat_completion(request)
194 - prompt = res.tokens
195 - else:
196 - prompt = dummy_inputs.get_dummy_processor_inputs(
197 - model_config.max_model_len,
198 - mm_counts,
199 - ).prompt
200 -
293 # Drop unnecessary keys and test single -> multi conversion
294 if rng.rand() < simplify_rate:
295 for k in list(mm_data.keys()):
@@ -208,8 +300,6 @@ def _test_processing_correctness(
300
301 _test_processing_correctness_one(
302 model_config,
211 - tokenizer,
212 - prompt,
303 mm_data,
304 baseline_processor,
305 cached_processor,
@@ -217,59 +307,17 @@ def _test_processing_correctness(
307 )
308
309
220 -# For some multimodal models, tokenizer will always add bos_token
221 -# at the beginning of prompt by default, causing hf_processor outputs
222 -# incorrect token ids. So we need use `add_special_tokens=False` here
223 -# to leave bos_token to be added by the processor.
224 -_ADD_SPECIAL_TOKENS_OVERRIDES = {
225 - "ovis": False,
226 - "ovis2_5": False,
227 - "paligemma": False,
228 - "ultravox": False,
229 - "whisper": False,
230 -}
231 -
232 -_IGNORE_MM_KEYS = {
233 - # In Ultravox, the audio_features can be different depending on padding
234 - # The slight difference should not be a problem though, since
235 - # attention_mask lets us ignore the difference.
236 - "ultravox": {"audio_features"},
237 -}
238 -
239 -MM_DATA_PATCHES = {
240 - # GLM4.1V and Qwen3-VL requires video metadata to be included in the input
241 - "glm4v": glm4_1v_patch_mm_data,
242 - "glm4v_moe": glm4_1v_patch_mm_data,
243 - "qwen3_vl": qwen3_vl_patch_mm_data,
244 - "qwen3_vl_moe": qwen3_vl_patch_mm_data,
245 -}
246 -
247 -
310 def _test_processing_correctness_one(
311 model_config: ModelConfig,
250 - tokenizer: AnyTokenizer,
251 - prompt: str | list[int],
312 mm_data: MultiModalDataDict,
313 baseline_processor: BaseMultiModalProcessor,
314 cached_processor: BaseMultiModalProcessor,
315 batch_idx: int,
316 ):
317 model_type = model_config.hf_config.model_type
258 - ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
259 - if model_type in MM_DATA_PATCHES:
260 - mm_data = MM_DATA_PATCHES[model_type](mm_data)
318
262 - if isinstance(prompt, str):
263 - text_prompt = prompt
264 - token_prompt = encode_tokens(
265 - tokenizer,
266 - prompt,
267 - add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type),
268 - )
269 - else:
270 - # Mistral does not support decode_tokens with skip_special_tokens=False
271 - text_prompt = None
272 - token_prompt = prompt
319 + text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
320 + ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
321
322 baseline_tokenized_result = baseline_processor.apply(
323 token_prompt,
@@ -324,81 +372,7 @@ def _test_processing_correctness_one(
372 )
373
374
327 -@pytest.mark.parametrize(
328 - "model_id",
329 - [
330 - "rhymes-ai/Aria",
331 - "CohereForAI/aya-vision-8b",
332 - "Open-Bee/Bee-8B-RL",
333 - "Salesforce/blip2-opt-2.7b",
334 - "facebook/chameleon-7b",
335 - "CohereLabs/command-a-vision-07-2025",
336 - "deepseek-ai/deepseek-vl2-tiny",
337 - "deepseek-ai/DeepSeek-OCR",
338 - "baidu/ERNIE-4.5-VL-28B-A3B-PT",
339 - "adept/fuyu-8b",
340 - "google/gemma-3-4b-it",
341 - "google/gemma-3n-E2B-it",
342 - "zai-org/glm-4v-9b",
343 - "zai-org/GLM-4.1V-9B-Thinking",
344 - "zai-org/GLM-4.5V",
345 - "ibm-granite/granite-speech-3.3-2b",
346 - "h2oai/h2ovl-mississippi-800m",
347 - "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
348 - "HuggingFaceM4/Idefics3-8B-Llama3",
349 - "internlm/Intern-S1",
350 - "OpenGVLab/InternVL2-1B",
351 - "OpenGVLab/InternVL3-1B",
352 - "OpenGVLab/InternVL3_5-1B",
353 - "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
354 - "OpenGVLab/InternVL3_5-30B-A3B",
355 - "Kwai-Keye/Keye-VL-8B-Preview",
356 - "Kwai-Keye/Keye-VL-1_5-8B",
357 - "moonshotai/Kimi-VL-A3B-Instruct",
358 - "meta-llama/Llama-4-Scout-17B-16E-Instruct",
359 - "llava-hf/llava-1.5-7b-hf",
360 - "llava-hf/llava-v1.6-mistral-7b-hf",
361 - "llava-hf/LLaVA-NeXT-Video-7B-hf",
362 - "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
363 - "TIGER-Lab/Mantis-8B-siglip-llama3",
364 - "mispeech/midashenglm-7b",
365 - "openbmb/MiniCPM-Llama3-V-2_5",
366 - "openbmb/MiniCPM-o-2_6",
367 - "openbmb/MiniCPM-V-2_6",
368 - "MiniMaxAI/MiniMax-VL-01",
369 - "allenai/Molmo-7B-D-0924",
370 - "allenai/Molmo-7B-O-0924",
371 - "nvidia/NVLM-D-72B",
372 - "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
373 - "AIDC-AI/Ovis1.6-Gemma2-9B",
374 - "AIDC-AI/Ovis1.6-Llama3.2-3B",
375 - "AIDC-AI/Ovis2-1B",
376 - "AIDC-AI/Ovis2.5-2B",
377 - "google/paligemma-3b-mix-224",
378 - "google/paligemma2-3b-ft-docci-448",
379 - "microsoft/Phi-3.5-vision-instruct",
380 - "microsoft/Phi-4-multimodal-instruct",
381 - "mistralai/Pixtral-12B-2409",
382 - "mistral-community/pixtral-12b",
383 - "Qwen/Qwen-VL-Chat",
384 - "Qwen/Qwen2-VL-2B-Instruct",
385 - "Qwen/Qwen2.5-VL-3B-Instruct",
386 - "Qwen/Qwen2-Audio-7B-Instruct",
387 - "Qwen/Qwen2.5-Omni-3B",
388 - "Qwen/Qwen3-VL-4B-Instruct",
389 - "Qwen/Qwen3-VL-30B-A3B-Instruct",
390 - "Qwen/Qwen3-Omni-30B-A3B-Instruct",
391 - "YannQi/R-4B",
392 - "Skywork/Skywork-R1V-38B",
393 - "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
394 - "stepfun-ai/step3",
395 - "fixie-ai/ultravox-v0_5-llama-3_2-1b",
396 - "openai/whisper-large-v3",
397 - "omni-research/Tarsier-7b",
398 - "omni-research/Tarsier2-Recap-7b",
399 - "mistralai/Voxtral-Mini-3B-2507",
400 - ],
401 -)
375 +@pytest.mark.parametrize("model_id", get_model_ids_to_test())
376 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
377 @pytest.mark.parametrize("num_batches", [32])
378 @pytest.mark.parametrize("simplify_rate", [1.0])
@@ -409,7 +383,12 @@ def test_processing_correctness(
383 simplify_rate: float,
384 ):
385 if model_id == "google/gemma-3n-E2B-it":
412 - pytest.skip("Skipping gemma-3n-E2B-it due to transformers #39911 bug.")
386 + pytest.skip("Fix later")
387 + if model_id == "OpenGVLab/InternVL2-2B":
388 + pytest.skip("Fix later")
389 + if model_id == "jinaai/jina-reranker-m0":
390 + pytest.skip("Fix later")
391 +
392 _test_processing_correctness(
393 model_id,
394 hit_rate=hit_rate,
tests/models/multimodal/processing/test_tensor_schema.py
+31 -66
@@ -9,9 +9,6 @@ from typing import Any, TypeAlias
9 import numpy as np
10 import pytest
11 import torch.nn as nn
12 -from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
13 -from mistral_common.protocol.instruct.messages import UserMessage
14 -from mistral_common.protocol.instruct.request import ChatCompletionRequest
12 from PIL import Image
13
14 from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
@@ -37,22 +34,9 @@ from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
34 from vllm.utils.collection_utils import is_list_of
35 from vllm.utils.torch_utils import set_default_torch_dtype
36
40 -from ...registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
37 +from ...registry import HF_EXAMPLE_MODELS
38 from ...utils import dummy_hf_overrides
42 -
43 -ARCH_TO_SKIP = {
44 - "MolmoForCausalLM": "incompatible requirements",
45 -}
46 -ARCH_NEEDS_EXTRAS = [
47 - "InternVLChatModel",
48 - "Idefics3ForConditionalGeneration",
49 - "LlavaForConditionalGeneration",
50 - "MiniCPMV",
51 - "PaliGemmaForConditionalGeneration",
52 -]
53 -REPO_ID_TO_SKIP = {
54 - "nm-testing/pixtral-12b-FP8-dynamic": "duplicated test",
55 -}
39 +from .test_common import get_model_ids_to_test, get_text_token_prompts
40
41 ImageInput = list[Image.Image]
42 VideoInput: TypeAlias = (
@@ -61,6 +45,18 @@ VideoInput: TypeAlias = (
45 AudioInput = list[tuple[np.ndarray, int]]
46
47
48 +MM_OPTIONS_OVERRIDES = {
49 + # Qwen3-VL's default profiling video size (64x64) can cause trouble
50 + # after resizing, so we override it here for testing.
51 + "qwen3_vl": dict(
52 + video=VideoDummyOptions(num_frames=128, width=256, height=256),
53 + ),
54 + "qwen3_vl_moe": dict(
55 + video=VideoDummyOptions(num_frames=128, width=256, height=256),
56 + ),
57 +}
58 +
59 +
60 def _resize_data(
61 _data: Image.Image | np.ndarray, size_factor: float
62 ) -> Image.Image | np.ndarray:
@@ -94,7 +90,7 @@ def resize_mm_data(
90 if is_list_of(data, (Image.Image, np.ndarray, list)):
91 return [_resize_data(d, s) for d, s in zip(data, size_factors)]
92 elif is_list_of(data, tuple):
97 - return [(_resize_data(d, s), meta) for (d, meta), s in zip(data, size_factors)]
93 + return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
94 raise ValueError("Unsupported multimodal data type.")
95
96
@@ -104,6 +100,8 @@ def create_batched_mm_kwargs(
100 processor: BaseMultiModalProcessor,
101 size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
102 ) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
103 + model_type = model_config.hf_config.model_type
104 +
105 processing_info = processor.info
106 dummy_inputs = processor.dummy_inputs
107 supported_mm_limits = processing_info.get_supported_mm_limits()
@@ -114,32 +112,19 @@ def create_batched_mm_kwargs(
112 processor_inputs = dummy_inputs.get_dummy_processor_inputs(
113 seq_len=model_config.max_model_len,
114 mm_counts=mm_counts,
115 + mm_options=MM_OPTIONS_OVERRIDES.get(model_type),
116 )
117 mm_data = processor_inputs.mm_data
118 resized_mm_data = {
119 modality: resize_mm_data(data, size_factors)
120 for modality, data in mm_data.items()
121 }
123 - # Mistral chat outputs tokens directly, rather than text prompts
124 - if model_config.tokenizer_mode == "mistral":
125 - images = resized_mm_data.get("image", [])
126 - request = ChatCompletionRequest(
127 - messages=[
128 - UserMessage(
129 - content=[
130 - TextChunk(text=""),
131 - *(ImageChunk(image=image) for image in images),
132 - ]
133 - ),
134 - ]
135 - )
136 - tokenizer = processing_info.get_tokenizer()
137 - res = tokenizer.mistral.encode_chat_completion(request)
138 - prompt = res.tokens
139 - else:
140 - prompt = processor_inputs.prompt
122 +
123 + # video metadata will be added back to the resized video data here.
124 + text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
125 +
126 mm_kwargs = processor.apply(
142 - prompt=prompt,
127 + prompt=token_prompt if text_prompt is None else text_prompt,
128 mm_data=resized_mm_data,
129 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
130 tokenization_kwargs=processor_inputs.tokenization_kwargs,
@@ -175,35 +160,15 @@ def initialize_dummy_model(
160 cleanup_dist_env_and_memory()
161
162
178 -def get_model_id_to_test(model_arch_list: Iterable[str]) -> list[tuple[str, str]]:
179 - filtered_results = []
180 - for model_arch in model_arch_list:
181 - model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
182 - if model_info.extras and model_arch in ARCH_NEEDS_EXTRAS:
183 - available_repos = list(
184 - map(
185 - lambda model_id: (model_arch, model_id),
186 - [model_info.default, *model_info.extras.values()],
187 - )
188 - )
189 - filtered_results.extend(available_repos)
190 - else:
191 - filtered_results.append((model_arch, model_info.default))
192 - return filtered_results
193 -
194 -
195 -@pytest.mark.parametrize(
196 - "model_arch, model_id", get_model_id_to_test(_MULTIMODAL_EXAMPLE_MODELS.keys())
197 -)
198 -def test_model_tensor_schema(model_arch: str, model_id: str):
199 - if model_arch in ARCH_TO_SKIP:
200 - pytest.skip(f"Skipping {model_arch} due to {ARCH_TO_SKIP[model_arch]}")
201 - if model_id in REPO_ID_TO_SKIP:
202 - pytest.skip(f"Skipping {model_id} due to {REPO_ID_TO_SKIP[model_id]}")
203 -
204 - model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
163 +@pytest.mark.parametrize("model_id", get_model_ids_to_test())
164 +def test_model_tensor_schema(model_id: str):
165 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
166 model_info.check_available_online(on_fail="skip")
206 - model_info.check_transformers_version(on_fail="skip", check_max_version=False)
167 + model_info.check_transformers_version(on_fail="skip")
168 +
169 + model_arch = next(
170 + arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
171 + )
172
173 hf_overrides_fn = partial(
174 dummy_hf_overrides,
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+12 -8
@@ -733,17 +733,21 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
733 else (pad_to_hop_length(audio[0], hop_length), audio[1])
734 for audio in audios
735 ]
736 - mm_kwargs = dict(
737 - **mm_kwargs,
738 - )
736 +
737 # TODO(Isotr0py): Remove this patch after upstream fix PR
738 # released and Transformers version update:
739 # https://github.com/huggingface/transformers/pull/41473
742 - if (
743 - Version(TRANSFORMERS_VERSION) < Version("4.58.0")
744 - and "truncation" not in mm_kwargs
745 - ):
746 - mm_kwargs["truncation"] = False
740 + mm_kwargs = dict(mm_kwargs)
741 + tok_kwargs = dict(tok_kwargs)
742 + if Version(TRANSFORMERS_VERSION) < Version("4.58.0"):
743 + # move truncation to audio_kwargs level to avoid conflict
744 + # with tok_kwargs
745 + mm_kwargs["audio_kwargs"] = {
746 + "truncation": mm_kwargs.pop("truncation", False)
747 + }
748 + mm_kwargs["text_kwargs"] = {
749 + "truncation": tok_kwargs.pop("truncation", False)
750 + }
751
752 hf_inputs = super()._call_hf_processor(
753 prompt=prompt,
vllm/model_executor/models/qwen3_vl.py
+3 -7
@@ -898,16 +898,12 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
898 processor = self.info.get_hf_processor(**mm_kwargs)
899
900 # Separate video processing from image processing. Because the videos
901 - # are processed into serval image patches
902 - if (
903 - "videos" in mm_data
904 - and isinstance(mm_data["videos"], list)
905 - and len(mm_data["videos"]) > 0
906 - ):
901 + # are processed into several image patches
902 + if videos := mm_data.pop("videos", []):
903 video_grid_thw_lst = []
904 pixel_values_videos_lst = []
905
910 - for item_idx, item in enumerate(mm_data.pop("videos", [])):
906 + for item in videos:
907 video_array, metadata = item
908
909 # NOTE: @JJJYmmm new attr metadata.frames_indices indicates