@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88c3e114d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 05:00 UTC

[Refactor] Move MM data parsing outside processor (#33408)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 1, 2026 at 00:46 UTC 88c3e114d8f564d1e88ff2ef5be56a979792e90b
43 files changed +228 -139
tests/models/multimodal/processing/test_common.py
+6 -5
@@ -176,7 +176,7 @@ def get_text_token_prompts(
176 if model_type in MM_DATA_PATCHES:
177 mm_data = MM_DATA_PATCHES[model_type](mm_data)
178
179 - parsed_data = processor.data_parser.parse_mm_data(mm_data)
179 + parsed_data = processor.info.parse_mm_data(mm_data)
180 mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
181
182 text_prompt: str | None
@@ -336,17 +336,18 @@ def _test_processing_correctness_one(
336 model_type = model_config.hf_config.model_type
337
338 text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
339 + mm_items = baseline_processor.info.parse_mm_data(mm_data)
340 ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
341
342 baseline_tokenized_result = baseline_processor.apply(
343 token_prompt,
343 - mm_data=mm_data,
344 + mm_items=mm_items,
345 hf_processor_mm_kwargs={},
346 )
347
348 cached_tokenized_result = cached_processor.apply(
349 token_prompt,
349 - mm_data=mm_data,
350 + mm_items=mm_items,
351 hf_processor_mm_kwargs={},
352 )
353
@@ -360,12 +361,12 @@ def _test_processing_correctness_one(
361 if text_prompt is not None:
362 baseline_text_result = baseline_processor.apply(
363 text_prompt,
363 - mm_data=mm_data,
364 + mm_items=mm_items,
365 hf_processor_mm_kwargs={},
366 )
367 cached_text_result = cached_processor.apply(
368 text_prompt,
368 - mm_data=mm_data,
369 + mm_items=mm_items,
370 hf_processor_mm_kwargs={},
371 )
372
tests/models/multimodal/processing/test_gemma3.py
+5 -1
@@ -175,7 +175,11 @@ def test_get_image_size_with_most_features(
175
176 for asset in image_assets:
177 mm_data = {"image": [asset.pil_image]}
178 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
178 + processed_inputs = processor.apply(
179 + prompt,
180 + mm_items=processor.info.parse_mm_data(mm_data),
181 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
182 + )
183 mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
184 num_patches_tensor = mm_kwargs_data["num_patches"]
185 tokens = int(num_patches_tensor.item()) * image_seq_length
tests/models/multimodal/processing/test_glm4_1v.py
+15 -3
@@ -52,7 +52,11 @@ def test_processor_override(
52 metadata["fps"] = fps
53 mm_data = {"video": [(video, metadata)]}
54
55 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
55 + processed_inputs = processor.apply(
56 + prompt,
57 + mm_items=processor.info.parse_mm_data(mm_data),
58 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
59 + )
60
61 # Ensure we have the right number of placeholders per num_crops size
62 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -100,8 +104,16 @@ def test_video_loader_consistency(
104 static_mm_data = {"video": [(static_video, static_metadata)]}
105 dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
106
103 - static_outputs = processor.apply(prompt, static_mm_data, hf_processor_mm_kwargs)
104 - dynamic_outputs = processor.apply(prompt, dynamic_mm_data, hf_processor_mm_kwargs)
107 + static_outputs = processor.apply(
108 + prompt,
109 + mm_items=processor.info.parse_mm_data(static_mm_data),
110 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
111 + )
112 + dynamic_outputs = processor.apply(
113 + prompt,
114 + mm_items=processor.info.parse_mm_data(dynamic_mm_data),
115 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
116 + )
117
118 assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"]
119 assert batched_tensors_equal(
tests/models/multimodal/processing/test_h2ovl.py
+5 -1
@@ -106,7 +106,11 @@ def _run_check(
106 for image in images
107 )
108
109 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
109 + processed_inputs = processor.apply(
110 + prompt,
111 + mm_items=processor.info.parse_mm_data(mm_data),
112 + hf_processor_mm_kwargs=mm_processor_kwargs,
113 + )
114
115 # Ensure we have the right number of placeholders per num_crops size
116 image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
tests/models/multimodal/processing/test_idefics3.py
+5 -1
@@ -55,7 +55,11 @@ def test_processor_override(
55 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
56 mm_data = {"image": [dummy_image] * num_imgs}
57
58 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
58 + processed_inputs = processor.apply(
59 + prompt,
60 + mm_items=processor.info.parse_mm_data(mm_data),
61 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
62 + )
63
64 # Ensure the placeholders format are correct
65 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
tests/models/multimodal/processing/test_internvl.py
+5 -1
@@ -66,7 +66,11 @@ def _run_check(
66 for image in images
67 )
68
69 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
69 + processed_inputs = processor.apply(
70 + prompt,
71 + mm_items=processor.info.parse_mm_data(mm_data),
72 + hf_processor_mm_kwargs=mm_processor_kwargs,
73 + )
74
75 # Ensure we have the right number of placeholders per num_crops size
76 image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
tests/models/multimodal/processing/test_llama4.py
+5 -1
@@ -49,7 +49,11 @@ def test_processor_override(
49 if tokenized_prompt:
50 prompt = tokenizer.encode(prompt)
51
52 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
52 + processed_inputs = processor.apply(
53 + prompt,
54 + mm_items=processor.info.parse_mm_data(mm_data),
55 + hf_processor_mm_kwargs=mm_processor_kwargs,
56 + )
57 mm_data = processed_inputs["mm_kwargs"].get_data()
58
59 # place holder replacements
tests/models/multimodal/processing/test_llava_next.py
+5 -1
@@ -87,7 +87,11 @@ def _validate_image_prompt_replacements_one(
87 try:
88 # The processor will throw an error if there is a mismatch
89 # in the prompt replacements
90 - processed_inputs = processor.apply(prompt, mm_data, {})
90 + processed_inputs = processor.apply(
91 + prompt,
92 + mm_items=processor.info.parse_mm_data(mm_data),
93 + hf_processor_mm_kwargs={},
94 + )
95
96 image_placeholders = processed_inputs["mm_placeholders"]["image"]
97 assert len(image_placeholders) == num_imgs
tests/models/multimodal/processing/test_llava_onevision.py
+5 -1
@@ -87,7 +87,11 @@ def _validate_image_prompt_replacements_one(
87 try:
88 # The processor will throw an error if there is a mismatch
89 # in the prompt replacements
90 - processed_inputs = processor.apply(prompt, mm_data, {})
90 + processed_inputs = processor.apply(
91 + prompt,
92 + mm_items=processor.info.parse_mm_data(mm_data),
93 + hf_processor_mm_kwargs={},
94 + )
95
96 image_placeholders = processed_inputs["mm_placeholders"]["image"]
97 assert len(image_placeholders) == num_imgs
tests/models/multimodal/processing/test_minimax_vl_01.py
+10 -2
@@ -29,7 +29,11 @@ def test_processor_override(
29 image = Image.new("RGB", size=(364, 364))
30 mm_data = {"image": [image] * num_imgs}
31
32 - processed_inputs = processor.apply(prompt, mm_data, {})
32 + processed_inputs = processor.apply(
33 + prompt,
34 + mm_items=processor.info.parse_mm_data(mm_data),
35 + hf_processor_mm_kwargs={},
36 + )
37 image_placeholders = processed_inputs["mm_placeholders"]["image"]
38
39 assert len(image_placeholders) == num_imgs
@@ -46,7 +50,11 @@ def _validate_image_prompt_replacements_one(
50 mm_data = {"image": [image] * num_imgs}
51
52 try:
49 - processed_inputs = processor.apply(prompt, mm_data, {})
53 + processed_inputs = processor.apply(
54 + prompt,
55 + mm_items=processor.info.parse_mm_data(mm_data),
56 + hf_processor_mm_kwargs={},
57 + )
58
59 image_placeholders = processed_inputs["mm_placeholders"]["image"]
60 assert len(image_placeholders) == num_imgs
tests/models/multimodal/processing/test_nemotron_vl.py
+5 -1
@@ -68,7 +68,11 @@ def _run_check(
68 for image in images
69 )
70 print(total_expected_num_patches)
71 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
71 + processed_inputs = processor.apply(
72 + prompt,
73 + mm_items=processor.info.parse_mm_data(mm_data),
74 + hf_processor_mm_kwargs=mm_processor_kwargs,
75 + )
76
77 # Ensure we have the right number of placeholders per num_crops size
78 image_token_id = tokenizer.convert_tokens_to_ids("<image>")
tests/models/multimodal/processing/test_phi3v.py
+5 -1
@@ -47,7 +47,11 @@ def test_processor_override(
47 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
48 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
49
50 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
50 + processed_inputs = processor.apply(
51 + prompt,
52 + mm_items=processor.info.parse_mm_data(mm_data),
53 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
54 + )
55
56 # Ensure we have the right number of placeholders per num_crops size
57 img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
tests/models/multimodal/processing/test_phi4mm.py
+5 -1
@@ -51,7 +51,11 @@ def test_processor_override(
51 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
52 mm_data = {"image": [dummy_image] * num_imgs}
53
54 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
54 + processed_inputs = processor.apply(
55 + prompt,
56 + mm_items=processor.info.parse_mm_data(mm_data),
57 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
58 + )
59
60 # Ensure we have the right number of placeholders per num_crops size
61 img_tok_count = processed_inputs["prompt_token_ids"].count(
tests/models/multimodal/processing/test_qwen2_vl.py
+10 -2
@@ -42,7 +42,11 @@ def test_processor_override(
42 prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
43 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
44
45 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
45 + processed_inputs = processor.apply(
46 + prompt,
47 + mm_items=processor.info.parse_mm_data(mm_data),
48 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
49 + )
50
51 # Ensure we have the right number of placeholders per num_crops size
52 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -83,7 +87,11 @@ def test_get_image_size_with_most_features(
87 prompt = "<|vision_start|><|image_pad|><|vision_end|>"
88 for asset in image_assets:
89 mm_data = {"image": [asset.pil_image]}
86 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
90 + processed_inputs = processor.apply(
91 + prompt,
92 + mm_items=processor.info.parse_mm_data(mm_data),
93 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
94 + )
95 grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
96 t, h, w = grid_thw[0]
97 tokens = (t * h * w) // (merge_size**2)
tests/models/multimodal/processing/test_qwen3_omni.py
+10 -2
@@ -51,7 +51,11 @@ def test_processor_with_audio_sample_rate(
51 hf_processor_mm_kwargs: dict[str, Any] = {
52 "audio_sample_rate": audio_sample_rate,
53 }
54 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
54 + processed_inputs = processor.apply(
55 + prompt,
56 + mm_items=processor.info.parse_mm_data(mm_data),
57 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
58 + )
59
60 # Verify audio tokens are generated
61 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -90,7 +94,11 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
94 hf_processor_mm_kwargs: dict[str, Any] = {
95 "audio_sample_rate": audio_sample_rate,
96 }
93 - processed = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
97 + processed = processor.apply(
98 + prompt,
99 + mm_items=processor.info.parse_mm_data(mm_data),
100 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
101 + )
102 hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
103 audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)
104 return processed["prompt_token_ids"].count(audio_token_id)
tests/models/multimodal/processing/test_smolvlm.py
+5 -1
@@ -55,7 +55,11 @@ def test_processor_override(
55 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
56 mm_data = {"image": [dummy_image] * num_imgs}
57
58 - processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
58 + processed_inputs = processor.apply(
59 + prompt,
60 + mm_items=processor.info.parse_mm_data(mm_data),
61 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
62 + )
63
64 # Ensure the placeholders format are correct
65 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
tests/models/multimodal/processing/test_tensor_schema.py
+6 -12
@@ -24,10 +24,7 @@ from vllm.distributed import (
24 init_distributed_environment,
25 initialize_model_parallel,
26 )
27 -from vllm.model_executor.models.interfaces import (
28 - SupportsMultiModal,
29 - supports_multimodal,
30 -)
27 +from vllm.model_executor.models.interfaces import supports_multimodal
28 from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
29 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
30 from vllm.multimodal.utils import group_mm_kwargs_by_modality
@@ -86,7 +83,6 @@ def resize_mm_data(
83
84
85 def create_batched_mm_kwargs(
89 - model_cls: type[SupportsMultiModal],
86 model_config: ModelConfig,
87 processor: BaseMultiModalProcessor,
88 size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
@@ -102,10 +98,10 @@ def create_batched_mm_kwargs(
98 seq_len=model_config.max_model_len,
99 mm_counts=mm_counts,
100 )
105 - mm_data = processor_inputs.mm_data
101 + mm_items = processor_inputs.mm_items
102 resized_mm_data = {
107 - modality: resize_mm_data(data, size_factors)
108 - for modality, data in mm_data.items()
103 + modality: resize_mm_data(items.data, size_factors)
104 + for modality, items in mm_items.items()
105 }
106
107 # video metadata will be added back to the resized video data here.
@@ -113,7 +109,7 @@ def create_batched_mm_kwargs(
109
110 mm_kwargs = processor.apply(
111 prompt=token_prompt if text_prompt is None else text_prompt,
116 - mm_data=resized_mm_data,
112 + mm_items=processor.info.parse_mm_data(resized_mm_data),
113 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
114 tokenization_kwargs=processor_inputs.tokenization_kwargs,
115 )["mm_kwargs"].require_data()
@@ -246,9 +242,7 @@ def test_model_tensor_schema(model_id: str):
242 processor = factories.build_processor(ctx, cache=None)
243
244 with initialize_dummy_model(model_cls, model_config) as model:
249 - for modality, _, mm_kwargs in create_batched_mm_kwargs(
250 - model_cls, model_config, processor
251 - ):
245 + for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
246 for method_name in inputs_parse_methods:
247 print(
248 f"Testing `{method_name}` with modality={modality} "
tests/models/multimodal/processing/test_transformers.py
+2 -2
@@ -21,7 +21,7 @@ def test_multimodal_processor(model_id):
21 str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
22 str_processed_inputs = mm_processor.apply(
23 prompt=str_prompt,
24 - mm_data=mm_data,
24 + mm_items=mm_processor.info.parse_mm_data(mm_data),
25 hf_processor_mm_kwargs={},
26 )
27
@@ -46,7 +46,7 @@ def test_multimodal_processor(model_id):
46 ]
47 ids_processed_inputs = mm_processor.apply(
48 prompt=ids_prompt,
49 - mm_data=mm_data,
49 + mm_items=mm_processor.info.parse_mm_data(mm_data),
50 hf_processor_mm_kwargs={},
51 )
52
tests/multimodal/test_processing.py
+1 -1
@@ -970,7 +970,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
970 with exc_ctx:
971 processor.apply(
972 "<image>" * num_images,
973 - mm_data=mm_data,
973 + mm_items=processor.info.parse_mm_data(mm_data),
974 hf_processor_mm_kwargs={},
975 )
976
vllm/inputs/preprocess.py
+2 -1
@@ -258,9 +258,10 @@ class InputPreprocessor:
258 if mm_processor_kwargs is None:
259 mm_processor_kwargs = {}
260
261 + mm_items = mm_processor.info.parse_mm_data(mm_data)
262 mm_input = mm_processor.apply(
263 prompt,
263 - mm_data,
264 + mm_items,
265 hf_processor_mm_kwargs=mm_processor_kwargs,
266 tokenization_kwargs=tokenization_kwargs,
267 mm_uuids=mm_uuids,
vllm/model_executor/models/aya_vision.py
+2 -3
@@ -227,9 +227,8 @@ class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingIn
227
228 # HF processor pops the `num_patches` kwarg, which is needed by vLLM
229 if (images := mm_data.get("images")) is not None:
230 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
231 - "image", ImageProcessorItems
232 - )
230 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
231 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
232 image_sizes = [
233 parsed_images.get_image_size(i) for i in range(len(parsed_images))
234 ]
vllm/model_executor/models/clip.py
+4 -4
@@ -201,20 +201,20 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
201 def apply(
202 self,
203 prompt: str | list[int],
204 - mm_data: MultiModalDataDict,
204 + mm_items: MultiModalDataItems,
205 hf_processor_mm_kwargs: Mapping[str, object],
206 tokenization_kwargs: Mapping[str, object] | None = None,
207 *,
208 mm_uuids: MultiModalUUIDDict | None = None,
209 ) -> MultiModalInputs:
210 - if prompt and mm_data:
210 + if prompt and mm_items:
211 raise ValueError(
212 "CLIP accepts text-only or image-only inputs, not both! "
213 "Image-only inputs means passing an image with an empty text "
214 "prompt."
215 )
216
217 - if mm_data:
217 + if mm_items:
218 # For multi-modal data, the prompt after processing should
219 # only contain the dummy image tokens
220 tokenization_kwargs = {
@@ -224,7 +224,7 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
224
225 return super().apply(
226 prompt=prompt,
227 - mm_data=mm_data,
227 + mm_items=mm_items,
228 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
229 tokenization_kwargs=tokenization_kwargs,
230 mm_uuids=mm_uuids,
vllm/model_executor/models/cohere2_vision.py
+2 -3
@@ -262,9 +262,8 @@ class Cohere2VisionMultiModalProcessor(
262 hf_processor = self.info.get_hf_processor(**mm_kwargs)
263
264 # Fallback calculation if HF processor didn't provide num_patches
265 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
266 - "image", ImageProcessorItems
267 - )
265 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
266 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
267
268 num_patches = [
269 self.info.get_num_patches(
vllm/model_executor/models/gemma3_mm.py
+2 -3
@@ -290,9 +290,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
290
291 # HF processor pops the `num_crops` kwarg, which is needed by vLLM
292 if (images := mm_data.get("images")) is not None:
293 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
294 - "image", ImageProcessorItems
295 - )
293 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
294 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
295 image_sizes = [
296 parsed_images.get_image_size(i) for i in range(len(parsed_images))
297 ]
vllm/model_executor/models/idefics3.py
+2 -3
@@ -349,9 +349,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
349 tok_kwargs,
350 )
351
352 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
353 - "image", ImageProcessorItems
354 - )
352 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
353 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
354 image_sizes = [
355 parsed_images.get_image_size(i) for i in range(len(parsed_images))
356 ]
vllm/model_executor/models/lfm2_vl.py
+2 -3
@@ -357,9 +357,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
357 tok_kwargs,
358 )
359
360 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
361 - "image", ImageProcessorItems
362 - )
360 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
361 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
362 image_sizes = [
363 parsed_images.get_image_size(i) for i in range(len(parsed_images))
364 ]
vllm/model_executor/models/llava.py
+2 -3
@@ -769,7 +769,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
769 def apply(
770 self,
771 prompt: str | list[int],
772 - mm_data: MultiModalDataDict,
772 + mm_items: MultiModalDataItems,
773 hf_processor_mm_kwargs: Mapping[str, object],
774 tokenization_kwargs: Mapping[str, object] | None = None,
775 mm_uuids: MultiModalUUIDDict | None = None,
@@ -785,13 +785,12 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
785
786 result = super().apply(
787 prompt,
788 - mm_data,
788 + mm_items,
789 hf_processor_mm_kwargs,
790 tokenization_kwargs,
791 mm_uuids=mm_uuids,
792 )
793
794 - mm_items = self._to_mm_items(mm_data)
794 mm_item_counts = mm_items.get_all_counts()
795 mm_kwargs = result["mm_kwargs"]
796 mm_hashes = result["mm_hashes"]
vllm/model_executor/models/minicpmo.py
+2 -1
@@ -300,7 +300,8 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
300 if (audios := mm_data.get("audios")) is None:
301 return {}
302
303 - parsed_audios = self.data_parser.parse_mm_data({"audio": audios}).get_items(
303 + mm_items = self.info.parse_mm_data({"audio": audios}, validate=False)
304 + parsed_audios = mm_items.get_items(
305 "audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
306 )
307
vllm/model_executor/models/minicpmv.py
+4 -2
@@ -767,7 +767,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
767 if (images := mm_data.get("images")) is None:
768 return {}
769
770 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
770 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
771 + parsed_images = mm_items.get_items(
772 "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
773 )
774
@@ -793,7 +794,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
794 if (videos := mm_data.get("videos")) is None:
795 return {}
796
796 - parsed_videos = self.data_parser.parse_mm_data({"video": videos}).get_items(
797 + mm_items = self.info.parse_mm_data({"video": videos}, validate=False)
798 + parsed_videos = mm_items.get_items(
799 "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
800 )
801
vllm/model_executor/models/mllama4.py
+2 -3
@@ -609,9 +609,8 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
609 )
610
611 images = mm_data["images"]
612 - parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
613 - "image", ImageProcessorItems
614 - )
612 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
613 + parsed_images = mm_items.get_items("image", ImageProcessorItems)
614
615 tile_size = vision_config.image_size
616 possible_resolutions = find_supported_resolutions(
vllm/model_executor/models/nemotron_parse.py
+1 -1
@@ -660,7 +660,7 @@ class NemotronParseMultiModalProcessor(
660 def create_encoder_prompt(
661 self,
662 prompt: str | list[int],
663 - mm_data: MultiModalDataDict,
663 + mm_items: MultiModalDataItems,
664 ) -> str | list[int]:
665 return [0]
666
vllm/model_executor/models/paligemma.py
+2 -2
@@ -225,14 +225,14 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
225 def apply(
226 self,
227 prompt: str | list[int],
228 - mm_data: MultiModalDataDict,
228 + mm_items: MultiModalDataItems,
229 hf_processor_mm_kwargs: Mapping[str, object],
230 tokenization_kwargs: Mapping[str, object] | None = None,
231 mm_uuids: MultiModalUUIDDict | None = None,
232 ) -> MultiModalInputs:
233 mm_inputs = super().apply(
234 prompt,
235 - mm_data,
235 + mm_items,
236 hf_processor_mm_kwargs,
237 tokenization_kwargs,
238 mm_uuids=mm_uuids,
vllm/model_executor/models/pixtral.py
+3 -1
@@ -303,9 +303,11 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
303 res = tokenizer.mistral.encode_chat_completion(request)
304 dummy_tokens = res.tokens
305
306 + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
307 +
308 return ProcessorInputs(
309 prompt=dummy_tokens,
308 - mm_data=dummy_mm_data,
310 + mm_items=dummy_mm_items,
311 tokenization_kwargs=tokenization_kwargs,
312 )
313
vllm/model_executor/models/siglip.py
+4 -4
@@ -187,20 +187,20 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
187 def apply(
188 self,
189 prompt: str | list[int],
190 - mm_data: MultiModalDataDict,
190 + mm_items: MultiModalDataItems,
191 hf_processor_mm_kwargs: Mapping[str, object],
192 tokenization_kwargs: Mapping[str, object] | None = None,
193 *,
194 mm_uuids: MultiModalUUIDDict | None = None,
195 ) -> MultiModalInputs:
196 - if prompt and mm_data:
196 + if prompt and mm_items:
197 raise ValueError(
198 "Siglip accepts text-only or image-only inputs, not both! "
199 "Image-only inputs means passing an image with an empty text "
200 "prompt."
201 )
202
203 - if mm_data:
203 + if mm_items:
204 # For multi-modal data, the prompt after processing should
205 # only contain the image token
206 tokenization_kwargs = {
@@ -210,7 +210,7 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
210
211 return super().apply(
212 prompt=prompt,
213 - mm_data=mm_data,
213 + mm_items=mm_items,
214 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
215 tokenization_kwargs=tokenization_kwargs,
216 mm_uuids=mm_uuids,
vllm/model_executor/models/terratorch.py
+6 -6
@@ -180,20 +180,20 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
180 def apply(
181 self,
182 prompt: str | list[int],
183 - mm_data: MultiModalDataDict,
183 + mm_items: MultiModalDataItems,
184 hf_processor_mm_kwargs: Mapping[str, object],
185 tokenization_kwargs: Mapping[str, object] | None = None,
186 mm_uuids: MultiModalUUIDDict | None = None,
187 ) -> MultiModalInputs:
188 - mm_items = self._to_mm_items(mm_data)
189 - tokenization_kwargs = tokenization_kwargs or {}
188 + if tokenization_kwargs is None:
189 + tokenization_kwargs = {}
190 +
191 mm_hashes = self._hash_mm_items(
192 mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
193 )
194
194 - mm_processed_data = BatchFeature(
195 - mm_data.get("image", mm_data), tensor_type="pt"
196 - )
195 + _, passthrough_data = self._get_hf_mm_data(mm_items)
196 + mm_processed_data = BatchFeature(dict(passthrough_data), tensor_type="pt")
197 mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
198
199 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
vllm/model_executor/models/transformers/multimodal.py
+1 -2
@@ -174,7 +174,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
174 def apply(
175 self,
176 prompt: str | list[int],
177 - mm_data: MultiModalDataDict,
177 + mm_items: MultiModalDataItems,
178 hf_processor_mm_kwargs: Mapping[str, object],
179 tokenization_kwargs: Mapping[str, object] | None = None,
180 mm_uuids: MultiModalUUIDDict | None = None,
@@ -188,7 +188,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
188 if tokenization_kwargs is None:
189 tokenization_kwargs = {}
190
191 - mm_items = self._to_mm_items(mm_data)
191 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
192 if not isinstance(prompt, str):
193 # the prompt is the tokenized ids which is not supported
vllm/model_executor/models/voxtral.py
+7 -4
@@ -262,11 +262,14 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
262 )
263 res = tokenizer.mistral.encode_chat_completion(request)
264 dummy_tokens = res.tokens
265 - # whixtral tokenizer adds padding to the audio
266 - # so we need to update the audio arrays
267 - dummy_mm_data["audio"] = [a.audio_array for a in res.audios]
265
269 - return ProcessorInputs(prompt=dummy_tokens, mm_data=dummy_mm_data)
266 + dummy_mm_inputs = self.info.parse_mm_data(
267 + # whixtral tokenizer adds padding to the audio
268 + # so we need to update the audio arrays
269 + {**dummy_mm_data, "audio": [a.audio_array for a in res.audios]},
270 + )
271 +
272 + return ProcessorInputs(prompt=dummy_tokens, mm_items=dummy_mm_inputs)
273
274
275 class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
vllm/model_executor/models/whisper.py
+1 -1
@@ -705,7 +705,7 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
705 def create_encoder_prompt(
706 self,
707 prompt: str | list[int],
708 - mm_data: MultiModalDataDict,
708 + mm_items: MultiModalDataItems,
709 ) -> str | list[int]:
710 # Strictly speaking, whisper encoder only accept audio features.
711 # We create a dummy encoder prompt here which will be padded to
vllm/multimodal/processing/context.py
+41 -1
@@ -14,7 +14,13 @@ import torch
14 from typing_extensions import TypeVar
15
16 from vllm.logger import init_logger
17 -from vllm.multimodal.parse import MultiModalDataParser
17 +from vllm.multimodal.inputs import MultiModalDataDict
18 +from vllm.multimodal.parse import (
19 + DictEmbeddingItems,
20 + EmbeddingItems,
21 + MultiModalDataItems,
22 + MultiModalDataParser,
23 +)
24 from vllm.tokenizers import TokenizerLike
25 from vllm.transformers_utils.processor import cached_processor_from_config
26 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -596,6 +602,10 @@ class BaseProcessingInfo:
602 expected_hidden_size=self._get_expected_hidden_size(),
603 )
604
605 + @cached_property
606 + def data_parser(self) -> MultiModalDataParser:
607 + return self.get_data_parser()
608 +
609 @property
610 def skip_prompt_length_check(self) -> bool:
611 return False
@@ -655,6 +665,36 @@ class BaseProcessingInfo:
665
666 raise ValueError(msg)
667
668 + def parse_mm_data(
669 + self,
670 + mm_data: MultiModalDataDict,
671 + *,
672 + validate: bool = True,
673 + ) -> MultiModalDataItems:
674 + """
675 + Normalize
676 + [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
677 + to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
678 + before passing them to
679 + [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
680 + """
681 + mm_items = self.data_parser.parse_mm_data(mm_data)
682 +
683 + if validate:
684 + mm_config = self.ctx.model_config.get_multimodal_config()
685 + if not mm_config.enable_mm_embeds:
686 + for modality, items in mm_items.items():
687 + if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
688 + raise ValueError(
689 + f"You must set `--enable-mm-embeds` to input "
690 + f"`{modality}_embeds`"
691 + )
692 +
693 + for modality, items in mm_items.items():
694 + self.validate_num_items(modality, len(items))
695 +
696 + return mm_items
697 +
698 def get_mm_max_tokens_per_item(
699 self,
700 seq_len: int,
vllm/multimodal/processing/dummy_inputs.py
+4 -4
@@ -18,6 +18,7 @@ from vllm.config.multimodal import (
18 from vllm.logger import init_logger
19
20 from ..inputs import MultiModalDataDict
21 +from ..parse import MultiModalDataItems
22 from .context import BaseProcessingInfo
23
24 _I = TypeVar("_I", bound=BaseProcessingInfo)
@@ -33,7 +34,7 @@ class ProcessorInputs:
34 """
35
36 prompt: str | list[int]
36 - mm_data: MultiModalDataDict
37 + mm_items: MultiModalDataItems
38 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
39 tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
40
@@ -93,15 +94,14 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
94 mm_options: Configurable options per modality (optional)
95 """
96 dummy_text = self.get_dummy_text(mm_counts)
96 -
97 - # Use the unified function for both legacy and configurable cases
97 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
98 + dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
99
100 tokenization_kwargs = {"truncation": False}
101
102 return ProcessorInputs(
103 prompt=dummy_text,
104 - mm_data=dummy_mm_data,
104 + mm_items=dummy_mm_items,
105 tokenization_kwargs=tokenization_kwargs,
106 )
107
vllm/multimodal/processing/processor.py
+15 -42
@@ -25,7 +25,6 @@ from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25
26 from ..hasher import MultiModalHasher
27 from ..inputs import (
28 - MultiModalDataDict,
28 MultiModalEncDecInputs,
29 MultiModalFieldConfig,
30 MultiModalHashes,
@@ -1013,39 +1012,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1012 def __call__(
1013 self,
1014 prompt: str,
1016 - mm_data: MultiModalDataDict,
1015 + mm_items: MultiModalDataItems,
1016 hf_processor_mm_kwargs: Mapping[str, object],
1017 *,
1018 mm_uuids: MultiModalUUIDDict | None = None,
1019 ) -> MultiModalInputs:
1021 - return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1022 -
1023 - def _to_mm_items(
1024 - self,
1025 - mm_data: MultiModalDataDict,
1026 - ) -> MultiModalDataItems:
1027 - """
1028 - Normalize
1029 - [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
1030 - to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
1031 - before passing them to
1032 - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1033 - """
1034 - mm_items = self.data_parser.parse_mm_data(mm_data)
1035 -
1036 - mm_config = self.info.ctx.model_config.get_multimodal_config()
1037 - if not mm_config.enable_mm_embeds:
1038 - for modality, items in mm_items.items():
1039 - if isinstance(items, (EmbeddingItems, DictEmbeddingItems)):
1040 - raise ValueError(
1041 - f"You must set `--enable-mm-embeds` to input "
1042 - f"`{modality}_embeds`"
1043 - )
1044 -
1045 - for modality, items in mm_items.items():
1046 - self.info.validate_num_items(modality, len(items))
1047 -
1048 - return mm_items
1020 + return self.apply(prompt, mm_items, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1021
1022 @abstractmethod
1023 def _get_mm_fields_config(
@@ -1409,6 +1381,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1381 ]
1382 for modality, items_is_cached in mm_is_cached.items()
1383 }
1384 +
1385 mm_missing_data = {}
1386 for modality, idxs in mm_missing_idxs.items():
1387 missing_modality_data = []
@@ -1423,7 +1396,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1396 missing_modality_data.append(data)
1397 mm_missing_data[modality] = missing_modality_data
1398
1426 - return mm_is_cached, self._to_mm_items(mm_missing_data)
1399 + mm_missing_items = self.info.parse_mm_data(mm_missing_data)
1400 +
1401 + return mm_is_cached, mm_missing_items
1402
1403 def _recompute_cached_prompt_update(
1404 self,
@@ -1774,7 +1749,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1749 def apply(
1750 self,
1751 prompt: str | list[int],
1777 - mm_data: MultiModalDataDict,
1752 + mm_items: MultiModalDataItems,
1753 hf_processor_mm_kwargs: Mapping[str, object],
1754 tokenization_kwargs: Mapping[str, object] | None = None,
1755 *,
@@ -1797,8 +1772,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1772 if request_id is not None:
1773 self.info.ctx.create_timing_stats(request_id)
1774
1800 - mm_items = self._to_mm_items(mm_data)
1801 -
1775 if tokenization_kwargs is None:
1776 tokenization_kwargs = {}
1777
@@ -1843,7 +1816,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1816 def create_encoder_prompt(
1817 self,
1818 prompt: str | list[int],
1846 - mm_data: MultiModalDataDict,
1819 + mm_items: MultiModalDataItems,
1820 ) -> str | list[int]:
1821 """
1822 Create input prompt for the encoder. HF processor will be applied on
@@ -1854,7 +1827,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1827 def create_decoder_prompt(
1828 self,
1829 prompt: str | list[int],
1857 - mm_data: MultiModalDataDict,
1830 + mm_items: MultiModalDataItems,
1831 ) -> str | list[int]:
1832 """Create input prompt for the decoder."""
1833 return prompt
@@ -1862,11 +1835,11 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1835 def _get_enc_dec_inputs(
1836 self,
1837 prompt: str | list[int],
1865 - mm_data: MultiModalDataDict,
1838 + mm_items: MultiModalDataItems,
1839 encoder_inputs: MultiModalInputs,
1840 ):
1841 tokenizer = self.info.get_tokenizer()
1869 - decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
1842 + decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
1843 if isinstance(decoder_prompt_raw, str):
1844 decoder_prompt_ids = tokenizer.encode(
1845 decoder_prompt_raw, add_special_tokens=False
@@ -1884,7 +1857,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1857 def apply(
1858 self,
1859 prompt: str | list[int],
1887 - mm_data: MultiModalDataDict,
1860 + mm_items: MultiModalDataItems,
1861 hf_processor_mm_kwargs: Mapping[str, object],
1862 tokenization_kwargs: Mapping[str, object] | None = None,
1863 *,
@@ -1897,10 +1870,10 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1870 2. Apply the HF processor on encoder prompt.
1871 3. Copy the input prompt text as decoder prompt inputs.
1872 """
1900 - encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
1873 + encoder_prompt = self.create_encoder_prompt(prompt, mm_items)
1874 encoder_inputs = super().apply(
1875 encoder_prompt,
1903 - mm_data,
1876 + mm_items,
1877 hf_processor_mm_kwargs,
1878 tokenization_kwargs,
1879 mm_uuids=mm_uuids,
@@ -1908,6 +1881,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1881
1882 return self._get_enc_dec_inputs(
1883 prompt=prompt,
1911 - mm_data=mm_data,
1884 + mm_items=mm_items,
1885 encoder_inputs=encoder_inputs,
1886 )
vllm/multimodal/registry.py
+1 -1
@@ -330,7 +330,7 @@ class MultiModalRegistry:
330 )
331 mm_inputs = processor.apply(
332 prompt=processor_inputs.prompt,
333 - mm_data=processor_inputs.mm_data,
333 + mm_items=processor_inputs.mm_items,
334 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
335 tokenization_kwargs=processor_inputs.tokenization_kwargs,
336 )
vllm/v1/engine/input_processor.py
+1 -1
@@ -212,7 +212,7 @@ class InputProcessor:
212
213 def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
214 mm_processor = self.input_preprocessor._get_mm_processor()
215 - return mm_processor.data_parser.parse_mm_data(mm_data)
215 + return mm_processor.info.parse_mm_data(mm_data)
216
217 def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:
218 if isinstance(prompt, str):