@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 392645454b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 21:30 UTC

[Refactor] Decouple TimingContext from InputProcessingContext (#35083)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 23, 2026 at 22:15 UTC 392645454b34fec6ff2b690465b275e773e6af09
38 files changed +423 -653
tests/models/multimodal/processing/test_common.py
+4 -4
@@ -389,13 +389,13 @@ def _test_processing_correctness_one(
389 mm_items = baseline_processor.info.parse_mm_data(mm_data)
390 ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
391
392 - baseline_tokenized_result = baseline_processor.apply(
392 + baseline_tokenized_result = baseline_processor(
393 token_prompt,
394 mm_items=mm_items,
395 hf_processor_mm_kwargs={},
396 )
397
398 - cached_tokenized_result = cached_processor.apply(
398 + cached_tokenized_result = cached_processor(
399 token_prompt,
400 mm_items=mm_items,
401 hf_processor_mm_kwargs={},
@@ -409,12 +409,12 @@ def _test_processing_correctness_one(
409 )
410
411 if text_prompt is not None:
412 - baseline_text_result = baseline_processor.apply(
412 + baseline_text_result = baseline_processor(
413 text_prompt,
414 mm_items=mm_items,
415 hf_processor_mm_kwargs={},
416 )
417 - cached_text_result = cached_processor.apply(
417 + cached_text_result = cached_processor(
418 text_prompt,
419 mm_items=mm_items,
420 hf_processor_mm_kwargs={},
tests/models/multimodal/processing/test_gemma3.py
+1 -1
@@ -176,7 +176,7 @@ def test_get_image_size_with_most_features(
176
177 for asset in image_assets:
178 mm_data = {"image": [asset.pil_image]}
179 - processed_inputs = processor.apply(
179 + processed_inputs = processor(
180 prompt,
181 mm_items=processor.info.parse_mm_data(mm_data),
182 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_glm4_1v.py
+3 -3
@@ -52,7 +52,7 @@ def test_processor_override(
52 metadata["fps"] = fps
53 mm_data = {"video": [(video, metadata)]}
54
55 - processed_inputs = processor.apply(
55 + processed_inputs = processor(
56 prompt,
57 mm_items=processor.info.parse_mm_data(mm_data),
58 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -104,12 +104,12 @@ def test_video_loader_consistency(
104 static_mm_data = {"video": [(static_video, static_metadata)]}
105 dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
106
107 - static_outputs = processor.apply(
107 + static_outputs = processor(
108 prompt,
109 mm_items=processor.info.parse_mm_data(static_mm_data),
110 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
111 )
112 - dynamic_outputs = processor.apply(
112 + dynamic_outputs = processor(
113 prompt,
114 mm_items=processor.info.parse_mm_data(dynamic_mm_data),
115 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_h2ovl.py
+1 -1
@@ -106,7 +106,7 @@ def _run_check(
106 for image in images
107 )
108
109 - processed_inputs = processor.apply(
109 + processed_inputs = processor(
110 prompt,
111 mm_items=processor.info.parse_mm_data(mm_data),
112 hf_processor_mm_kwargs=mm_processor_kwargs,
tests/models/multimodal/processing/test_idefics3.py
+1 -1
@@ -61,7 +61,7 @@ def test_processor_override(
61 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
62 mm_data = {"image": [dummy_image] * num_imgs}
63
64 - processed_inputs = processor.apply(
64 + processed_inputs = processor(
65 prompt,
66 mm_items=processor.info.parse_mm_data(mm_data),
67 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_internvl.py
+1 -1
@@ -66,7 +66,7 @@ def _run_check(
66 for image in images
67 )
68
69 - processed_inputs = processor.apply(
69 + processed_inputs = processor(
70 prompt,
71 mm_items=processor.info.parse_mm_data(mm_data),
72 hf_processor_mm_kwargs=mm_processor_kwargs,
tests/models/multimodal/processing/test_llama4.py
+1 -1
@@ -49,7 +49,7 @@ def test_processor_override(
49 if tokenized_prompt:
50 prompt = tokenizer.encode(prompt)
51
52 - processed_inputs = processor.apply(
52 + processed_inputs = processor(
53 prompt,
54 mm_items=processor.info.parse_mm_data(mm_data),
55 hf_processor_mm_kwargs=mm_processor_kwargs,
tests/models/multimodal/processing/test_llava_next.py
+1 -1
@@ -87,7 +87,7 @@ def _validate_image_prompt_replacements_one(
87 try:
88 # The processor will throw an error if there is a mismatch
89 # in the prompt replacements
90 - processed_inputs = processor.apply(
90 + processed_inputs = processor(
91 prompt,
92 mm_items=processor.info.parse_mm_data(mm_data),
93 hf_processor_mm_kwargs={},
tests/models/multimodal/processing/test_llava_onevision.py
+1 -1
@@ -87,7 +87,7 @@ def _validate_image_prompt_replacements_one(
87 try:
88 # The processor will throw an error if there is a mismatch
89 # in the prompt replacements
90 - processed_inputs = processor.apply(
90 + processed_inputs = processor(
91 prompt,
92 mm_items=processor.info.parse_mm_data(mm_data),
93 hf_processor_mm_kwargs={},
tests/models/multimodal/processing/test_minimax_vl_01.py
+2 -2
@@ -29,7 +29,7 @@ def test_processor_override(
29 image = Image.new("RGB", size=(364, 364))
30 mm_data = {"image": [image] * num_imgs}
31
32 - processed_inputs = processor.apply(
32 + processed_inputs = processor(
33 prompt,
34 mm_items=processor.info.parse_mm_data(mm_data),
35 hf_processor_mm_kwargs={},
@@ -50,7 +50,7 @@ def _validate_image_prompt_replacements_one(
50 mm_data = {"image": [image] * num_imgs}
51
52 try:
53 - processed_inputs = processor.apply(
53 + processed_inputs = processor(
54 prompt,
55 mm_items=processor.info.parse_mm_data(mm_data),
56 hf_processor_mm_kwargs={},
tests/models/multimodal/processing/test_nemotron_vl.py
+1 -1
@@ -68,7 +68,7 @@ def _run_check(
68 for image in images
69 )
70 print(total_expected_num_patches)
71 - processed_inputs = processor.apply(
71 + processed_inputs = processor(
72 prompt,
73 mm_items=processor.info.parse_mm_data(mm_data),
74 hf_processor_mm_kwargs=mm_processor_kwargs,
tests/models/multimodal/processing/test_phi3v.py
+1 -1
@@ -47,7 +47,7 @@ def test_processor_override(
47 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
48 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
49
50 - processed_inputs = processor.apply(
50 + processed_inputs = processor(
51 prompt,
52 mm_items=processor.info.parse_mm_data(mm_data),
53 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_phi4mm.py
+1 -1
@@ -51,7 +51,7 @@ def test_processor_override(
51 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
52 mm_data = {"image": [dummy_image] * num_imgs}
53
54 - processed_inputs = processor.apply(
54 + processed_inputs = processor(
55 prompt,
56 mm_items=processor.info.parse_mm_data(mm_data),
57 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_qwen2_vl.py
+2 -2
@@ -42,7 +42,7 @@ def test_processor_override(
42 prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
43 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
44
45 - processed_inputs = processor.apply(
45 + processed_inputs = processor(
46 prompt,
47 mm_items=processor.info.parse_mm_data(mm_data),
48 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -88,7 +88,7 @@ def test_get_image_size_with_most_features(
88 prompt = "<|vision_start|><|image_pad|><|vision_end|>"
89 for asset in image_assets:
90 mm_data = {"image": [asset.pil_image]}
91 - processed_inputs = processor.apply(
91 + processed_inputs = processor(
92 prompt,
93 mm_items=processor.info.parse_mm_data(mm_data),
94 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_qwen3_omni.py
+2 -2
@@ -51,7 +51,7 @@ def test_processor_with_audio_sample_rate(
51 hf_processor_mm_kwargs: dict[str, Any] = {
52 "audio_sample_rate": audio_sample_rate,
53 }
54 - processed_inputs = processor.apply(
54 + processed_inputs = processor(
55 prompt,
56 mm_items=processor.info.parse_mm_data(mm_data),
57 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -94,7 +94,7 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
94 hf_processor_mm_kwargs: dict[str, Any] = {
95 "audio_sample_rate": audio_sample_rate,
96 }
97 - processed = processor.apply(
97 + processed = processor(
98 prompt,
99 mm_items=processor.info.parse_mm_data(mm_data),
100 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_smolvlm.py
+1 -1
@@ -61,7 +61,7 @@ def test_processor_override(
61 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
62 mm_data = {"image": [dummy_image] * num_imgs}
63
64 - processed_inputs = processor.apply(
64 + processed_inputs = processor(
65 prompt,
66 mm_items=processor.info.parse_mm_data(mm_data),
67 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tests/models/multimodal/processing/test_tensor_schema.py
+2 -3
@@ -99,7 +99,7 @@ def create_batched_mm_kwargs(
99 mm_counts=mm_counts,
100 mm_options={},
101 )
102 - mm_items = processor_inputs.mm_items
102 + mm_items = processor_inputs.mm_data_items
103 resized_mm_data = {
104 modality: resize_mm_data(items.data, size_factors)
105 for modality, items in mm_items.items()
@@ -108,11 +108,10 @@ def create_batched_mm_kwargs(
108 # video metadata will be added back to the resized video data here.
109 text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)
110
111 - mm_kwargs = processor.apply(
111 + mm_kwargs = processor(
112 prompt=token_prompt if text_prompt is None else text_prompt,
113 mm_items=processor.info.parse_mm_data(resized_mm_data),
114 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
115 - tokenization_kwargs=processor_inputs.tokenization_kwargs,
115 )["mm_kwargs"].require_data()
116
117 return group_mm_kwargs_by_modality(
tests/models/multimodal/processing/test_transformers.py
+2 -2
@@ -19,7 +19,7 @@ def test_multimodal_processor(model_id):
19 image_pil = ImageAsset("cherry_blossom").pil_image
20 mm_data = {"image": image_pil}
21 str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
22 - str_processed_inputs = mm_processor.apply(
22 + str_processed_inputs = mm_processor(
23 prompt=str_prompt,
24 mm_items=mm_processor.info.parse_mm_data(mm_data),
25 hf_processor_mm_kwargs={},
@@ -44,7 +44,7 @@ def test_multimodal_processor(model_id):
44 77091,
45 198,
46 ]
47 - ids_processed_inputs = mm_processor.apply(
47 + ids_processed_inputs = mm_processor(
48 prompt=ids_prompt,
49 mm_items=mm_processor.info.parse_mm_data(mm_data),
50 hf_processor_mm_kwargs={},
tests/multimodal/test_processing.py
+1 -1
@@ -934,7 +934,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
934 exc_ctx = nullcontext() if is_valid else pytest.raises(ValueError, match="At most")
935
936 with exc_ctx:
937 - processor.apply(
937 + processor(
938 "<image>" * num_images,
939 mm_items=processor.info.parse_mm_data(mm_data),
940 hf_processor_mm_kwargs={},
vllm/benchmarks/mm_processor.py
+34 -42
@@ -17,8 +17,9 @@ import argparse
17 import dataclasses
18 import json
19 import time
20 +from collections import defaultdict
21 from datetime import datetime
21 -from typing import TYPE_CHECKING, Any
22 +from typing import TYPE_CHECKING, Any, Literal
23
24 import numpy as np
25
@@ -59,12 +60,13 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
60 Example:
61 {
62 'request-123': {
62 - 'hf_processor_time': 0.45,
63 - 'hashing_time': 0.02,
64 - 'cache_lookup_time': 0.01,
65 - 'prompt_update_time': 0.03,
66 - 'preprocessor_total_time': 0.51,
67 - 'encoder_forward_time': 0.23,
63 + 'get_mm_hashes_secs': 0.02,
64 + 'get_cache_missing_items_secs': 0.01,
65 + 'apply_hf_processor_secs': 0.45,
66 + 'merge_mm_kwargs_secs': 0.01,
67 + 'apply_prompt_updates_secs': 0.03,
68 + 'preprocessor_total_secs': 0.51,
69 + 'encoder_forward_secs': 0.23,
70 'num_encoder_calls': 1
71 }
72 }
@@ -74,8 +76,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
76 return {}
77
78 renderer = llm_engine.renderer
77 - mm_processor = renderer.get_mm_processor()
78 - preprocessing_stats = mm_processor.info.ctx.get_all_timing_stats()
79 + mm_processor_stats = renderer._mm_timing_registry.stat()
80
81 encoder_stats = dict[str, dict[str, float]]()
82 for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"):
@@ -88,10 +89,10 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
89 else:
90 # Aggregate timing metrics across workers
91 current_time = encoder_stats[request_id].get(
91 - "encoder_forward_time", 0.0
92 + "encoder_forward_secs", 0.0
93 )
93 - new_time = stats_dict.get("encoder_forward_time", 0.0)
94 - encoder_stats[request_id]["encoder_forward_time"] = max(
94 + new_time = stats_dict.get("encoder_forward_secs", 0.0)
95 + encoder_stats[request_id]["encoder_forward_secs"] = max(
96 current_time, new_time
97 )
98
@@ -103,7 +104,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
104
105 merged_stats = dict[str, dict[str, float]]()
106
106 - for request_id, prep_dict in preprocessing_stats.items():
107 + for request_id, prep_dict in mm_processor_stats.items():
108 merged_stats[request_id] = dict(prep_dict)
109
110 for request_id, enc_dict in encoder_stats.items():
@@ -124,34 +125,18 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f
125 return merged_stats
126
127
127 -def collect_mm_processor_stats(
128 - llm_engine: LLMEngine,
129 - num_warmup_reqs: int = 0,
130 -) -> dict[str, list[float]]:
128 +def collect_mm_processor_stats(llm_engine: LLMEngine) -> dict[str, list[float]]:
129 """
130 Collect multimodal processor timing stats.
131 Returns a dictionary mapping stage names to lists of timing values (in seconds).
132 """
133 all_stats = get_timing_stats_from_engine(llm_engine)
134
137 - stat_keys = [
138 - "hf_processor_time",
139 - "hashing_time",
140 - "cache_lookup_time",
141 - "prompt_update_time",
142 - "preprocessor_total_time",
143 - "encoder_forward_time",
144 - "num_encoder_calls",
145 - ]
146 - stats_by_stage = {key: [] for key in stat_keys}
147 -
148 - # Skip warmup requests
149 - stats_list = list(all_stats.values())[num_warmup_reqs:]
135 + stats_by_stage = defaultdict[str, list[float]](list)
136
151 - for stats_dict in stats_list:
152 - for key in stat_keys:
153 - if key in stats_dict:
154 - stats_by_stage[key].append(stats_dict[key])
137 + for stats_dict in all_stats.values():
138 + for stat_key, stat_val in stats_dict.items():
139 + stats_by_stage[stat_key].append(stat_val)
140
141 return stats_by_stage
142
@@ -159,13 +144,20 @@ def collect_mm_processor_stats(
144 def calculate_mm_processor_metrics(
145 stats_by_stage: dict[str, list[float]],
146 selected_percentiles: list[float],
147 + *,
148 + unit: Literal["us", "ms", "s"] = "ms",
149 ) -> dict[str, dict[str, float]]:
150 """
151 Calculate aggregate metrics from stats by stage.
152 """
153 + unit2mult = {"us": 1000000, "ms": 1000, "s": 1}
154 + unit_mult = unit2mult[unit]
155 +
156 metrics = {}
157
168 - for stage_name, times in stats_by_stage.items():
158 + for stage, times in stats_by_stage.items():
159 + stage_name = stage.replace("_secs", "_" + unit)
160 +
161 if not times:
162 metrics[stage_name] = {
163 "mean": 0.0,
@@ -175,8 +167,8 @@ def calculate_mm_processor_metrics(
167 }
168 continue
169
178 - is_count_metric = stage_name == "num_encoder_calls"
179 - values = times if is_count_metric else [t * 1000 for t in times]
170 + is_count_metric = stage == "num_encoder_calls"
171 + values = times if is_count_metric else [t * unit_mult for t in times]
172
173 metrics[stage_name] = {
174 "mean": float(np.mean(values)),
@@ -285,6 +277,9 @@ def benchmark_multimodal_processor(
277 use_tqdm=not getattr(args, "disable_tqdm", False),
278 )
279
280 + # Clear stats from warmup requests
281 + collect_mm_processor_stats(llm.llm_engine)
282 +
283 print(f"Processing {len(prompts)} requests...")
284 start_time = time.perf_counter()
285
@@ -295,7 +290,7 @@ def benchmark_multimodal_processor(
290 end_time = time.perf_counter()
291 total_time = end_time - start_time
292
298 - mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine, num_warmups)
293 + mm_stats_by_stage = collect_mm_processor_stats(llm.llm_engine)
294
295 if not any(mm_stats_by_stage.values()):
296 print(
@@ -475,11 +470,8 @@ def main(args: argparse.Namespace) -> None:
470 ]
471 mm_data = []
472 for stage, metrics in result["mm_processor_stats"].items():
478 - is_count = stage == "num_encoder_calls"
479 - unit = "" if is_count else " (ms)"
480 -
473 row = {
482 - "Stage": stage + unit,
474 + "Stage": stage,
475 "Mean": f"{metrics['mean']:.2f}",
476 "Median": f"{metrics['median']:.2f}",
477 "Std": f"{metrics['std']:.2f}",
vllm/model_executor/models/clip.py
+12 -20
@@ -41,15 +41,16 @@ from vllm.multimodal.parse import (
41 ImageProcessorItems,
42 ImageSize,
43 MultiModalDataItems,
44 - MultiModalUUIDItems,
44 )
45 from vllm.multimodal.processing import (
46 BaseDummyInputsBuilder,
47 BaseMultiModalProcessor,
48 BaseProcessingInfo,
49 + ProcessorInputs,
50 PromptIndexTargets,
51 PromptReplacement,
52 PromptUpdate,
53 + TimingContext,
54 )
55 from vllm.sequence import IntermediateTensors
56 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -204,23 +205,20 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
205
206 def apply(
207 self,
207 - prompt: str | list[int],
208 - mm_items: MultiModalDataItems,
209 - mm_uuid_items: MultiModalUUIDItems | None = None,
210 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
211 - tokenization_kwargs: Mapping[str, object] | None = None,
208 + inputs: ProcessorInputs,
209 + timing_ctx: TimingContext,
210 ) -> MultiModalInputs:
213 - if mm_items:
214 - if isinstance(prompt, str):
215 - if len(prompt) > 0:
211 + if inputs.mm_data_items:
212 + if isinstance(inputs.prompt, str):
213 + if len(inputs.prompt) > 0:
214 raise ValueError(
215 "CLIP accepts text-only or image-only inputs, not both! "
216 "You must pass an image with an empty text prompt."
217 )
218 else:
219 special_tokens = self.info.get_tokenizer().all_special_ids
222 - if all(tok in special_tokens for tok in prompt):
223 - prompt = []
220 + if all(tok in special_tokens for tok in inputs.prompt):
221 + inputs.prompt = []
222 else:
223 raise ValueError(
224 "CLIP accepts text-only or image-only inputs, not both! "
@@ -229,18 +227,12 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
227
228 # For multi-modal data, the prompt after processing should
229 # only contain the dummy image tokens
232 - tokenization_kwargs = {
233 - **(tokenization_kwargs or {}),
230 + inputs.tokenization_kwargs = {
231 + **inputs.tokenization_kwargs,
232 "add_special_tokens": False,
233 }
234
237 - return super().apply(
238 - prompt=prompt,
239 - mm_items=mm_items,
240 - mm_uuid_items=mm_uuid_items,
241 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
242 - tokenization_kwargs=tokenization_kwargs,
243 - )
235 + return super().apply(inputs, timing_ctx)
236
237 def _hf_processor_applies_updates(
238 self,
vllm/model_executor/models/deepseek_vl2.py
+7 -21
@@ -30,15 +30,16 @@ from vllm.multimodal.parse import (
30 ImageProcessorItems,
31 ImageSize,
32 MultiModalDataItems,
33 - MultiModalUUIDItems,
33 )
34 from vllm.multimodal.processing import BaseDummyInputsBuilder
35 from vllm.multimodal.processing.processor import (
36 BaseMultiModalProcessor,
37 BaseProcessingInfo,
38 MultiModalProcessingInfo,
39 + ProcessorInputs,
40 PromptReplacement,
41 PromptUpdate,
42 + TimingContext,
43 )
44 from vllm.sequence import IntermediateTensors
45 from vllm.tokenizers import cached_tokenizer_from_config
@@ -310,32 +311,17 @@ class DeepseekVL2MultiModalProcessor(
311
312 def _cached_apply_hf_processor(
313 self,
313 - prompt: str | list[int],
314 - mm_data_items: MultiModalDataItems,
315 - mm_uuid_items: MultiModalUUIDItems | None,
316 - hf_processor_mm_kwargs: Mapping[str, object],
317 - tokenization_kwargs: Mapping[str, object],
314 + inputs: ProcessorInputs,
315 + timing_ctx: TimingContext,
316 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
317 # The processor logic is different for len(images) <= 2 vs > 2
318 # Since the processing cache assumes that the processor output is
319 # invariant of how many images are passed per prompt, we only
320 # perform caching for the most common case
323 - if mm_data_items.get_count("image", strict=False) > 2:
324 - return self._apply_hf_processor(
325 - prompt=prompt,
326 - mm_data_items=mm_data_items,
327 - mm_uuid_items=mm_uuid_items,
328 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
329 - tokenization_kwargs=tokenization_kwargs,
330 - )
321 + if inputs.mm_data_items.get_count("image", strict=False) > 2:
322 + return self._apply_hf_processor(inputs, timing_ctx)
323
332 - return super()._cached_apply_hf_processor(
333 - prompt=prompt,
334 - mm_data_items=mm_data_items,
335 - mm_uuid_items=mm_uuid_items,
336 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
337 - tokenization_kwargs=tokenization_kwargs,
338 - )
324 + return super()._cached_apply_hf_processor(inputs, timing_ctx)
325
326
327 @MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/h2ovl.py
+7 -21
@@ -21,13 +21,14 @@ from vllm.multimodal.parse import (
21 ImageEmbeddingItems,
22 ImageProcessorItems,
23 MultiModalDataItems,
24 - MultiModalUUIDItems,
24 )
25 from vllm.multimodal.processing.processor import (
26 MultiModalProcessingInfo,
27 + ProcessorInputs,
28 PromptReplacement,
29 PromptUpdate,
30 PromptUpdateDetails,
31 + TimingContext,
32 )
33 from vllm.tokenizers import TokenizerLike
34
@@ -490,32 +491,17 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
491
492 def _cached_apply_hf_processor(
493 self,
493 - prompt: str | list[int],
494 - mm_data_items: MultiModalDataItems,
495 - mm_uuid_items: MultiModalUUIDItems | None,
496 - hf_processor_mm_kwargs: Mapping[str, object],
497 - tokenization_kwargs: Mapping[str, object],
494 + inputs: ProcessorInputs,
495 + timing_ctx: TimingContext,
496 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
497 # The processor logic is different for len(images) <= 1 vs > 1
498 # Since the processing cache assumes that the processor output is
499 # invariant of how many images are passed per prompt, we only
500 # perform caching for the most common case
503 - if mm_data_items.get_count("image", strict=False) > 1:
504 - return self._apply_hf_processor(
505 - prompt=prompt,
506 - mm_data_items=mm_data_items,
507 - mm_uuid_items=mm_uuid_items,
508 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
509 - tokenization_kwargs=tokenization_kwargs,
510 - )
501 + if inputs.mm_data_items.get_count("image", strict=False) > 1:
502 + return self._apply_hf_processor(inputs, timing_ctx)
503
512 - return super()._cached_apply_hf_processor(
513 - prompt=prompt,
514 - mm_data_items=mm_data_items,
515 - mm_uuid_items=mm_uuid_items,
516 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
517 - tokenization_kwargs=tokenization_kwargs,
518 - )
504 + return super()._cached_apply_hf_processor(inputs, timing_ctx)
505
506
507 @MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/llava.py
+8 -16
@@ -37,16 +37,17 @@ from vllm.multimodal.parse import (
37 ImageProcessorItems,
38 ImageSize,
39 MultiModalDataItems,
40 - MultiModalUUIDItems,
40 )
41 from vllm.multimodal.processing import (
42 BaseDummyInputsBuilder,
43 BaseMultiModalProcessor,
44 BaseProcessingInfo,
45 InputProcessingContext,
46 + ProcessorInputs,
47 PromptReplacement,
48 PromptUpdate,
49 PromptUpdateDetails,
50 + TimingContext,
51 )
52 from vllm.sequence import IntermediateTensors
53 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -770,11 +771,8 @@ class MantisProcessingInfo(LlavaProcessingInfo):
771 class MantisMultiModalProcessor(LlavaMultiModalProcessor):
772 def apply(
773 self,
773 - prompt: str | list[int],
774 - mm_items: MultiModalDataItems,
775 - mm_uuid_items: MultiModalUUIDItems | None = None,
776 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
777 - tokenization_kwargs: Mapping[str, object] | None = None,
774 + inputs: ProcessorInputs,
775 + timing_ctx: TimingContext,
776 ) -> MultiModalInputs:
777 hf_config = self.info.get_hf_config()
778 image_token_id = hf_config.image_token_index
@@ -785,15 +783,9 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
783 image_height=-1,
784 )
785
788 - result = super().apply(
789 - prompt,
790 - mm_items,
791 - mm_uuid_items,
792 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
793 - tokenization_kwargs=tokenization_kwargs,
794 - )
786 + result = super().apply(inputs, timing_ctx)
787
796 - mm_item_counts = mm_items.get_all_counts()
788 + mm_item_counts = inputs.mm_data_items.get_all_counts()
789 mm_kwargs = result["mm_kwargs"]
790 mm_hashes = result["mm_hashes"]
791
@@ -825,8 +817,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
817 )
818
819 orig_repls = self._get_mm_prompt_updates(
828 - mm_items,
829 - hf_processor_mm_kwargs,
820 + inputs.mm_data_items,
821 + inputs.hf_processor_mm_kwargs,
822 mm_kwargs,
823 )
824 mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls)
vllm/model_executor/models/paligemma.py
+5 -13
@@ -21,16 +21,17 @@ from vllm.multimodal.parse import (
21 ImageEmbeddingItems,
22 ImageProcessorItems,
23 MultiModalDataItems,
24 - MultiModalUUIDItems,
24 )
25 from vllm.multimodal.processing import (
26 BaseDummyInputsBuilder,
27 BaseMultiModalProcessor,
28 BaseProcessingInfo,
29 + ProcessorInputs,
30 PromptIndexTargets,
31 PromptInsertion,
32 PromptUpdate,
33 PromptUpdateDetails,
34 + TimingContext,
35 )
36 from vllm.renderers import TokenizeParams
37 from vllm.sequence import IntermediateTensors
@@ -228,19 +229,10 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
229
230 def apply(
231 self,
231 - prompt: str | list[int],
232 - mm_items: MultiModalDataItems,
233 - mm_uuid_items: MultiModalUUIDItems | None = None,
234 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
235 - tokenization_kwargs: Mapping[str, object] | None = None,
232 + inputs: ProcessorInputs,
233 + timing_ctx: TimingContext,
234 ) -> MultiModalInputs:
237 - mm_inputs = super().apply(
238 - prompt,
239 - mm_items,
240 - mm_uuid_items,
241 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
242 - tokenization_kwargs=tokenization_kwargs,
243 - )
235 + mm_inputs = super().apply(inputs, timing_ctx)
236 prompt_token_ids = mm_inputs["prompt_token_ids"]
237
238 tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/pixtral.py
+7 -20
@@ -50,16 +50,17 @@ from vllm.multimodal.parse import (
50 ImageProcessorItems,
51 ImageSize,
52 MultiModalDataItems,
53 - MultiModalUUIDItems,
53 )
55 -from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
54 +from vllm.multimodal.processing import BaseDummyInputsBuilder
55 from vllm.multimodal.processing.processor import (
56 BaseMultiModalProcessor,
57 BaseProcessingInfo,
58 MultiModalProcessingInfo,
59 + ProcessorInputs,
60 PromptReplacement,
61 PromptUpdate,
62 PromptUpdateDetails,
63 + TimingContext,
64 )
65 from vllm.platforms import current_platform
66 from vllm.sequence import IntermediateTensors
@@ -277,7 +278,6 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
278 dummy_text = self.get_dummy_text(mm_counts)
279 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
280 dummy_images = dummy_mm_data.get("image", [])
280 - tokenization_kwargs = {"truncation": False}
281
282 request = ChatCompletionRequest(
283 messages=[
@@ -294,11 +294,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
294
295 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
296
297 - return ProcessorInputs(
298 - prompt=dummy_tokens,
299 - mm_items=dummy_mm_items,
300 - tokenization_kwargs=tokenization_kwargs,
301 - )
297 + return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
298
299
300 class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
@@ -344,19 +340,10 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
340
341 def _cached_apply_hf_processor(
342 self,
347 - prompt: str | list[int],
348 - mm_data_items: MultiModalDataItems,
349 - mm_uuid_items: MultiModalUUIDItems | None,
350 - hf_processor_mm_kwargs: Mapping[str, object],
351 - tokenization_kwargs: Mapping[str, object],
343 + inputs: ProcessorInputs,
344 + timing_ctx: TimingContext,
345 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
353 - prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
354 - prompt=prompt,
355 - mm_data_items=mm_data_items,
356 - mm_uuid_items=mm_uuid_items,
357 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
358 - tokenization_kwargs=tokenization_kwargs,
359 - )
346 + prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
347
348 # NOTE: The tokens are already inserted by the chat template
349 return prompt_ids, mm_info, True
vllm/model_executor/models/siglip.py
+13 -21
@@ -47,15 +47,16 @@ from vllm.multimodal.parse import (
47 ImageProcessorItems,
48 ImageSize,
49 MultiModalDataItems,
50 - MultiModalUUIDItems,
50 )
51 from vllm.multimodal.processing import (
52 BaseDummyInputsBuilder,
53 BaseMultiModalProcessor,
54 BaseProcessingInfo,
55 + ProcessorInputs,
56 PromptIndexTargets,
57 PromptReplacement,
58 PromptUpdate,
59 + TimingContext,
60 )
61 from vllm.sequence import IntermediateTensors
62 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -190,23 +191,20 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
191
192 def apply(
193 self,
193 - prompt: str | list[int],
194 - mm_items: MultiModalDataItems,
195 - mm_uuid_items: MultiModalUUIDItems | None = None,
196 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
197 - tokenization_kwargs: Mapping[str, object] | None = None,
194 + inputs: ProcessorInputs,
195 + timing_ctx: TimingContext,
196 ) -> MultiModalInputs:
199 - if mm_items:
200 - if isinstance(prompt, str):
201 - if len(prompt) > 0:
197 + if inputs.mm_data_items:
198 + if isinstance(inputs.prompt, str):
199 + if len(inputs.prompt) > 0:
200 raise ValueError(
201 "SigLIP accepts text-only or image-only inputs, not both! "
202 "You must pass an image with an empty text prompt."
203 )
204 else:
205 special_tokens = self.info.get_tokenizer().all_special_ids
208 - if all(tok in special_tokens for tok in prompt):
209 - prompt = []
206 + if all(tok in special_tokens for tok in inputs.prompt):
207 + inputs.prompt = []
208 else:
209 raise ValueError(
210 "SigLIP accepts text-only or image-only inputs, not both! "
@@ -214,19 +212,13 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
212 )
213
214 # For multi-modal data, the prompt after processing should
217 - # only contain the image token
218 - tokenization_kwargs = {
219 - **(tokenization_kwargs or {}),
215 + # only contain the dummy image tokens
216 + inputs.tokenization_kwargs = {
217 + **inputs.tokenization_kwargs,
218 "add_special_tokens": False,
219 }
220
223 - return super().apply(
224 - prompt=prompt,
225 - mm_items=mm_items,
226 - mm_uuid_items=mm_uuid_items,
227 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
228 - tokenization_kwargs=tokenization_kwargs,
229 - )
221 + return super().apply(inputs, timing_ctx)
222
223 def _hf_processor_applies_updates(
224 self,
vllm/model_executor/models/terratorch.py
+21 -23
@@ -54,13 +54,14 @@ from vllm.multimodal.parse import (
54 ModalityDataItems,
55 MultiModalDataItems,
56 MultiModalDataParser,
57 - MultiModalUUIDItems,
57 )
58 from vllm.multimodal.processing import (
59 BaseDummyInputsBuilder,
60 BaseMultiModalProcessor,
61 BaseProcessingInfo,
62 + ProcessorInputs,
63 PromptUpdate,
64 + TimingContext,
65 )
66 from vllm.sequence import IntermediateTensors
67
@@ -193,29 +194,21 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
194
195 def apply(
196 self,
196 - prompt: str | list[int],
197 - mm_items: MultiModalDataItems,
198 - mm_uuid_items: MultiModalUUIDItems | None = None,
199 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
200 - tokenization_kwargs: Mapping[str, object] | None = None,
197 + inputs: ProcessorInputs,
198 + timing_ctx: TimingContext,
199 ) -> MultiModalInputs:
202 - if hf_processor_mm_kwargs is None:
203 - hf_processor_mm_kwargs = {}
204 - if tokenization_kwargs is None:
205 - tokenization_kwargs = {}
206 -
207 - mm_hashes = self._hash_mm_items(
208 - mm_items,
209 - mm_uuid_items,
210 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
211 - )
212 -
213 - _, passthrough_data = self._get_hf_mm_data(mm_items)
214 - mm_processed_data = BatchFeature(
215 - {k: torch.as_tensor(v).unsqueeze(0) for k, v in passthrough_data.items()},
216 - tensor_type="pt",
217 - )
218 - mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
200 + mm_items = inputs.mm_data_items
201 + hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
202 +
203 + with timing_ctx.record("apply_hf_processor"):
204 + _, passthrough_data = self._get_hf_mm_data(mm_items)
205 + mm_processed_data = BatchFeature(
206 + {
207 + k: torch.as_tensor(v).unsqueeze(0)
208 + for k, v in passthrough_data.items()
209 + },
210 + tensor_type="pt",
211 + )
212
213 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
214 mm_processed_data,
@@ -226,6 +219,11 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
219 ),
220 )
221
222 + with timing_ctx.record("get_mm_hashes"):
223 + mm_hashes = inputs.get_mm_hashes(self.info.model_id)
224 +
225 + mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
226 +
227 return mm_inputs(
228 prompt_token_ids=[1],
229 mm_kwargs=mm_kwargs,
vllm/model_executor/models/transformers/multimodal.py
+32 -37
@@ -37,12 +37,13 @@ from vllm.multimodal.inputs import (
37 from vllm.multimodal.parse import (
38 ImageProcessorItems,
39 MultiModalDataItems,
40 - MultiModalUUIDItems,
40 )
41 from vllm.multimodal.processing import (
42 BaseDummyInputsBuilder,
43 BaseMultiModalProcessor,
44 BaseProcessingInfo,
45 + ProcessorInputs,
46 + TimingContext,
47 )
48 from vllm.platforms import current_platform
49 from vllm.sequence import IntermediateTensors
@@ -177,11 +178,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
178
179 def apply(
180 self,
180 - prompt: str | list[int],
181 - mm_items: MultiModalDataItems,
182 - mm_uuid_items: MultiModalUUIDItems | None = None,
183 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
184 - tokenization_kwargs: Mapping[str, object] | None = None,
181 + inputs: ProcessorInputs,
182 + timing_ctx: TimingContext,
183 ) -> MultiModalInputs:
184 """
185 Process multi-modal inputs to be used in vLLM.
@@ -189,29 +187,30 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
187 Apply HF Processor on prompt text and multi-modal data together,
188 outputting token IDs and processed tensors.
189 """
192 - if hf_processor_mm_kwargs is None:
193 - hf_processor_mm_kwargs = {}
194 - if tokenization_kwargs is None:
195 - tokenization_kwargs = {}
196 -
197 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
198 - if not isinstance(prompt, str):
199 - # the prompt is the tokenized ids which is not supported
200 - # by the hf_processor, which is why we would need to decode the ids
201 - # into string
202 - prompt = hf_processor.decode(prompt)
203 -
204 - # Bypass cached processor and always apply to the full set of mm inputs
205 - # NOTE: we can't just set caching=False because base class method
206 - # transforms outputs to `MultiModalKwargs` which is not going to
207 - # work for Transformers. We have a lot of logic tied to
208 - # `mm_tokens_per_modality` below
209 - prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
210 - prompt_text=prompt,
211 - mm_items=mm_items,
212 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
213 - tokenization_kwargs=tokenization_kwargs,
214 - )
190 + prompt = inputs.prompt
191 + mm_items = inputs.mm_data_items
192 + hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
193 + tokenization_kwargs = inputs.tokenization_kwargs
194 +
195 + with timing_ctx.record("apply_hf_processor"):
196 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
197 + if not isinstance(prompt, str):
198 + # the prompt is the tokenized ids which is not supported
199 + # by the hf_processor, which is why we would need to decode the ids
200 + # into string
201 + prompt = hf_processor.decode(prompt)
202 +
203 + # Bypass cached processor and always apply to the full set of mm inputs
204 + # NOTE: we can't just set caching=False because base class method
205 + # transforms outputs to `MultiModalKwargs` which is not going to
206 + # work for Transformers. We have a lot of logic tied to
207 + # `mm_tokens_per_modality` below
208 + prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
209 + prompt_text=prompt,
210 + mm_items=mm_items,
211 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
212 + tokenization_kwargs=tokenization_kwargs,
213 + )
214
215 # For gemma3 we check `token_type_ids` as the key
216 token_type_key = (
@@ -225,15 +224,14 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
224 # it for each input `mm_data`.
225 mm_positions = torch.where(mm_token_type_ids == 1)[1]
226 images = mm_items.get_items("image", ImageProcessorItems)
228 - multimodal_config = self.info.ctx.model_config.multimodal_config
229 - mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
227 image_sizes = []
228 for item_idx in range(len(images)):
229 image_size = images.get_image_size(item_idx)
230 image_sizes.append((image_size.height, image_size.width))
231
232 mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
236 - image_sizes=image_sizes, **mm_processor_kwargs
233 + image_sizes=image_sizes,
234 + **self.info.ctx.get_merged_mm_kwargs({}),
235 )
236
237 mm_placeholders = {}
@@ -261,11 +259,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
259 )
260
261 # Use overrides if provided; fallback to data-dependent hashing.
264 - mm_hashes = self._hash_mm_items(
265 - mm_items,
266 - mm_uuid_items,
267 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
268 - )
262 + with timing_ctx.record("get_mm_hashes"):
263 + mm_hashes = inputs.get_mm_hashes(self.info.model_id)
264
265 return mm_inputs(
266 prompt_token_ids=prompt_ids,
vllm/model_executor/models/voxtral.py
+8 -16
@@ -47,16 +47,17 @@ from vllm.multimodal.parse import (
47 AudioProcessorItems,
48 MultiModalDataItems,
49 MultiModalDataParser,
50 - MultiModalUUIDItems,
50 )
52 -from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
51 +from vllm.multimodal.processing import BaseDummyInputsBuilder
52 from vllm.multimodal.processing.processor import (
53 BaseMultiModalProcessor,
54 BaseProcessingInfo,
55 MultiModalProcessingInfo,
56 PlaceholderFeaturesInfo,
57 + ProcessorInputs,
58 PromptReplacement,
59 PromptUpdate,
60 + TimingContext,
61 )
62 from vllm.sequence import IntermediateTensors
63 from vllm.tokenizers import cached_tokenizer_from_config
@@ -265,13 +266,13 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
266 res = tokenizer.mistral.encode_chat_completion(request)
267 dummy_tokens = res.tokens
268
268 - dummy_mm_inputs = self.info.parse_mm_data(
269 + dummy_mm_items = self.info.parse_mm_data(
270 # whixtral tokenizer adds padding to the audio
271 # so we need to update the audio arrays
272 {**dummy_mm_data, "audio": [a.audio_array for a in res.audios]},
273 )
274
274 - return ProcessorInputs(prompt=dummy_tokens, mm_items=dummy_mm_inputs)
275 + return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
276
277
278 class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
@@ -361,19 +362,10 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
362
363 def _cached_apply_hf_processor(
364 self,
364 - prompt: str | list[int],
365 - mm_data_items: MultiModalDataItems,
366 - mm_uuid_items: MultiModalUUIDItems | None,
367 - hf_processor_mm_kwargs: Mapping[str, object],
368 - tokenization_kwargs: Mapping[str, object],
365 + inputs: ProcessorInputs,
366 + timing_ctx: TimingContext,
367 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
370 - prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
371 - prompt=prompt,
372 - mm_data_items=mm_data_items,
373 - mm_uuid_items=mm_uuid_items,
374 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
375 - tokenization_kwargs=tokenization_kwargs,
376 - )
368 + prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(inputs, timing_ctx)
369
370 # NOTE: The tokens are already inserted by the chat template
371 return prompt_ids, mm_info, True
vllm/multimodal/processing/__init__.py
+4 -2
@@ -1,7 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from .context import BaseProcessingInfo, InputProcessingContext
4 -from .dummy_inputs import BaseDummyInputsBuilder, ProcessorInputs
3 +from .context import BaseProcessingInfo, InputProcessingContext, TimingContext
4 +from .dummy_inputs import BaseDummyInputsBuilder
5 +from .inputs import ProcessorInputs
6 from .processor import (
7 BaseMultiModalProcessor,
8 EncDecMultiModalProcessor,
@@ -15,6 +16,7 @@ from .processor import (
16 __all__ = [
17 "BaseProcessingInfo",
18 "InputProcessingContext",
19 + "TimingContext",
20 "BaseDummyInputsBuilder",
21 "ProcessorInputs",
22 "BaseMultiModalProcessor",
vllm/multimodal/processing/context.py
+30 -163
@@ -1,10 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import contextvars
4 -import threading
3 import time
4 from abc import abstractmethod
7 -from collections.abc import Generator, Mapping
5 +from collections.abc import Mapping
6 from contextlib import contextmanager
7 from dataclasses import dataclass, field
8 from functools import cached_property
@@ -33,104 +31,53 @@ if TYPE_CHECKING:
31 from transformers.feature_extraction_utils import BatchFeature
32 from transformers.processing_utils import ProcessorMixin
33
36 - from vllm.config import ModelConfig, ObservabilityConfig
34 + from vllm.config import ModelConfig
35 else:
36 PretrainedConfig = object
37 BatchFeature = object
38 ProcessorMixin = object
39
40 ModelConfig = object
43 - ObservabilityConfig = object
41
42 logger = init_logger(__name__)
43
44
48 -_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
49 - "_request_id_context", default=None
50 -)
51 -
52 -
53 -def get_current_request_id() -> str | None:
54 - """Get the current request_id from the context, if available."""
55 - return _request_id_context.get()
56 -
57 -
58 -@contextmanager
59 -def set_request_id(request_id: str) -> Generator[None, None, None]:
60 - """Context manager to set the request_id for the current context."""
61 - token = _request_id_context.set(request_id)
62 - try:
63 - yield
64 - finally:
65 - _request_id_context.reset(token)
66 -
67 -
45 @dataclass
69 -class MultiModalProcessorTimingStats:
70 - """Per-request timing statistics for multimodal processor stages."""
71 -
72 - hf_processor_time: float = 0.0
73 - """Time spent in HuggingFace processor calls (seconds)."""
46 +class TimingContext:
47 + """Helper class to record execution times during multi-modal processing."""
48
75 - hashing_time: float = 0.0
76 - """Time spent computing multimodal item hashes (seconds)."""
49 + enabled: bool = True
50 + """If disabled, `TimingContext.record` becomes a no-op."""
51
78 - cache_lookup_time: float = 0.0
79 - """Time spent in cache lookups and merges (seconds)."""
52 + stage_secs: dict[str, float] = field(default_factory=dict)
53 + """The execution time (in seconds) for each processing stage."""
54
81 - prompt_update_time: float = 0.0
82 - """Time spent applying prompt updates and finding placeholders (seconds)."""
55 + @property
56 + def total_secs(self) -> float:
57 + return sum(self.stage_secs.values())
58
84 - preprocessor_total_time: float = 0.0
85 - """Total preprocessing time (seconds)."""
59 + @contextmanager
60 + def record(self, stage: str):
61 + """Record the execution time for a processing stage."""
62 + if not self.enabled:
63 + yield
64 + return
65
87 - def to_dict(self) -> dict[str, float]:
88 - """Convert stats to a dictionary for JSON serialization."""
89 - return {
90 - "hf_processor_time": self.hf_processor_time,
91 - "hashing_time": self.hashing_time,
92 - "cache_lookup_time": self.cache_lookup_time,
93 - "prompt_update_time": self.prompt_update_time,
94 - "preprocessor_total_time": self.preprocessor_total_time,
66 + start_time = time.perf_counter()
67 + try:
68 + yield
69 + finally:
70 + elapsed = time.perf_counter() - start_time
71 + self.stage_secs.setdefault(stage, 0.0)
72 + self.stage_secs[stage] += elapsed
73 +
74 + def get_stats_dict(self):
75 + stats_dict = {
76 + f"{stage}_secs": time_s for stage, time_s in self.stage_secs.items()
77 }
78 + stats_dict["preprocessor_total_secs"] = self.total_secs
79
97 -
98 -@contextmanager
99 -def timed_preprocessor_operation(ctx: "InputProcessingContext", stage_name: str):
100 - """
101 - Context manager to time an operation using the context's timing stats.
102 -
103 - The request_id is automatically retrieved from the context variable,
104 - so it doesn't need to be passed as a parameter.
105 -
106 - Args:
107 - ctx: The InputProcessingContext containing the timing stats registry.
108 - stage_name: Name of the stage being timed.
109 - """
110 - request_id = get_current_request_id()
111 - if ctx is None or request_id is None:
112 - yield
113 - return
114 -
115 - stats = ctx.get_timing_stats(request_id)
116 - if stats is None:
117 - yield
118 - return
119 -
120 - start_time = time.perf_counter()
121 - try:
122 - yield
123 - finally:
124 - elapsed = time.perf_counter() - start_time
125 - if stage_name == "hf_processor":
126 - stats.hf_processor_time += elapsed
127 - elif stage_name == "hashing":
128 - stats.hashing_time += elapsed
129 - elif stage_name == "cache_lookup":
130 - stats.cache_lookup_time += elapsed
131 - elif stage_name == "prompt_update":
132 - stats.prompt_update_time += elapsed
133 - stats.preprocessor_total_time += elapsed
80 + return stats_dict
81
82
83 _T = TypeVar("_T")
@@ -151,21 +98,6 @@ class InputProcessingContext:
98 tokenizer: TokenizerLike | None
99 """The tokenizer used to tokenize the inputs."""
100
154 - observability_config: "ObservabilityConfig | None" = field(
155 - default=None, compare=False, repr=False
156 - )
157 - """Configuration for observability features."""
158 -
159 - timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
160 - default_factory=dict, compare=False, repr=False
161 - )
162 - """Registry for storing timing stats keyed by request_id."""
163 -
164 - _timing_stats_registry_lock: threading.Lock = field(
165 - default_factory=threading.Lock, compare=False, repr=False
166 - )
167 - """Lock for thread-safe access to timing_stats_registry."""
168 -
101 def get_tokenizer(self) -> TokenizerLike:
102 if self.tokenizer is None:
103 raise ValueError(
@@ -379,71 +311,6 @@ class InputProcessingContext:
311
312 return self._postprocess_output(output)
313
382 - def get_timing_stats(
383 - self, request_id: str
384 - ) -> MultiModalProcessorTimingStats | None:
385 - """
386 - Get timing stats for a request.
387 - """
388 - if (
389 - self.observability_config is None
390 - or not self.observability_config.enable_mm_processor_stats
391 - ):
392 - return None
393 - with self._timing_stats_registry_lock:
394 - return self.timing_stats_registry.get(request_id)
395 -
396 - def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
397 - """
398 - Create and store timing stats in the registry for a request.
399 -
400 - This should be called at the start of processing for a request.
401 - The stats object is created immediately and stored in the registry.
402 - """
403 - if (
404 - self.observability_config is None
405 - or not self.observability_config.enable_mm_processor_stats
406 - ):
407 - return MultiModalProcessorTimingStats()
408 -
409 - with self._timing_stats_registry_lock:
410 - if request_id in self.timing_stats_registry:
411 - raise ValueError(
412 - f"Timing stats already exist for request_id: {request_id}"
413 - )
414 - stats = MultiModalProcessorTimingStats()
415 - self.timing_stats_registry[request_id] = stats
416 - return stats
417 -
418 - def clear_timing_stats_registry(self) -> int:
419 - """
420 - Clear all stats from the registry. Returns the number of stats cleared.
421 - """
422 - if (
423 - self.observability_config is None
424 - or not self.observability_config.enable_mm_processor_stats
425 - ):
426 - return 0
427 - with self._timing_stats_registry_lock:
428 - count = len(self.timing_stats_registry)
429 - self.timing_stats_registry.clear()
430 - return count
431 -
432 - def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
433 - """
434 - Get all timing stats as a dictionary for API endpoints.
435 - """
436 - if (
437 - self.observability_config is None
438 - or not self.observability_config.enable_mm_processor_stats
439 - ):
440 - return {}
441 - with self._timing_stats_registry_lock:
442 - return {
443 - rid: stats.to_dict()
444 - for rid, stats in self.timing_stats_registry.items()
445 - }
446 -
314
315 class BaseProcessingInfo:
316 """Base class to provide the information necessary for data processing."""
vllm/multimodal/processing/dummy_inputs.py
+2 -16
@@ -2,7 +2,6 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from abc import ABC, abstractmethod
4 from collections.abc import Mapping
5 -from dataclasses import dataclass, field
5 from typing import Generic, TypeVar
6
7 import numpy as np
@@ -18,27 +17,14 @@ from vllm.config.multimodal import (
17 from vllm.logger import init_logger
18
19 from ..inputs import MultiModalDataDict
21 -from ..parse import MultiModalDataItems
20 from .context import BaseProcessingInfo
21 +from .inputs import ProcessorInputs
22
23 _I = TypeVar("_I", bound=BaseProcessingInfo)
24
25 logger = init_logger(__name__)
26
27
29 -@dataclass
30 -class ProcessorInputs:
31 - """
32 - Represents the keyword arguments to
33 - [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
34 - """
35 -
36 - prompt: str | list[int]
37 - mm_items: MultiModalDataItems
38 - hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
39 - tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
40 -
41 -
28 class BaseDummyInputsBuilder(ABC, Generic[_I]):
29 """
30 Abstract base class that constructs the dummy data to profile
@@ -101,7 +87,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
87
88 return ProcessorInputs(
89 prompt=dummy_text,
104 - mm_items=dummy_mm_items,
90 + mm_data_items=dummy_mm_items,
91 tokenization_kwargs=tokenization_kwargs,
92 )
93
vllm/multimodal/processing/inputs.py new
+70
@@ -0,0 +1,70 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from collections.abc import Mapping
4 +from dataclasses import dataclass, field
5 +
6 +from ..hasher import MultiModalHasher
7 +from ..inputs import MultiModalHashes
8 +from ..parse import MultiModalDataItems, MultiModalUUIDItems
9 +
10 +
11 +@dataclass
12 +class ProcessorInputs:
13 + """
14 + Represents the keyword arguments to
15 + [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
16 + """
17 +
18 + prompt: str | list[int]
19 + mm_data_items: MultiModalDataItems
20 + mm_uuid_items: MultiModalUUIDItems | None = None
21 + hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
22 + tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
23 +
24 + def get_mm_hashes(self, model_id: str) -> MultiModalHashes:
25 + mm_data_items = self.mm_data_items
26 + mm_uuid_items = self.mm_uuid_items or {}
27 + hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
28 +
29 + mm_hashes: MultiModalHashes = {}
30 + hasher = MultiModalHasher
31 +
32 + for modality, data_items in mm_data_items.items():
33 + if modality in mm_uuid_items:
34 + uuid_items = mm_uuid_items[modality]
35 +
36 + # For None entries, compute a hash; otherwise, use provided ID.
37 + hashes: list[str] = []
38 + for i, item in enumerate(data_items.get_all_items_for_hash()):
39 + uuid_item = uuid_items[i]
40 +
41 + # NOTE: Even if a uuid_item is provided, we still compute a hash
42 + # if `hf_processor_mm_kwargs` is provided.
43 + # This is because the processed multimodal inputs can be different
44 + # depending on the processor kwargs.
45 + if uuid_item is None or hf_processor_mm_kwargs:
46 + # NOTE: use provided hash string to hash with kwargs
47 + # if available for better performance.
48 + item = uuid_item if uuid_item is not None else item
49 + hashes.append(
50 + hasher.hash_kwargs(
51 + model_id=model_id,
52 + **{modality: item},
53 + **hf_processor_mm_kwargs,
54 + )
55 + )
56 + else:
57 + hashes.append(uuid_item)
58 +
59 + mm_hashes[modality] = hashes
60 + else:
61 + mm_hashes[modality] = [
62 + hasher.hash_kwargs(
63 + model_id=model_id,
64 + **{modality: item},
65 + **hf_processor_mm_kwargs,
66 + )
67 + for item in data_items
68 + ]
69 +
70 + return mm_hashes
vllm/multimodal/processing/processor.py
+75 -164
@@ -23,7 +23,6 @@ from vllm.logger import init_logger
23 from vllm.tokenizers import TokenizerLike
24 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
25
26 -from ..hasher import MultiModalHasher
26 from ..inputs import (
27 MultiModalEncDecInputs,
28 MultiModalFieldConfig,
@@ -42,12 +41,9 @@ from ..parse import (
41 MultiModalDataItems,
42 MultiModalUUIDItems,
43 )
45 -from .context import (
46 - BaseProcessingInfo,
47 - get_current_request_id,
48 - timed_preprocessor_operation,
49 -)
44 +from .context import BaseProcessingInfo, TimingContext
45 from .dummy_inputs import BaseDummyInputsBuilder
46 +from .inputs import ProcessorInputs
47
48 if TYPE_CHECKING:
49 from transformers.feature_extraction_utils import BatchFeature
@@ -1017,13 +1013,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1013 mm_uuid_items: MultiModalUUIDItems | None = None,
1014 hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1015 ) -> MultiModalInputs:
1020 - return self.apply(
1016 + processor_inputs = ProcessorInputs(
1017 prompt,
1018 mm_items,
1019 mm_uuid_items,
1024 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1020 + hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
1021 )
1022
1023 + return self.apply(processor_inputs, TimingContext(enabled=False))
1024 +
1025 @abstractmethod
1026 def _get_mm_fields_config(
1027 self,
@@ -1139,12 +1137,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1137 Call the HF processor on the prompt text and
1138 associated multi-modal data.
1139 """
1142 - with timed_preprocessor_operation(self.info.ctx, "hf_processor"):
1143 - return self.info.ctx.call_hf_processor(
1144 - self.info.get_hf_processor(**mm_kwargs),
1145 - dict(text=prompt, **mm_data),
1146 - dict(**mm_kwargs, **tok_kwargs),
1147 - )
1140 + return self.info.ctx.call_hf_processor(
1141 + self.info.get_hf_processor(**mm_kwargs),
1142 + dict(text=prompt, **mm_data),
1143 + dict(**mm_kwargs, **tok_kwargs),
1144 + )
1145
1146 def _hf_processor_applies_updates(
1147 self,
@@ -1306,60 +1303,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1303
1304 return prompt_ids, mm_processed_data, False
1305
1309 - def _hash_mm_items(
1310 - self,
1311 - mm_data_items: MultiModalDataItems,
1312 - mm_uuid_items: MultiModalUUIDItems | None,
1313 - hf_processor_mm_kwargs: Mapping[str, object],
1314 - ) -> MultiModalHashes:
1315 - model_id = self.info.model_id
1316 -
1317 - if mm_uuid_items is None:
1318 - mm_uuid_items = {}
1319 -
1320 - mm_hashes: MultiModalHashes = {}
1321 - hasher = MultiModalHasher
1322 -
1323 - for modality, data_items in mm_data_items.items():
1324 - if modality in mm_uuid_items:
1325 - uuid_items = mm_uuid_items[modality]
1326 -
1327 - # For None entries, compute a hash; otherwise, use provided ID.
1328 - hashes: list[str] = []
1329 - for i, item in enumerate(data_items.get_all_items_for_hash()):
1330 - uuid_item = uuid_items[i]
1331 -
1332 - # NOTE: Even if a uuid_item is provided, we still compute a hash
1333 - # if `hf_processor_mm_kwargs` is provided.
1334 - # This is because the processed multimodal inputs can be different
1335 - # depending on the processor kwargs.
1336 - if uuid_item is None or hf_processor_mm_kwargs:
1337 - # NOTE: use provided hash string to hash with kwargs
1338 - # if available for better performance.
1339 - item = uuid_item if uuid_item is not None else item
1340 - hashes.append(
1341 - hasher.hash_kwargs(
1342 - model_id=model_id,
1343 - **{modality: item},
1344 - **hf_processor_mm_kwargs,
1345 - )
1346 - )
1347 - else:
1348 - hashes.append(uuid_item)
1349 -
1350 - mm_hashes[modality] = hashes
1351 - else:
1352 - mm_hashes[modality] = [
1353 - hasher.hash_kwargs(
1354 - model_id=model_id,
1355 - **{modality: item},
1356 - **hf_processor_mm_kwargs,
1357 - )
1358 - for item in data_items
1359 - ]
1360 -
1361 - return mm_hashes
1362 -
1306 def _get_cache_missing_items(
1307 self,
1308 cache: BaseMultiModalProcessorCache,
@@ -1461,40 +1404,36 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1404
1405 def _apply_hf_processor(
1406 self,
1464 - prompt: str | list[int],
1465 - mm_data_items: MultiModalDataItems,
1466 - mm_uuid_items: MultiModalUUIDItems | None,
1467 - hf_processor_mm_kwargs: Mapping[str, object],
1468 - tokenization_kwargs: Mapping[str, object],
1407 + inputs: ProcessorInputs,
1408 + timing_ctx: TimingContext,
1409 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1470 - (
1471 - prompt_ids,
1472 - mm_processed_data,
1473 - is_update_applied,
1474 - ) = self._apply_hf_processor_main(
1475 - prompt=prompt,
1476 - mm_items=mm_data_items,
1477 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1478 - tokenization_kwargs=tokenization_kwargs,
1479 - enable_hf_prompt_update=True,
1480 - )
1410 + with timing_ctx.record("apply_hf_processor"):
1411 + (
1412 + prompt_ids,
1413 + mm_processed_data,
1414 + is_update_applied,
1415 + ) = self._apply_hf_processor_main(
1416 + prompt=inputs.prompt,
1417 + mm_items=inputs.mm_data_items,
1418 + hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
1419 + tokenization_kwargs=inputs.tokenization_kwargs,
1420 + enable_hf_prompt_update=True,
1421 + )
1422
1423 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
1424 mm_processed_data,
1484 - self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
1425 + self._get_mm_fields_config(
1426 + mm_processed_data, inputs.hf_processor_mm_kwargs
1427 + ),
1428 )
1429
1430 # Use overrides if provided; fallback to data-dependent hashing.
1488 - with timed_preprocessor_operation(self.info.ctx, "hashing"):
1489 - mm_hashes = self._hash_mm_items(
1490 - mm_data_items,
1491 - mm_uuid_items,
1492 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1493 - )
1431 + with timing_ctx.record("get_mm_hashes"):
1432 + mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1433
1434 mm_prompt_updates = self._get_mm_prompt_updates(
1496 - mm_data_items,
1497 - hf_processor_mm_kwargs,
1435 + inputs.mm_data_items,
1436 + inputs.hf_processor_mm_kwargs,
1437 mm_kwargs,
1438 )
1439
@@ -1508,11 +1447,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1447
1448 def _cached_apply_hf_processor(
1449 self,
1511 - prompt: str | list[int],
1512 - mm_data_items: MultiModalDataItems,
1513 - mm_uuid_items: MultiModalUUIDItems | None,
1514 - hf_processor_mm_kwargs: Mapping[str, object],
1515 - tokenization_kwargs: Mapping[str, object],
1450 + inputs: ProcessorInputs,
1451 + timing_ctx: TimingContext,
1452 ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1453 """
1454 Apply the HF processor on the full prompt text,
@@ -1520,59 +1456,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1456 """
1457 cache = self.cache
1458
1523 - _, passthrough_data = self._get_hf_mm_data(mm_data_items)
1459 + _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
1460 if cache is None or passthrough_data:
1525 - return self._apply_hf_processor(
1526 - prompt=prompt,
1527 - mm_data_items=mm_data_items,
1528 - mm_uuid_items=mm_uuid_items,
1529 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1530 - tokenization_kwargs=tokenization_kwargs,
1531 - )
1461 + return self._apply_hf_processor(inputs, timing_ctx)
1462
1533 - with timed_preprocessor_operation(self.info.ctx, "hashing"):
1534 - mm_hashes = self._hash_mm_items(
1535 - mm_data_items,
1536 - mm_uuid_items,
1537 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1538 - )
1463 + with timing_ctx.record("get_mm_hashes"):
1464 + mm_hashes = inputs.get_mm_hashes(self.info.model_id)
1465
1540 - with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1466 + with timing_ctx.record("get_cache_missing_items"):
1467 mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
1468 cache=cache,
1543 - mm_data_items=mm_data_items,
1469 + mm_data_items=inputs.mm_data_items,
1470 mm_hashes=mm_hashes,
1471 )
1472
1473 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1474 # so we can't apply prompt updates until the new multimodal
1475 # items are combined with the cached multimodal items
1550 - (
1551 - prompt_ids,
1552 - mm_missing_processed_data,
1553 - is_update_applied,
1554 - ) = self._apply_hf_processor_main(
1555 - prompt=prompt,
1556 - mm_items=mm_missing_data_items,
1557 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1558 - tokenization_kwargs=tokenization_kwargs,
1559 - enable_hf_prompt_update=False,
1560 - )
1476 + with timing_ctx.record("apply_hf_processor"):
1477 + (
1478 + prompt_ids,
1479 + mm_missing_processed_data,
1480 + is_update_applied,
1481 + ) = self._apply_hf_processor_main(
1482 + prompt=inputs.prompt,
1483 + mm_items=mm_missing_data_items,
1484 + hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
1485 + tokenization_kwargs=inputs.tokenization_kwargs,
1486 + enable_hf_prompt_update=False,
1487 + )
1488
1489 mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
1490 mm_missing_processed_data,
1491 self._get_mm_fields_config(
1565 - mm_missing_processed_data, hf_processor_mm_kwargs
1492 + mm_missing_processed_data, inputs.hf_processor_mm_kwargs
1493 ),
1494 )
1495
1496 mm_missing_prompt_updates = self._get_mm_prompt_updates(
1497 mm_missing_data_items,
1571 - hf_processor_mm_kwargs,
1498 + inputs.hf_processor_mm_kwargs,
1499 mm_missing_kwargs,
1500 )
1501
1575 - with timed_preprocessor_operation(self.info.ctx, "cache_lookup"):
1502 + with timing_ctx.record("merge_mm_kwargs"):
1503 mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1504 cache,
1505 mm_hashes=mm_hashes,
@@ -1742,11 +1669,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1669
1670 def apply(
1671 self,
1745 - prompt: str | list[int],
1746 - mm_items: MultiModalDataItems,
1747 - mm_uuid_items: MultiModalUUIDItems | None = None,
1748 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1749 - tokenization_kwargs: Mapping[str, object] | None = None,
1672 + inputs: ProcessorInputs,
1673 + timing_ctx: TimingContext,
1674 ) -> MultiModalInputs:
1675 """
1676 Process multi-modal inputs to be used in vLLM.
@@ -1761,31 +1685,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1685 3. Extract information about the placeholder tokens from the
1686 processed token IDs.
1687 """
1764 - request_id = get_current_request_id()
1765 - if request_id is not None:
1766 - self.info.ctx.create_timing_stats(request_id)
1767 -
1768 - if hf_processor_mm_kwargs is None:
1769 - hf_processor_mm_kwargs = {}
1770 - if tokenization_kwargs is None:
1771 - tokenization_kwargs = {}
1772 -
1688 (
1689 prompt_ids,
1690 mm_info,
1691 is_update_applied,
1777 - ) = self._cached_apply_hf_processor(
1778 - prompt,
1779 - mm_items,
1780 - mm_uuid_items,
1781 - hf_processor_mm_kwargs,
1782 - tokenization_kwargs=tokenization_kwargs,
1783 - )
1692 + ) = self._cached_apply_hf_processor(inputs, timing_ctx)
1693
1694 # NOTE: tokenization_kwargs are not required to init processor
1786 - with timed_preprocessor_operation(self.info.ctx, "prompt_update"):
1695 + with timing_ctx.record("apply_prompt_updates"):
1696 prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1788 - mm_items=mm_items,
1697 + mm_items=inputs.mm_data_items,
1698 prompt_ids=prompt_ids,
1699 mm_kwargs=mm_info.kwargs,
1700 mm_prompt_updates=mm_info.prompt_updates,
@@ -1851,11 +1760,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1760
1761 def apply(
1762 self,
1854 - prompt: str | list[int],
1855 - mm_items: MultiModalDataItems,
1856 - mm_uuid_items: MultiModalUUIDItems | None = None,
1857 - hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1858 - tokenization_kwargs: Mapping[str, object] | None = None,
1763 + inputs: ProcessorInputs,
1764 + timing_ctx: TimingContext,
1765 ) -> MultiModalEncDecInputs:
1766 """
1767 Process multi-modal inputs to be used in vLLM.
@@ -1864,17 +1770,22 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1770 2. Apply the HF processor on encoder prompt.
1771 3. Copy the input prompt text as decoder prompt inputs.
1772 """
1867 - encoder_prompt = self.create_encoder_prompt(prompt, mm_items)
1868 - encoder_inputs = super().apply(
1773 + encoder_prompt = self.create_encoder_prompt(
1774 + inputs.prompt,
1775 + inputs.mm_data_items,
1776 + )
1777 + encoder_processor_inputs = ProcessorInputs(
1778 encoder_prompt,
1870 - mm_items,
1871 - mm_uuid_items,
1872 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1873 - tokenization_kwargs=tokenization_kwargs,
1779 + inputs.mm_data_items,
1780 + inputs.mm_uuid_items,
1781 + hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
1782 + tokenization_kwargs=inputs.tokenization_kwargs,
1783 )
1784
1785 + encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
1786 +
1787 return self._get_enc_dec_inputs(
1877 - prompt=prompt,
1878 - mm_items=mm_items,
1788 + prompt=inputs.prompt,
1789 + mm_items=inputs.mm_data_items,
1790 encoder_inputs=encoder_inputs,
1791 )
vllm/multimodal/registry.py
+39 -14
@@ -1,11 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import threading
4 +from collections import defaultdict
5 from collections.abc import Mapping
6 from dataclasses import dataclass
7 from multiprocessing.synchronize import Lock as LockType
8 from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
9
8 -from vllm.config.observability import ObservabilityConfig
10 from vllm.logger import init_logger
11 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
12
@@ -24,6 +25,7 @@ from .processing import (
25 BaseMultiModalProcessor,
26 BaseProcessingInfo,
27 InputProcessingContext,
28 + TimingContext,
29 )
30
31 if TYPE_CHECKING:
@@ -174,32 +176,26 @@ class MultiModalRegistry:
176 def _create_processing_ctx(
177 self,
178 model_config: "ModelConfig",
177 - observability_config: "ObservabilityConfig | None" = None,
179 tokenizer: TokenizerLike | None = None,
180 ) -> InputProcessingContext:
181 if tokenizer is None:
182 tokenizer = cached_tokenizer_from_config(model_config)
183
183 - return InputProcessingContext(
184 - model_config, tokenizer, observability_config=observability_config
185 - )
184 + return InputProcessingContext(model_config, tokenizer)
185
186 def _create_processing_info(
187 self,
188 model_config: "ModelConfig",
190 - observability_config: "ObservabilityConfig | None" = None,
191 - *,
189 tokenizer: TokenizerLike | None = None,
190 ) -> BaseProcessingInfo:
191 model_cls = self._get_model_cls(model_config)
192 factories = model_cls._processor_factory
196 - ctx = self._create_processing_ctx(model_config, observability_config, tokenizer)
193 + ctx = self._create_processing_ctx(model_config, tokenizer)
194 return factories.info(ctx)
195
196 def create_processor(
197 self,
198 model_config: "ModelConfig",
202 - observability_config: "ObservabilityConfig | None" = None,
199 *,
200 tokenizer: TokenizerLike | None = None,
201 cache: BaseMultiModalProcessorCache | None = None,
@@ -213,7 +209,7 @@ class MultiModalRegistry:
209 model_cls = self._get_model_cls(model_config)
210 factories = model_cls._processor_factory
211
216 - ctx = self._create_processing_ctx(model_config, observability_config, tokenizer)
212 + ctx = self._create_processing_ctx(model_config, tokenizer)
213
214 return factories.build_processor(ctx, cache=cache)
215
@@ -242,10 +238,8 @@ class MultiModalRegistry:
238 mm_options=mm_config.limit_per_prompt,
239 )
240 mm_inputs = processor.apply(
245 - prompt=processor_inputs.prompt,
246 - mm_items=processor_inputs.mm_items,
247 - hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
248 - tokenization_kwargs=processor_inputs.tokenization_kwargs,
241 + processor_inputs,
242 + timing_ctx=TimingContext(enabled=False),
243 )
244
245 prompt_token_ids = mm_inputs["prompt_token_ids"]
@@ -335,3 +329,34 @@ class MultiModalRegistry:
329 return ShmObjectStoreReceiverCache(vllm_config, shared_worker_lock)
330 else:
331 raise ValueError(f"Unknown cache type: {cache_type!r}")
332 +
333 +
334 +class MultiModalTimingRegistry:
335 + def __init__(self, observability_config: "ObservabilityConfig | None") -> None:
336 + super().__init__()
337 +
338 + if observability_config and observability_config.enable_mm_processor_stats:
339 + self._lock = threading.Lock()
340 + self._ctx_by_request_id = defaultdict[str, TimingContext](TimingContext)
341 + self._enabled = True
342 + else:
343 + self._enabled = False
344 +
345 + def get(self, request_id: str) -> TimingContext:
346 + if not self._enabled:
347 + return TimingContext(enabled=False)
348 +
349 + with self._lock:
350 + return self._ctx_by_request_id[request_id]
351 +
352 + def stat(self) -> dict[str, dict[str, float]]:
353 + if not self._enabled:
354 + return {}
355 +
356 + with self._lock:
357 + stats = {
358 + req_id: ctx.get_stats_dict()
359 + for req_id, ctx in self._ctx_by_request_id.items()
360 + }
361 + self._ctx_by_request_id.clear()
362 + return stats
vllm/renderers/base.py
+17 -11
@@ -85,13 +85,13 @@ class BaseRenderer(ABC, Generic[_T]):
85 self._mm_cache_stats: MultiModalCacheStats | None = None
86 if config.model_config.is_multimodal_model:
87 from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
88 + from vllm.multimodal.registry import MultiModalTimingRegistry
89
90 mm_processor_cache = mm_registry.processor_cache_from_config(config)
91
92 with set_default_torch_num_threads():
93 self.mm_processor = mm_registry.create_processor(
94 config.model_config,
94 - config.observability_config,
95 tokenizer=tokenizer,
96 cache=mm_processor_cache,
97 )
@@ -102,6 +102,9 @@ class BaseRenderer(ABC, Generic[_T]):
102 # This is used to generate internal request ID for MM processing
103 # It has no relation to the request ID for engine core
104 self._mm_req_counter = AtomicCounter()
105 + self._mm_timing_registry = MultiModalTimingRegistry(
106 + config.observability_config
107 + )
108
109 def get_tokenizer(self) -> _T:
110 tokenizer = self.tokenizer
@@ -534,7 +537,7 @@ class BaseRenderer(ABC, Generic[_T]):
537 tokenization_kwargs: dict[str, Any] | None,
538 ) -> "MultiModalInputs":
539 from vllm.multimodal.parse import parse_mm_uuids
537 - from vllm.multimodal.processing.context import set_request_id
540 + from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
541
542 mm_req_id = f"renderer-mm-{self._mm_req_counter.inc(1)}"
543
@@ -543,18 +546,21 @@ class BaseRenderer(ABC, Generic[_T]):
546 mm_data_items = mm_processor.info.parse_mm_data(mm_data)
547 mm_uuid_items = parse_mm_uuids(mm_uuids)
548
546 - mm_uuids = self._process_mm_uuids(
549 + mm_uuid_items = self._process_mm_uuids(
550 mm_data, mm_data_items, mm_uuid_items, mm_req_id
551 )
552
550 - with set_request_id(mm_req_id), set_default_torch_num_threads():
551 - mm_inputs = mm_processor.apply(
552 - prompt,
553 - mm_data_items,
554 - mm_uuid_items,
555 - hf_processor_mm_kwargs=mm_processor_kwargs,
556 - tokenization_kwargs=tokenization_kwargs,
557 - )
553 + mm_processor_inputs = MMProcessorInputs(
554 + prompt,
555 + mm_data_items,
556 + mm_uuid_items,
557 + hf_processor_mm_kwargs=mm_processor_kwargs or {},
558 + tokenization_kwargs=tokenization_kwargs or {},
559 + )
560 + mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
561 +
562 + with set_default_torch_num_threads():
563 + mm_inputs = mm_processor.apply(mm_processor_inputs, mm_timing_ctx)
564
565 self.update_mm_cache_stats()
566
vllm/v1/worker/gpu_model_runner.py
+3 -3
@@ -6272,7 +6272,7 @@ class GPUModelRunner(
6272 self.encoder_timing_registry[req_id] = EncoderTimingStats()
6273
6274 stats = self.encoder_timing_registry[req_id]
6275 - stats.encoder_forward_time += per_request_time
6275 + stats.encoder_forward_secs += per_request_time
6276 stats.num_encoder_calls += 1
6277
6278
@@ -6280,7 +6280,7 @@ class GPUModelRunner(
6280 class EncoderTimingStats:
6281 """Per-request timing statistics for encoder forward pass."""
6282
6283 - encoder_forward_time: float = 0.0
6283 + encoder_forward_secs: float = 0.0
6284 """Time spent in vision encoder forward pass (seconds)."""
6285
6286 num_encoder_calls: int = 0
@@ -6288,6 +6288,6 @@ class EncoderTimingStats:
6288
6289 def to_dict(self) -> dict[str, float | int]:
6290 return {
6291 - "encoder_forward_time": self.encoder_forward_time,
6291 + "encoder_forward_secs": self.encoder_forward_secs,
6292 "num_encoder_calls": self.num_encoder_calls,
6293 }