@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 751f6807d9

[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Sep 19, 2026 at 23:38 UTC 751f6807d9cb3de50c27a5f27188c4fb04fe0e2b
23 files changed +340 -398
tests/models/multimodal/processing/test_audio_in_video.py
+10 -14
@@ -63,36 +63,32 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
63 mm_processor_cache_gb=1,
64 )
65
66 - # Baseline: no cache, always processes from scratch.
67 - baseline_processor = MULTIMODAL_REGISTRY.create_processor(
68 - ctx.model_config, cache=None
69 - )
66 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
67 +
68 # Sender cache: on a cache hit returns (None, prompt_updates) for each
69 # item, setting mm_kwargs["video"] = [None] – the exact condition that
70 # triggered the original bug.
71 sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
74 - cached_processor = MULTIMODAL_REGISTRY.create_processor(
75 - ctx.model_config, cache=sender_cache
76 - )
72
78 - video_token_id = baseline_processor.info.get_hf_config().video_token_id
73 + video_token_id = processor.info.get_hf_config().video_token_id
74
75 mm_data = create_mm_data(num_videos)
76 hf_processor_mm_kwargs = {"use_audio_in_video": True}
77
83 - def run(processor):
78 + def run(processor, cache):
79 return processor(
80 [video_token_id] * num_videos,
86 - mm_items=baseline_processor.info.parse_mm_data(mm_data),
81 + mm_items=processor.info.parse_mm_data(mm_data),
82 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
83 + cache=cache,
84 )["prompt_token_ids"]
85
90 - baseline_ids = run(baseline_processor)
86 + baseline_ids = run(processor, cache=None)
87
88 # First call on the sender-cache processor: cache miss.
89 # mm_kwargs["video"] items are real tensors; use_audio_in_video is
90 # detected normally from the item data.
95 - first_ids = run(cached_processor)
91 + first_ids = run(processor, cache=sender_cache)
92 assert first_ids == baseline_ids, (
93 "Cache-miss call produced different prompt_token_ids than baseline.\n"
94 f" baseline : {baseline_ids}\n"
@@ -103,7 +99,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
99 # MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
100 # so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
101 # not detected, yielding wrong token ids.
106 - second_ids = run(cached_processor)
102 + second_ids = run(processor, cache=sender_cache)
103 assert second_ids == baseline_ids, (
104 "Cache-hit call produced different prompt_token_ids than baseline.\n"
105 "This is the regression introduced when use_audio_in_video detection\n"
@@ -123,7 +119,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
119 model_id,
120 limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
121 )
126 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
122 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
123 video_token_id = processor.info.get_hf_config().video_token_id
124
125 rng = np.random.RandomState(0)
tests/models/multimodal/processing/test_common.py
+12 -10
@@ -72,6 +72,8 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
72
73
74 _IGNORE_MM_KEYS = {
75 + # Dithering causes minor divergence
76 + "cohere_asr": {"input_features"},
77 # In Ultravox, the audio_features can be different depending on padding
78 # The slight difference should not be a problem though, since
79 # attention_mask lets us ignore the difference.
@@ -286,8 +288,7 @@ def _test_processing_correctness(
288 }
289 )
290
289 - baseline_processor = factories.build_processor(ctx, cache=None)
290 - cached_processor = factories.build_processor(ctx, cache=cache)
291 + processor = factories.build_processor(ctx)
292
293 rng = np.random.RandomState(0)
294
@@ -336,41 +337,42 @@ def _test_processing_correctness(
337 _test_processing_correctness_one(
338 model_config,
339 mm_data,
339 - baseline_processor,
340 - cached_processor,
340 + processor,
341 batch_idx,
342 hit_rate,
343 num_batches,
344 simplify_rate,
345 + cache=cache,
346 )
347
348
349 def _test_processing_correctness_one(
350 model_config: ModelConfig,
351 mm_data: MultiModalDataDict,
351 - baseline_processor: BaseMultiModalProcessor,
352 - cached_processor: BaseMultiModalProcessor,
352 + processor: BaseMultiModalProcessor,
353 batch_idx: int,
354 hit_rate: float,
355 num_batches: int,
356 simplify_rate: float,
357 + cache: MultiModalProcessorOnlyCache,
358 ):
359 model_type = model_config.hf_config.model_type
360
360 - token_prompt = get_token_prompt(baseline_processor, mm_data)
361 - mm_items = baseline_processor.info.parse_mm_data(mm_data)
361 + token_prompt = get_token_prompt(processor, mm_data)
362 + mm_items = processor.info.parse_mm_data(mm_data)
363 ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
364
364 - baseline_tokenized_result = baseline_processor(
365 + baseline_tokenized_result = processor(
366 token_prompt,
367 mm_items=mm_items,
368 hf_processor_mm_kwargs={},
369 )
370
370 - cached_tokenized_result = cached_processor(
371 + cached_tokenized_result = processor(
372 token_prompt,
373 mm_items=mm_items,
374 hf_processor_mm_kwargs={},
375 + cache=cache,
376 )
377
378 _assert_inputs_equal(
tests/models/multimodal/processing/test_gemma4.py
+8 -7
@@ -221,7 +221,7 @@ def test_video_cache_is_independent_of_image_kwargs(
221 mm_processor_cache_gb=1,
222 )
223 cache = MultiModalProcessorOnlyCache(ctx.model_config)
224 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
224 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
225 hf_processor = processor.info.get_hf_processor()
226 image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
227 frames = np.stack([np.asarray(image)] * 2)
@@ -230,15 +230,16 @@ def test_video_cache_is_independent_of_image_kwargs(
230 {"image": image, "video": [(frames, metadata)]}
231 )
232
233 - def process(mm_kwargs):
233 + def process(mm_kwargs, cache):
234 return processor(
235 hf_processor.image_token + hf_processor.video_token,
236 mm_items,
237 mm_uuid_items={"video": [video_uuid]},
238 hf_processor_mm_kwargs=mm_kwargs,
239 + cache=cache,
240 )
241
241 - baseline = process({})
242 + baseline = process({}, cache=None)
243 if kwargs_on_init:
244 ctx = build_model_context(
245 model_id,
@@ -247,12 +248,12 @@ def test_video_cache_is_independent_of_image_kwargs(
248 mm_processor_cache_gb=1,
249 )
250 cache = MultiModalProcessorOnlyCache(ctx.model_config)
250 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
251 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
252 request_kwargs = {} if kwargs_on_init else kwargs
252 - process(request_kwargs)
253 - cached = process(request_kwargs)
253 + process(request_kwargs, cache=cache)
254 + cached = process(request_kwargs, cache=cache)
255 cache.clear_cache()
255 - fresh = process(request_kwargs)
256 + fresh = process(request_kwargs, cache=cache)
257
258 def pixels(result, modality, field):
259 return result["mm_kwargs"][modality][0][field].data
tests/models/multimodal/processing/test_mllama4.py
+2 -4
@@ -23,11 +23,9 @@ def test_profiling(model_id: str, max_model_len: int):
23 model_config_kwargs=model_config_kwargs,
24 limit_mm_per_prompt=mm_counts,
25 )
26 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
27
27 - mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
28 - ctx.model_config,
29 - mm_counts=mm_counts,
30 - )
28 + mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
29
30 hf_config = ctx.get_hf_config(Llama4Config)
31 image_size = hf_config.vision_config.image_size
tests/models/multimodal/processing/test_moss_audio.py
+4 -2
@@ -168,7 +168,6 @@ def _build_moss_audio_processor(cache=None):
168 MossAudioMultiModalProcessor(
169 info,
170 MossAudioDummyInputsBuilder(info),
171 - cache=cache,
171 ),
172 ctx,
173 )
@@ -259,21 +258,24 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
258 )
259
260 cache = MultiModalProcessorOnlyCache(ctx.model_config)
262 - cached_processor, _ = _build_moss_audio_processor(cache=cache)
261 + cached_processor, _ = _build_moss_audio_processor()
262 cached_text_miss = cached_processor(
263 prompt,
264 mm_items=mm_items,
265 hf_processor_mm_kwargs={},
266 + cache=cache,
267 )
268 cached_text_hit = cached_processor(
269 prompt,
270 mm_items=mm_items,
271 hf_processor_mm_kwargs={},
272 + cache=cache,
273 )
274 cached_token_hit = cached_processor(
275 token_prompt,
276 mm_items=mm_items,
277 hf_processor_mm_kwargs={},
278 + cache=cache,
279 )
280
281 expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
tests/models/multimodal/processing/test_qwen2_vl.py
+1 -1
@@ -26,7 +26,6 @@ def test_jina_vl_processing_order() -> None:
26 processor = MULTIMODAL_REGISTRY.create_processor(
27 ctx.model_config,
28 tokenizer=ctx.tokenizer,
29 - cache=cache,
29 )
30
31 placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
@@ -37,6 +36,7 @@ def test_jina_vl_processing_order() -> None:
36 return processor(
37 placeholder * len(images),
38 mm_items=processor.info.parse_mm_data({"image": images}),
39 + cache=cache,
40 )
41
42 query = process([query_image])
tests/models/multimodal/processing/test_tensor_schema.py
+1 -1
@@ -255,7 +255,7 @@ def test_model_tensor_schema(model_id: str):
255 for modality, count in limit_mm_per_prompt.items()
256 }
257 )
258 - processor = factories.build_processor(ctx, cache=None)
258 + processor = factories.build_processor(ctx)
259
260 with initialize_dummy_model(model_cls, model_config) as model:
261 for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
tests/models/multimodal/processing/test_transformers_image.py
+1
@@ -245,6 +245,7 @@ def test_repeated_image_hits_the_processor_cache():
245 prompt="<image>\nWhat is this?",
246 mm_items=mm_processor.info.parse_mm_data({"image": image}),
247 hf_processor_mm_kwargs={},
248 + cache=cache,
249 )
250
251 first, second = process(), process()
tests/models/multimodal/processing/transformers_backend.py
+1 -1
@@ -48,4 +48,4 @@ def create_cached_processor(model_id: str, processor_cls):
48 )
49 info = MultiModalProcessingInfo(ctx)
50 cache = MultiModalProcessorOnlyCache(model_config)
51 - return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
51 + return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
tests/multimodal/test_cache.py
+34 -35
@@ -1,8 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import multiprocessing as mp
4 -from concurrent.futures import ThreadPoolExecutor
5 -from functools import partial
4 from types import SimpleNamespace
5 from unittest.mock import Mock
6
@@ -37,10 +35,13 @@ from vllm.multimodal.inputs import (
35 PlaceholderRange,
36 )
37 from vllm.multimodal.processing import PromptInsertion
40 -from vllm.renderers.base import BaseRenderer
38 +from vllm.renderers import renderer_from_config
39 from vllm.utils.async_utils import make_async
40 from vllm.utils.mem_constants import GiB_bytes, MiB_bytes
41 +from vllm.v1.engine.async_llm import AsyncLLM
42 +from vllm.v1.engine.llm_engine import LLMEngine
43
44 +from ..models.utils import build_model_context
45 from ..utils import create_new_process_for_each_test
46
47 pytestmark = pytest.mark.cpu_test
@@ -773,12 +774,14 @@ def test_processor_cache_shared_across_loras():
774 @pytest.mark.asyncio
775 async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
776 """Release must not leave a sender hit pointing at a cleared receiver."""
776 - from vllm.v1.engine.async_llm import AsyncLLM
777 - from vllm.v1.engine.llm_engine import LLMEngine
778 -
779 - model_config = SimpleNamespace(
780 - get_multimodal_config=lambda: MultiModalConfig(mm_processor_cache_gb=1)
777 + ctx = build_model_context(
778 + "llava-hf/llava-v1.6-mistral-7b-hf",
779 + mm_processor_kwargs=None,
780 + limit_mm_per_prompt={"image": 1},
781 + mm_processor_cache_gb=1,
782 )
783 + model_config = ctx.model_config
784 +
785 sender = MultiModalProcessorSenderCache(model_config)
786 receiver = MultiModalReceiverCache(model_config)
787 item = _dummy_item({"pixel_values": 16})
@@ -787,43 +790,39 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
790 assert receiver.get_and_update_item(payload, mm_hash) is item
791 assert sender.get_and_update_item(None, mm_hash)[0] is None
792
790 - renderer = SimpleNamespace(mm_processor_cache=sender, _mm_cache_stats=None)
791 - renderer.clear_mm_cache = partial(BaseRenderer.clear_mm_cache, renderer)
793 + renderer = renderer_from_config(VllmConfig(model_config=model_config))
794 + renderer._mm_processor_cache = sender
795
796 def release():
797 if release_error:
798 raise RuntimeError(release_error)
799 receiver.clear_cache()
800
798 - with ThreadPoolExecutor(max_workers=1) as executor:
799 - renderer._clear_mm_cache_async = make_async(
800 - renderer.clear_mm_cache, executor=executor
801 - )
802 - renderer.clear_mm_cache_async = partial(
803 - BaseRenderer.clear_mm_cache_async, renderer
804 - )
805 - engine = SimpleNamespace(
806 - renderer=renderer,
807 - engine_core=SimpleNamespace(
808 - release_kv_cache_memory=release,
809 - release_kv_cache_memory_async=make_async(release, executor=executor),
801 + engine = SimpleNamespace(
802 + renderer=renderer,
803 + engine_core=SimpleNamespace(
804 + release_kv_cache_memory=release,
805 + release_kv_cache_memory_async=make_async(
806 + release,
807 + executor=renderer._executor,
808 ),
811 - logger_manager=Mock(),
812 - )
813 -
814 - async def call_release():
815 - if use_async:
816 - await AsyncLLM.release_kv_cache_memory(engine)
817 - else:
818 - LLMEngine.release_kv_cache_memory(engine)
809 + ),
810 + logger_manager=Mock(),
811 + )
812
820 - if release_error:
821 - with pytest.raises(RuntimeError, match=release_error):
822 - await call_release()
823 - engine.logger_manager.record_sleep_state.assert_not_called()
813 + async def call_release():
814 + if use_async:
815 + await AsyncLLM.release_kv_cache_memory(engine)
816 else:
817 + LLMEngine.release_kv_cache_memory(engine)
818 +
819 + if release_error:
820 + with pytest.raises(RuntimeError, match=release_error):
821 await call_release()
826 - engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
822 + engine.logger_manager.record_sleep_state.assert_not_called()
823 + else:
824 + await call_release()
825 + engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0)
826
827 payload, _ = sender.get_and_update_item((item, []), mm_hash)
828 assert payload is item
tests/multimodal/test_processing.py
+36 -1
@@ -7,7 +7,7 @@ from contextlib import nullcontext
7 import numpy as np
8 import pytest
9
10 -from vllm.config import ModelConfig
10 +from vllm.config import ModelConfig, SchedulerConfig
11 from vllm.exceptions import VLLMValidationError
12 from vllm.multimodal import MULTIMODAL_REGISTRY
13 from vllm.multimodal.hasher import MultiModalHasher
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
32 )
33 from vllm.utils.collection_utils import flatten_2d_lists
34
35 +from ..models.utils import build_model_context
36 from .utils import random_image
37
38 pytestmark = pytest.mark.cpu_test
@@ -1361,3 +1362,37 @@ def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
1362 ).get_mm_hashes("test-model", "blake3")["image"][0]
1363
1364 assert hash_with(left) != hash_with(right)
1365 +
1366 +
1367 +@pytest.mark.parametrize(
1368 + ("chunked_prefill", "max_model_len", "expected_seq_len"),
1369 + [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
1370 +)
1371 +def test_dummy_inputs_scheduler_budget(
1372 + chunked_prefill, max_model_len, expected_seq_len
1373 +):
1374 + ctx = build_model_context(
1375 + "llava-hf/llava-v1.6-mistral-7b-hf",
1376 + mm_processor_kwargs=None,
1377 + limit_mm_per_prompt={"image": 1},
1378 + )
1379 + ctx.model_config.max_model_len = max_model_len
1380 +
1381 + processor = MULTIMODAL_REGISTRY.create_processor(
1382 + ctx.model_config,
1383 + tokenizer=ctx.tokenizer,
1384 + )
1385 + processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]}
1386 +
1387 + kwargs = {}
1388 + if chunked_prefill is not None:
1389 + kwargs["scheduler_config"] = SchedulerConfig(
1390 + max_model_len=max_model_len,
1391 + is_encoder_decoder=False,
1392 + max_num_batched_tokens=8192,
1393 + max_num_seqs=1,
1394 + enable_chunked_prefill=chunked_prefill,
1395 + )
1396 +
1397 + result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs)
1398 + assert len(result["prompt_token_ids"]) == expected_seq_len
tests/multimodal/test_registry.py
-30
@@ -5,11 +5,9 @@ Qwen2.5-VL visual component loading behavior.
5 """
6
7 from types import SimpleNamespace
8 -from unittest.mock import MagicMock
8
9 import pytest
10
12 -from vllm.config import SchedulerConfig
11 from vllm.multimodal import MULTIMODAL_REGISTRY
12
13 from ..models.utils import build_model_context
@@ -17,34 +15,6 @@ from ..models.utils import build_model_context
15 pytestmark = pytest.mark.cpu_test
16
17
20 -@pytest.mark.parametrize(
21 - ("chunked_prefill", "max_model_len", "expected_seq_len"),
22 - [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
23 -)
24 -def test_dummy_inputs_scheduler_budget(
25 - chunked_prefill, max_model_len, expected_seq_len
26 -):
27 - model_config = MagicMock()
28 - model_config.max_model_len = max_model_len
29 - processor = MagicMock()
30 - processor.apply.return_value = {"prompt_token_ids": [7]}
31 - kwargs = {}
32 - if chunked_prefill is not None:
33 - kwargs["scheduler_config"] = SchedulerConfig(
34 - max_model_len=max_model_len,
35 - is_encoder_decoder=False,
36 - max_num_batched_tokens=8192,
37 - max_num_seqs=1,
38 - enable_chunked_prefill=chunked_prefill,
39 - )
40 - result = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
41 - model_config, {"image": 1}, processor=processor, **kwargs
42 - )
43 - get_inputs = processor.dummy_inputs.get_dummy_processor_inputs
44 - assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
45 - assert len(result["prompt_token_ids"]) == expected_seq_len
46 -
47 -
18 @pytest.mark.parametrize(
19 "model_id,limit_mm_per_prompt,expected",
20 [
tests/renderers/test_warmup.py
+82 -90
@@ -5,6 +5,8 @@
5 These tests exercise:
6 - Zero-limit modalities are filtered from mm_counts passed to
7 get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8 + - The warmup runs through the processor-only cache and clears it afterwards,
9 + so its dummy inputs never land in the sender cache used by the serving path
10 - MM warmup is skipped entirely when mm_processor is None
11 - The multimodal warmup is launched as a task on the single-worker
12 _mm_executor to overlap engine-core init (future lifecycle, join by
@@ -21,10 +23,13 @@ that acts as the renderer instance.
23 """
24
25 from concurrent.futures import Future, ThreadPoolExecutor
26 +from types import MethodType, SimpleNamespace
27 from unittest.mock import MagicMock, patch
28
29 import pytest
30
31 +from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
32 +from vllm.multimodal.processing import BaseMultiModalProcessor
33 from vllm.renderers.base import BaseRenderer
34 from vllm.renderers.params import ChatParams
35
@@ -34,9 +39,11 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
39
40 render_chat is mocked to raise ChatTemplateResolutionError so the chat
41 warmup block is skipped cleanly, keeping the test focused on MM warmup.
37 - """
38 - from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
42
43 + The processor is a mock at its I/O boundaries (``dummy_inputs`` and
44 + ``apply``) but keeps the real ``get_dummy_mm_inputs``, since that is where
45 + the warmup's seq_len capping and cache plumbing actually live.
46 + """
47 renderer = MagicMock()
48
49 # chat warmup: make render_chat raise so we skip past it cleanly
@@ -45,13 +52,17 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
52 # MM processor with configurable limits
53 mm_processor = MagicMock()
54 mm_processor.info.allowed_mm_limits = mm_limits
55 + mm_processor.info.ctx.model_config = renderer.model_config
56 mm_processor.apply.return_value = {"prompt_token_ids": [1]}
57 + mm_processor.get_dummy_mm_inputs = MethodType(
58 + BaseMultiModalProcessor.get_dummy_mm_inputs, mm_processor
59 + )
60 renderer.mm_processor = mm_processor
50 - renderer._readonly_mm_processor = None
61 + renderer._mm_processor_cache = MagicMock(name="sender_cache")
62 + renderer._mm_processor_only_cache = MagicMock(name="processor_only_cache")
63 renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__(
64 renderer, BaseRenderer
65 )
54 - renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
66 renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer)
67 renderer.start_mm_warmup_in_background = (
68 BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer)
@@ -64,7 +75,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
75 renderer._mm_warmup_future = None
76 # MM warmup has not run yet; warmup_mm must actually execute on the mock.
77 renderer._mm_warmup_done = False
67 - renderer.clear_mm_cache = MagicMock()
78 renderer.model_config.max_model_len = 128
79 renderer.config.scheduler_config.max_num_batched_tokens = 8192
80 renderer.config.scheduler_config.enable_chunked_prefill = True
@@ -80,8 +90,7 @@ class TestMmWarmupZeroLimitFiltering:
90 """A modality with limit=0 must not appear in mm_counts."""
91 renderer = _make_renderer_mock({"image": 1, "video": 0})
92
83 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
84 - BaseRenderer.warmup(renderer, ChatParams())
93 + BaseRenderer.warmup(renderer, ChatParams())
94
95 get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
96 get_inputs.assert_called_once()
@@ -93,8 +102,7 @@ class TestMmWarmupZeroLimitFiltering:
102 """When all limits are 0, mm_counts must be empty."""
103 renderer = _make_renderer_mock({"image": 0, "video": 0})
104
96 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
97 - BaseRenderer.warmup(renderer, ChatParams())
105 + BaseRenderer.warmup(renderer, ChatParams())
106
107 get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
108 get_inputs.assert_called_once()
@@ -105,8 +113,7 @@ class TestMmWarmupZeroLimitFiltering:
113 """All modalities with limit > 0 must be present in mm_counts."""
114 renderer = _make_renderer_mock({"image": 2, "video": 1})
115
108 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
109 - BaseRenderer.warmup(renderer, ChatParams())
116 + BaseRenderer.warmup(renderer, ChatParams())
117
118 get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
119 get_inputs.assert_called_once()
@@ -125,8 +132,7 @@ class TestMmWarmupRunsNormally:
132 renderer = _make_renderer_mock({"image": 1, "video": 1})
133 renderer.model_config.max_model_len = max_model_len
134
128 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
129 - BaseRenderer.warmup(renderer, ChatParams())
135 + BaseRenderer.warmup(renderer, ChatParams())
136
137 get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
138 assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
@@ -134,26 +140,44 @@ class TestMmWarmupRunsNormally:
140 def test_processor_apply_called(self):
141 renderer = _make_renderer_mock({"image": 1})
142
137 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
138 - BaseRenderer.warmup(renderer, ChatParams())
143 + BaseRenderer.warmup(renderer, ChatParams())
144
145 renderer.mm_processor.apply.assert_called_once()
146
142 - def test_mm_cache_cleared_after_warmup(self):
147 + def test_render_chat_called_with_warmup_message(self):
148 renderer = _make_renderer_mock({"image": 1})
149
145 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
146 - BaseRenderer.warmup(renderer, ChatParams())
150 + BaseRenderer.warmup(renderer, ChatParams())
151
148 - renderer.clear_mm_cache.assert_called_once()
152 + renderer.render_chat.assert_called_once()
153
150 - def test_render_chat_called_with_warmup_message(self):
154 +
155 +class TestMmWarmupCacheRouting:
156 + # The warmup runs through the processor-only cache and drops it afterwards,
157 + # so the dummy inputs never reach the sender cache used by the serving path.
158 +
159 + def test_warmup_passes_processor_only_cache(self):
160 renderer = _make_renderer_mock({"image": 1})
161
153 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
162 + with patch.object(
163 + renderer.mm_processor,
164 + "get_dummy_mm_inputs",
165 + wraps=renderer.mm_processor.get_dummy_mm_inputs,
166 + ) as spy:
167 BaseRenderer.warmup(renderer, ChatParams())
168
156 - renderer.render_chat.assert_called_once()
169 + assert spy.call_args.kwargs["cache"] is renderer._mm_processor_only_cache
170 + assert (
171 + spy.call_args.kwargs["scheduler_config"] is renderer.config.scheduler_config
172 + )
173 +
174 + def test_only_cache_cleared_and_sender_cache_untouched(self):
175 + renderer = _make_renderer_mock({"image": 1})
176 +
177 + BaseRenderer.warmup(renderer, ChatParams())
178 +
179 + renderer._mm_processor_only_cache.clear_cache.assert_called_once()
180 + renderer._mm_processor_cache.clear_cache.assert_not_called()
181
182
183 class TestMmWarmupSkippedWhenNoProcessor:
@@ -168,51 +192,30 @@ class TestMmWarmupSkippedWhenNoProcessor:
192 renderer.model_config.get_multimodal_config.assert_not_called()
193
194
171 -class TestReadonlyMmWarmup:
172 - """Readonly MM processor warmup must mirror the render path behavior."""
173 -
174 - def test_readonly_processor_apply_called_and_cache_cleared(self):
175 - renderer = _make_renderer_mock({"image": 1})
176 - readonly_mm_processor = MagicMock()
177 - readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
178 - renderer._readonly_mm_processor = readonly_mm_processor
179 -
180 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
181 - BaseRenderer.warmup(renderer, ChatParams())
182 -
183 - readonly_mm_processor.apply.assert_called_once()
184 - readonly_mm_processor.cache.clear_cache.assert_called_once()
185 -
186 -
195 class TestWarmupFaultIsolation:
188 - # A failure during a multimodal processor warmup is caught so it does not
189 - # abort the remaining warmup steps; warmup itself must not raise.
196 + # A failure during a warmup step is caught so it does not abort the other
197 + # step; warmup itself must not raise.
198
199 def test_chat_failure_does_not_abort_mm_warmup(self):
200 renderer = _make_renderer_mock({"image": 1})
201 renderer.render_chat.side_effect = RuntimeError("chat boom")
202
195 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
196 - BaseRenderer.warmup(renderer, ChatParams()) # must not raise
203 + BaseRenderer.warmup(renderer, ChatParams()) # must not raise
204
205 renderer.mm_processor.apply.assert_called_once()
206
200 - def test_mm_failure_does_not_abort_readonly_warmup(self):
207 + def test_mm_failure_is_swallowed_and_cache_cleared(self):
208 renderer = _make_renderer_mock({"image": 1})
202 - readonly_mm_processor = MagicMock()
203 - readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
204 - renderer._readonly_mm_processor = readonly_mm_processor
205 - # main processor warmup blows up before apply()
209 + # the warmup blows up before apply()
210 renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
211 RuntimeError("mm boom")
212 )
213
210 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
211 - BaseRenderer.warmup(renderer, ChatParams()) # must not raise
214 + BaseRenderer.warmup(renderer, ChatParams()) # must not raise
215
213 - readonly_mm_processor.apply.assert_called_once()
214 - # cache is still cleared in the failed task's finally
215 - renderer.clear_mm_cache.assert_called_once()
216 + renderer.mm_processor.apply.assert_not_called()
217 + # the cache is still cleared in the failed warmup's finally
218 + renderer._mm_processor_only_cache.clear_cache.assert_called_once()
219
220
221 class TestBackgroundMmWarmup:
@@ -234,16 +237,15 @@ class TestBackgroundMmWarmup:
237 def test_start_submits_future_to_mm_executor(self):
238 renderer = self._make_bg_renderer({"image": 1})
239 try:
237 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
238 - # Spy on the real executor's submit to confirm the warmup is
239 - # dispatched through _mm_executor (not a separate Thread).
240 - with patch.object(
241 - renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
242 - ) as spy:
243 - renderer.start_mm_warmup_in_background()
244 - assert isinstance(renderer._mm_warmup_future, Future)
245 - assert spy.called
246 - renderer._join_mm_warmup()
240 + # Spy on the real executor's submit to confirm the warmup is
241 + # dispatched through _mm_executor (not a separate Thread).
242 + with patch.object(
243 + renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit
244 + ) as spy:
245 + renderer.start_mm_warmup_in_background()
246 + assert isinstance(renderer._mm_warmup_future, Future)
247 + assert spy.called
248 + renderer._join_mm_warmup()
249 assert renderer._mm_warmup_future is None
250 renderer.mm_processor.apply.assert_called_once()
251 finally:
@@ -252,7 +254,6 @@ class TestBackgroundMmWarmup:
254 def test_start_noop_for_text_only_model(self):
255 renderer = _make_renderer_mock({})
256 renderer.mm_processor = None
255 - # _readonly_mm_processor is already None
257
258 renderer.start_mm_warmup_in_background()
259
@@ -261,12 +262,11 @@ class TestBackgroundMmWarmup:
262 def test_start_is_run_at_most_once(self):
263 renderer = self._make_bg_renderer({"image": 1})
264 try:
264 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
265 - renderer.start_mm_warmup_in_background()
266 - first = renderer._mm_warmup_future
267 - renderer.start_mm_warmup_in_background() # must not spawn a second
268 - assert renderer._mm_warmup_future is first
269 - renderer._join_mm_warmup()
265 + renderer.start_mm_warmup_in_background()
266 + first = renderer._mm_warmup_future
267 + renderer.start_mm_warmup_in_background() # must not spawn a second
268 + assert renderer._mm_warmup_future is first
269 + renderer._join_mm_warmup()
270 finally:
271 renderer._mm_executor.shutdown(wait=True)
272
@@ -275,9 +275,8 @@ class TestBackgroundMmWarmup:
275 # run only the chat warmup — the MM warmup must not run twice.
276 renderer = self._make_bg_renderer({"image": 1})
277 try:
278 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
279 - renderer.start_mm_warmup_in_background()
280 - BaseRenderer.warmup(renderer, ChatParams())
278 + renderer.start_mm_warmup_in_background()
279 + BaseRenderer.warmup(renderer, ChatParams())
280
281 # MM apply called exactly once (by the background warmup task).
282 renderer.mm_processor.apply.assert_called_once()
@@ -294,13 +293,12 @@ class TestBackgroundMmWarmup:
293 # re-run the MM warmup — the _mm_warmup_done flag survives the join.
294 renderer = self._make_bg_renderer({"image": 1})
295 try:
297 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
298 - renderer.start_mm_warmup_in_background()
299 - # Simulate reset_mm_cache joining the background warmup first
300 - # (clears _mm_warmup_future, sets _mm_warmup_done via the join).
301 - renderer._join_mm_warmup()
302 - assert renderer._mm_warmup_future is None # joined & cleared
303 - BaseRenderer.warmup(renderer, ChatParams())
296 + renderer.start_mm_warmup_in_background()
297 + # Simulate reset_mm_cache joining the background warmup first
298 + # (clears _mm_warmup_future, sets _mm_warmup_done via the join).
299 + renderer._join_mm_warmup()
300 + assert renderer._mm_warmup_future is None # joined & cleared
301 + BaseRenderer.warmup(renderer, ChatParams())
302
303 # MM apply called exactly once (by the background warmup task);
304 # warmup() did not re-run warmup_mm despite the future being cleared.
@@ -313,9 +311,8 @@ class TestBackgroundMmWarmup:
311 # Direct repeated calls to warmup_mm run the MM warmup only once.
312 renderer = _make_renderer_mock({"image": 1})
313
316 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
317 - renderer.warmup_mm()
318 - renderer.warmup_mm() # second call must be a no-op
314 + renderer.warmup_mm()
315 + renderer.warmup_mm() # second call must be a no-op
316
317 renderer.mm_processor.apply.assert_called_once()
318
@@ -324,10 +321,9 @@ class TestBackgroundMmWarmup:
321 # so the mm_processor_cache is never touched concurrently.
322 renderer = self._make_bg_renderer({"image": 1})
323
327 - with patch("vllm.multimodal.processing.TimingContext", autospec=True):
328 - renderer.start_mm_warmup_in_background()
329 - future = renderer._mm_warmup_future
330 - renderer.shutdown()
324 + renderer.start_mm_warmup_in_background()
325 + future = renderer._mm_warmup_future
326 + renderer.shutdown()
327
328 assert renderer._mm_warmup_future is None
329 # The background warmup was allowed to complete (apply ran) and the
@@ -344,10 +340,6 @@ class TestEngineStartWarmupHook:
340 # These tests pin the renderer plumbing contract on EngineCoreClient.
341
342 def _mock_config(self):
347 - from types import SimpleNamespace
348 -
349 - from vllm.v1.engine.core_client import EngineCoreClient # noqa: F401
350 -
343 cfg = SimpleNamespace(
344 parallel_config=SimpleNamespace(
345 data_parallel_size=1,
vllm/model_executor/models/llava.py
-5
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.activation import get_act_fn
25 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
26 from vllm.model_executor.layers.quantization import QuantizationConfig
27 from vllm.multimodal import MULTIMODAL_REGISTRY
28 -from vllm.multimodal.cache import BaseMultiModalProcessorCache
28 from vllm.multimodal.inputs import (
29 MultiModalFieldConfig,
30 MultiModalKwargsItem,
@@ -407,21 +406,17 @@ def _build_llava_or_pixtral_hf_info(
406 def _build_llava_or_pixtral_hf_processor(
407 info: _I,
408 dummy_inputs: BaseDummyInputsBuilder[_I],
410 - *,
411 - cache: BaseMultiModalProcessorCache | None = None,
409 ) -> BaseMultiModalProcessor:
410 if isinstance(info, PixtralHFProcessingInfo):
411 return PixtralHFMultiModalProcessor(
412 info,
413 dummy_inputs, # type: ignore
417 - cache=cache,
414 )
415
416 if isinstance(info, LlavaProcessingInfo):
417 return LlavaMultiModalProcessor(
418 info,
419 dummy_inputs, # type: ignore
424 - cache=cache,
420 )
421
422 raise NotImplementedError(type(info))
vllm/model_executor/models/qwen3_asr_realtime.py
+16 -19
@@ -25,21 +25,10 @@ import torch
25 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
26 from vllm.inputs import PromptType, TokensPrompt
27 from vllm.logger import init_logger
28 -from vllm.model_executor.models.interfaces import (
29 - SupportsRealtime,
30 -)
31 -from vllm.model_executor.models.qwen3_asr import (
32 - Qwen3ASRDummyInputsBuilder,
33 - Qwen3ASRForConditionalGeneration,
34 - Qwen3ASRMultiModalProcessor,
35 - Qwen3ASRProcessingInfo,
36 - _get_feat_extract_output_lengths,
37 -)
28 from vllm.multimodal import MULTIMODAL_REGISTRY
39 -from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
29 from vllm.multimodal.parse import MultiModalDataItems
30 +from vllm.multimodal.processing import ProcessorInputs, TimingContext
31 from vllm.multimodal.processing.processor import (
42 - BaseDummyInputsBuilder,
32 MultiModalProcessingResult,
33 PlaceholderFeaturesInfo,
34 cached_encode,
@@ -47,6 +36,15 @@ from vllm.multimodal.processing.processor import (
36 from vllm.tokenizers import cached_tokenizer_from_config
37 from vllm.transformers_utils.processor import cached_processor_from_config
38
39 +from .interfaces import SupportsRealtime
40 +from .qwen3_asr import (
41 + Qwen3ASRDummyInputsBuilder,
42 + Qwen3ASRForConditionalGeneration,
43 + Qwen3ASRMultiModalProcessor,
44 + Qwen3ASRProcessingInfo,
45 + _get_feat_extract_output_lengths,
46 +)
47 +
48 logger = init_logger(__name__)
49
50 _PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60
@@ -101,14 +99,13 @@ class Qwen3ASRRealtimeBuffer:
99
100
101 class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
104 - def __init__(
102 + def _cached_apply_hf_processor(
103 self,
106 - info: _I,
107 - dummy_inputs: BaseDummyInputsBuilder[_I],
108 - *,
109 - cache: BaseMultiModalProcessorCache | None = None,
110 - ) -> None:
111 - super().__init__(info, dummy_inputs, cache=None)
104 + inputs: ProcessorInputs,
105 + timing_ctx: TimingContext,
106 + ) -> MultiModalProcessingResult:
107 + # realtime can't make use of a cache yet
108 + return self._apply_hf_processor(inputs, timing_ctx)
109
110 def _maybe_apply_prompt_updates(
111 self,
vllm/model_executor/models/voxtral_realtime.py
+7 -12
@@ -27,9 +27,8 @@ from vllm.model_executor.models.voxtral import (
27 VoxtralProcessingInfo,
28 )
29 from vllm.multimodal import MULTIMODAL_REGISTRY
30 -from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
30 from vllm.multimodal.parse import MultiModalDataItems
32 -from vllm.multimodal.processing import BaseDummyInputsBuilder
31 +from vllm.multimodal.processing import ProcessorInputs, TimingContext
32 from vllm.multimodal.processing.processor import (
33 MultiModalProcessingResult,
34 PlaceholderFeaturesInfo,
@@ -38,23 +37,19 @@ from vllm.sequence import IntermediateTensors
37 from vllm.tokenizers import cached_tokenizer_from_config
38 from vllm.utils.torch_utils import is_torch_equal_or_newer
39
41 -from .utils import (
42 - _flatten_embeddings,
43 -)
40 +from .utils import _flatten_embeddings
41
42 logger = init_logger(__name__)
43
44
45 class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
49 - def __init__(
46 + def _cached_apply_hf_processor(
47 self,
51 - info: _I,
52 - dummy_inputs: BaseDummyInputsBuilder[_I],
53 - *,
54 - cache: BaseMultiModalProcessorCache | None = None,
55 - ) -> None:
48 + inputs: ProcessorInputs,
49 + timing_ctx: TimingContext,
50 + ) -> MultiModalProcessingResult:
51 # realtime can't make use of a cache yet
57 - super().__init__(info, dummy_inputs, cache=None)
52 + return self._apply_hf_processor(inputs, timing_ctx)
53
54 def _maybe_apply_prompt_updates(
55 self,
vllm/multimodal/encoder_budget.py
+33 -32
@@ -2,8 +2,9 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 from collections.abc import Mapping
4
5 -from vllm.config import ModelConfig, VllmConfig
5 +from vllm.config import VllmConfig
6 from vllm.logger import init_logger
7 +from vllm.multimodal.cache import BaseMultiModalProcessorCache
8 from vllm.multimodal.inputs import MultiModalKwargsItem
9 from vllm.multimodal.processing import BaseMultiModalProcessor
10 from vllm.multimodal.registry import MultiModalRegistry
@@ -14,26 +15,21 @@ logger = init_logger(__name__)
15
16
17 def get_mm_max_toks_per_item(
17 - model_config: ModelConfig,
18 - mm_registry: MultiModalRegistry,
18 processor: BaseMultiModalProcessor,
19 mm_counts: Mapping[str, int],
20 + cache: BaseMultiModalProcessorCache | None,
21 ) -> Mapping[str, int]:
22 """Get the maximum number of tokens per data item from each modality based
23 on underlying model configuration.
24 """
25 max_tokens_per_item = processor.info.get_mm_max_tokens_per_item(
26 - seq_len=model_config.max_model_len,
26 + seq_len=processor.info.ctx.model_config.max_model_len,
27 mm_counts=mm_counts,
28 )
29 if max_tokens_per_item is not None:
30 return max_tokens_per_item
31
32 - mm_inputs = mm_registry.get_dummy_mm_inputs(
33 - model_config,
34 - mm_counts=mm_counts,
35 - processor=processor,
36 - )
32 + mm_inputs = processor.get_dummy_mm_inputs(mm_counts, cache=cache)
33
34 return {
35 modality: sum(item.get_num_embeds() for item in placeholders)
@@ -65,7 +61,7 @@ class MultiModalBudget:
61 if enable_cache
62 else None
63 )
68 - processor = mm_registry.create_processor(model_config, cache=cache)
64 + processor = mm_registry.create_processor(model_config)
65
66 self.cache = cache
67 self.processor = processor
@@ -91,9 +87,8 @@ class MultiModalBudget:
87 active_modalities = tower_modalities | embed_only_modalities
88
89 all_mm_max_toks_per_item = get_mm_max_toks_per_item(
94 - model_config,
95 - mm_registry,
90 processor,
91 + cache=cache,
92 mm_counts=dict.fromkeys(active_modalities, 1),
93 )
94
@@ -194,26 +189,32 @@ class MultiModalBudget:
189 def get_encoder_budget(self) -> int:
190 return min(self.encoder_compute_budget, self.encoder_cache_size)
191
197 - def reset_cache(self) -> None:
198 - if self.cache is not None:
199 - self.cache.clear_cache()
200 -
192 + def get_dummy_encoder_profile_inputs(
193 + self,
194 + modality: str | None = None,
195 + max_items_per_batch: int | None = None,
196 + ) -> list[tuple[str, MultiModalKwargsItem]]:
197 + if self.get_encoder_budget() <= 0 or not self.mm_max_toks_per_item:
198 + return []
199 +
200 + if modality is None:
201 + modality = self.get_modality_with_max_tokens()
202 + if max_items_per_batch is None:
203 + max_items_per_batch = self.mm_max_items_per_batch[modality]
204 +
205 + # Don't use `max_items_per_batch` here to avoid redundant computation
206 + dummy_mm_inputs = self.processor.get_dummy_mm_inputs(
207 + mm_counts={modality: 1},
208 + cache=self.cache,
209 + )
210 + dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
211
202 -def get_dummy_encoder_profile_inputs(
203 - mm_registry: MultiModalRegistry,
204 - budget: MultiModalBudget,
205 -) -> list[tuple[str, MultiModalKwargsItem]]:
206 - if budget.get_encoder_budget() <= 0 or not budget.mm_max_toks_per_item:
207 - return []
212 + # We use the cache so that the item is saved to the cache,
213 + # but not read from the cache
214 + assert dummy_mm_item is not None, "Item should not already be cached"
215
209 - modality = budget.get_modality_with_max_tokens()
210 - max_items_per_batch = budget.mm_max_items_per_batch[modality]
211 - dummy_mm_inputs = mm_registry.get_dummy_mm_inputs(
212 - budget.model_config,
213 - mm_counts={modality: 1},
214 - processor=budget.processor,
215 - )
216 - dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
217 - assert dummy_mm_item is not None, "Dummy item should be generated"
216 + return [(modality, dummy_mm_item)] * max_items_per_batch
217
219 - return [(modality, dummy_mm_item)] * max_items_per_batch
218 + def reset_cache(self) -> None:
219 + if self.cache is not None:
220 + self.cache.clear_cache()
vllm/multimodal/processing/inputs.py
+3
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
6 from vllm.config.multimodal import MMHasherAlgorithm
7 from vllm.inputs import MultiModalHashes
8
9 +from ..cache import BaseMultiModalProcessorCache
10 from ..hasher import MultiModalHasher
11 from ..parse import MultiModalDataItems, MultiModalUUIDItems
12
@@ -28,6 +29,8 @@ class ProcessorInputs:
29 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
30 media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
31
32 + cache: BaseMultiModalProcessorCache | None = None
33 +
34 def get_mm_hashes(
35 self,
36 model_id: str,
vllm/multimodal/processing/processor.py
+39 -4
@@ -18,6 +18,7 @@ from typing import (
18 import torch
19 from typing_extensions import TypeVar, assert_never
20
21 +from vllm.config import SchedulerConfig
22 from vllm.inputs import (
23 MultiModalEncDecInput,
24 MultiModalHashes,
@@ -1039,8 +1040,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1040 self,
1041 info: _I,
1042 dummy_inputs: "BaseDummyInputsBuilder[_I]",
1042 - *,
1043 - cache: BaseMultiModalProcessorCache | None = None,
1043 ) -> None:
1044 super().__init__()
1045
@@ -1053,16 +1052,50 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1052
1053 self.info = info
1054 self.dummy_inputs = dummy_inputs
1056 - self.cache = cache
1055
1056 self.data_parser = self.info.get_data_parser()
1057
1058 + def get_dummy_mm_inputs(
1059 + self,
1060 + mm_counts: Mapping[str, int],
1061 + *,
1062 + cache: BaseMultiModalProcessorCache | None = None,
1063 + scheduler_config: "SchedulerConfig | None" = None,
1064 + ) -> MultiModalInput:
1065 + """Create dummy data for profiling the memory usage of a model."""
1066 + model_config = self.info.ctx.model_config
1067 + seq_len = model_config.max_model_len
1068 + if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
1069 + seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
1070 +
1071 + mm_config = model_config.get_multimodal_config()
1072 +
1073 + processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
1074 + seq_len=seq_len,
1075 + mm_counts=mm_counts,
1076 + mm_options=mm_config.limit_per_prompt,
1077 + )
1078 + processor_inputs.cache = cache
1079 +
1080 + mm_inputs = self.apply(
1081 + processor_inputs,
1082 + timing_ctx=TimingContext(enabled=False),
1083 + )
1084 +
1085 + prompt_token_ids = mm_inputs["prompt_token_ids"]
1086 + total_len = len(prompt_token_ids)
1087 + if total_len < seq_len:
1088 + prompt_token_ids.extend([0] * (seq_len - total_len))
1089 +
1090 + return mm_inputs
1091 +
1092 def __call__(
1093 self,
1094 prompt: str | list[int],
1095 mm_items: MultiModalDataItems,
1096 mm_uuid_items: MultiModalUUIDItems | None = None,
1097 hf_processor_mm_kwargs: Mapping[str, object] | None = None,
1098 + cache: BaseMultiModalProcessorCache | None = None,
1099 ) -> MultiModalInput:
1100 if isinstance(prompt, str):
1101 tokenizer = self.info.get_tokenizer()
@@ -1076,6 +1109,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1109 mm_items,
1110 mm_uuid_items,
1111 hf_processor_mm_kwargs=hf_processor_mm_kwargs or {},
1112 + cache=cache,
1113 )
1114
1115 return self.apply(processor_inputs, TimingContext(enabled=False))
@@ -1420,7 +1454,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1454 """Apply the HF processor on the full prompt text,
1455 caching the results and reusing cached results.
1456 """
1423 - cache = self.cache
1457 + cache = inputs.cache
1458 has_passthrough_data = any(
1459 len(items.get_passthrough_data()) > 0
1460 for items in inputs.mm_data_items.values()
@@ -1827,6 +1861,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1861 inputs.mm_uuid_items,
1862 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
1863 media_io_kwargs=inputs.media_io_kwargs,
1864 + cache=inputs.cache,
1865 )
1866
1867 encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
vllm/multimodal/registry.py
+3 -53
@@ -2,12 +2,10 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import threading
4 from collections import defaultdict
5 -from collections.abc import Mapping
5 from dataclasses import dataclass
6 from multiprocessing.synchronize import Lock as LockType
7 from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
8
10 -from vllm.inputs import MultiModalInput
9 from vllm.logger import init_logger
10 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
11
@@ -29,12 +27,7 @@ from .processing import (
27 )
28
29 if TYPE_CHECKING:
32 - from vllm.config import (
33 - ModelConfig,
34 - ObservabilityConfig,
35 - SchedulerConfig,
36 - VllmConfig,
37 - )
30 + from vllm.config import ModelConfig, ObservabilityConfig, VllmConfig
31 from vllm.model_executor.models.interfaces import SupportsMultiModal
32
33 logger = init_logger(__name__)
@@ -75,8 +68,6 @@ class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]
68 self,
69 info: _I,
70 dummy_inputs: BaseDummyInputsBuilder[_I],
78 - *,
79 - cache: BaseMultiModalProcessorCache | None = None,
71 ) -> BaseMultiModalProcessor[_I]: ...
72
73
@@ -89,12 +80,10 @@ class _ProcessorFactories(Generic[_I]):
80 def build_processor(
81 self,
82 ctx: InputProcessingContext,
92 - *,
93 - cache: BaseMultiModalProcessorCache | None = None,
83 ):
84 info = self.info(ctx)
85 dummy_inputs_builder = self.dummy_inputs(info)
97 - return self.processor(info, dummy_inputs_builder, cache=cache)
86 + return self.processor(info, dummy_inputs_builder)
87
88
89 class MultiModalRegistry:
@@ -218,7 +207,6 @@ class MultiModalRegistry:
207 model_config: "ModelConfig",
208 *,
209 tokenizer: TokenizerLike | None = None,
221 - cache: BaseMultiModalProcessorCache | None = None,
210 ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
211 """Create a multi-modal processor for a specific model and tokenizer."""
212 if not model_config.is_multimodal_model:
@@ -230,45 +218,7 @@ class MultiModalRegistry:
218
219 ctx = self._create_processing_ctx(model_config, tokenizer)
220
233 - return factories.build_processor(ctx, cache=cache)
234 -
235 - def get_dummy_mm_inputs(
236 - self,
237 - model_config: "ModelConfig",
238 - mm_counts: Mapping[str, int],
239 - *,
240 - cache: BaseMultiModalProcessorCache | None = None,
241 - processor: BaseMultiModalProcessor | None = None,
242 - scheduler_config: "SchedulerConfig | None" = None,
243 - ) -> MultiModalInput:
244 - """Create dummy data for profiling the memory usage of a model.
245 -
246 - The model is identified by `model_config`.
247 - """
248 - seq_len = model_config.max_model_len
249 - if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
250 - seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
251 -
252 - if processor is None:
253 - processor = self.create_processor(model_config, cache=cache)
254 -
255 - mm_config = model_config.get_multimodal_config()
256 - processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
257 - seq_len=seq_len,
258 - mm_counts=mm_counts,
259 - mm_options=mm_config.limit_per_prompt,
260 - )
261 - mm_inputs = processor.apply(
262 - processor_inputs,
263 - timing_ctx=TimingContext(enabled=False),
264 - )
265 -
266 - prompt_token_ids = mm_inputs["prompt_token_ids"]
267 - total_len = len(prompt_token_ids)
268 - if total_len < seq_len:
269 - prompt_token_ids.extend([0] * (seq_len - total_len))
270 -
271 - return mm_inputs
221 + return factories.build_processor(ctx)
222
223 def _get_cache_type(
224 self,
vllm/renderers/base.py
+42 -62
@@ -127,8 +127,7 @@ class BaseRenderer(ABC, Generic[_T]):
127 self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
128
129 self.mm_processor: BaseMultiModalProcessor | None = None
130 - self._readonly_mm_processor: BaseMultiModalProcessor | None = None
131 - self._mm_cache_stats: MultiModalCacheStats | None = None
130 +
131 self._clear_mm_cache_async = make_async(
132 self.clear_mm_cache, executor=self._mm_executor
133 )
@@ -138,6 +137,16 @@ class BaseRenderer(ABC, Generic[_T]):
137 self._safe_load_prompt_embeds_async = make_async(
138 safe_load_prompt_embeds, executor=self._executor
139 )
140 +
141 + self._mm_processor_cache: BaseMultiModalProcessorCache | None = None
142 +
143 + # A second processor with its own processor-only cache.
144 + # Used by the tokenize endpoint so that tokenize-only
145 + # requests don't pollute the sender cache.
146 + self._mm_processor_only_cache: BaseMultiModalProcessorCache | None = None
147 +
148 + self._mm_cache_stats: MultiModalCacheStats | None = None
149 +
150 if mm_registry.supports_multimodal_inputs(config.model_config):
151 # Install the process-global GPU memory pool used to gate
152 # frontend GPU-side multimodal decoding (no-op when the budget
@@ -149,30 +158,16 @@ class BaseRenderer(ABC, Generic[_T]):
158 config.parallel_config._api_process_count,
159 )
160
152 - mm_processor_cache = mm_registry.processor_cache_from_config(config)
153 -
161 with set_default_torch_num_threads():
162 self.mm_processor = mm_registry.create_processor(
163 config.model_config,
164 tokenizer=self.tokenizer,
158 - cache=mm_processor_cache,
165 )
166
161 - if mm_processor_cache:
162 - self._mm_cache_stats = MultiModalCacheStats()
163 - self._resources.callback(mm_processor_cache.close)
164 -
165 - # A second processor with its own processor-only cache.
166 - # Used by the tokenize endpoint so that tokenize-only
167 - # requests don't pollute the sender cache.
168 - ro_cache = mm_registry.processor_only_cache_from_config(config)
169 - if ro_cache is not None:
170 - with set_default_torch_num_threads():
171 - self._readonly_mm_processor = mm_registry.create_processor(
172 - config.model_config,
173 - tokenizer=self.tokenizer,
174 - cache=ro_cache,
175 - )
167 + self._mm_processor_cache = mm_registry.processor_cache_from_config(config)
168 + self._mm_processor_only_cache = (
169 + mm_registry.processor_only_cache_from_config(config)
170 + )
171
172 # This is used to generate internal request ID for MM processing
173 # It has no relation to the request ID for engine core
@@ -181,6 +176,12 @@ class BaseRenderer(ABC, Generic[_T]):
176 config.observability_config
177 )
178
179 + if self._mm_processor_cache:
180 + self._mm_cache_stats = MultiModalCacheStats()
181 + self._resources.callback(self._mm_processor_cache.close)
182 + if self._mm_processor_only_cache:
183 + self._resources.callback(self._mm_processor_only_cache.close)
184 +
185 def get_tokenizer(self) -> _T:
186 tokenizer = self.tokenizer
187 if tokenizer is None:
@@ -198,11 +199,8 @@ class BaseRenderer(ABC, Generic[_T]):
199 return self.mm_processor
200
201 @property
201 - def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":
202 - if self.mm_processor is None:
203 - return None
204 -
205 - return self.mm_processor.cache
202 + def mm_processor_cache(self) -> BaseMultiModalProcessorCache | None:
203 + return self._mm_processor_cache
204
205 def stat_mm_cache(self) -> MultiModalCacheStats | None:
206 mm_cache_stats = self._mm_cache_stats
@@ -214,7 +212,7 @@ class BaseRenderer(ABC, Generic[_T]):
212 return mm_cache_stats
213
214 def update_mm_cache_stats(self) -> None:
217 - mm_processor_cache = self.mm_processor_cache
215 + mm_processor_cache = self._mm_processor_cache
216 mm_cache_stats = self._mm_cache_stats
217
218 if mm_processor_cache and mm_cache_stats:
@@ -222,38 +220,26 @@ class BaseRenderer(ABC, Generic[_T]):
220 mm_cache_stats.record(delta.total, delta.hits)
221
222 def clear_mm_cache(self) -> None:
225 - mm_processor_cache = self.mm_processor_cache
223 + mm_processor_cache = self._mm_processor_cache
224 if mm_processor_cache is not None:
225 mm_processor_cache.clear_cache()
226
227 if self._mm_cache_stats is not None:
228 self._mm_cache_stats.reset = True
229
232 - @staticmethod
233 - def _clear_processor_cache(
234 - processor: "BaseMultiModalProcessor | None",
235 - ) -> None:
236 - if processor is None:
237 - return
238 -
239 - processor_cache = processor.cache
240 - if processor_cache is not None:
241 - processor_cache.clear_cache()
242 -
230 def _warmup_mm_processor(
231 self,
245 - processor: "BaseMultiModalProcessor",
232 + processor: BaseMultiModalProcessor,
233 + cache: BaseMultiModalProcessorCache | None,
234 *,
235 log_prefix: str,
236 ) -> None:
249 - model_config = self.model_config
237 mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
238
239 start_time = time.perf_counter()
253 - _ = mm_registry.get_dummy_mm_inputs(
254 - model_config,
255 - mm_counts=dict.fromkeys(mm_limits, 1),
256 - processor=processor,
240 + _ = processor.get_dummy_mm_inputs(
241 + dict.fromkeys(mm_limits, 1),
242 + cache=cache,
243 scheduler_config=self.config.scheduler_config,
244 )
245
@@ -274,28 +260,20 @@ class BaseRenderer(ABC, Generic[_T]):
260 # prevent MM processor hangs
261 with set_default_torch_num_threads(1):
262 if self.mm_processor:
263 + mm_processor_only_cache = self._mm_processor_only_cache
264 +
265 try:
266 logger.debug("Warming up multi-modal processing...")
267 self._warmup_mm_processor(
268 self.mm_processor,
269 + mm_processor_only_cache,
270 log_prefix="Multi-modal",
271 )
272 except Exception:
273 logger.warning("Multi-modal warmup failed")
274 finally:
286 - self.clear_mm_cache()
287 -
288 - if self._readonly_mm_processor is not None:
289 - try:
290 - logger.debug("Warming up readonly multi-modal processing...")
291 - self._warmup_mm_processor(
292 - self._readonly_mm_processor,
293 - log_prefix="Readonly multi-modal",
294 - )
295 - except Exception:
296 - logger.warning("Readonly multi-modal warmup failed")
297 - finally:
298 - self._clear_processor_cache(self._readonly_mm_processor)
275 + if mm_processor_only_cache:
276 + mm_processor_only_cache.clear_cache()
277 finally:
278 self._mm_warmup_done = True
279
@@ -323,7 +301,7 @@ class BaseRenderer(ABC, Generic[_T]):
301 # which is not designed for concurrent access (its get_and_update must
302 # stay ordered with the engine-core cache) — is never touched
303 # concurrently by the warmup and the serving path.
326 - if self.mm_processor is None and self._readonly_mm_processor is None:
304 + if self.mm_processor is None:
305 return # text-only model: nothing to warm up
306 if self._mm_warmup_future is not None:
307 return # already launched
@@ -853,10 +831,7 @@ class BaseRenderer(ABC, Generic[_T]):
831 *,
832 skip_mm_cache: bool = False,
833 ) -> "MultiModalInput":
856 - if skip_mm_cache and self._readonly_mm_processor is not None:
857 - mm_processor = self._readonly_mm_processor
858 - else:
859 - mm_processor = self.get_mm_processor()
834 + mm_processor = self.get_mm_processor()
835
836 mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
837
@@ -873,6 +848,11 @@ class BaseRenderer(ABC, Generic[_T]):
848 mm_uuid_items,
849 hf_processor_mm_kwargs=mm_processor_kwargs or {},
850 media_io_kwargs=media_io_kwargs or {},
851 + cache=(
852 + self._mm_processor_only_cache
853 + if skip_mm_cache
854 + else self._mm_processor_cache
855 + ),
856 )
857 mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
858
vllm/v1/worker/gpu/model_runner.py
+1 -5
@@ -57,7 +57,6 @@ from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry
57 from vllm.multimodal import MULTIMODAL_REGISTRY
58 from vllm.multimodal.encoder_budget import (
59 MultiModalBudget,
60 - get_dummy_encoder_profile_inputs,
60 )
61 from vllm.sequence import IntermediateTensors
62 from vllm.tasks import SupportedTask
@@ -935,10 +934,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
934 self.mm_registry,
935 enable_cache=False,
936 )
938 - dummy_mm_inputs = get_dummy_encoder_profile_inputs(
939 - self.mm_registry,
940 - mm_budget,
941 - )
937 + dummy_mm_inputs = mm_budget.get_dummy_encoder_profile_inputs()
938 self.model_state.encoder_runner.profile_encoder_cache(
939 dummy_mm_inputs, mm_budget
940 )
vllm/v1/worker/gpu_model_runner.py
+4 -10
@@ -5837,21 +5837,15 @@ class GPUModelRunner(
5837 assert self.mm_budget is not None
5838
5839 # Don't use `max_items_per_batch` here to avoid redundant computation
5840 - dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
5841 - self.model_config,
5842 - mm_counts={modality: 1},
5843 - cache=self.mm_budget.cache,
5840 + dummy_mm_inputs = self.mm_budget.get_dummy_encoder_profile_inputs(
5841 + modality,
5842 + max_items_per_batch,
5843 )
5845 - dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
5846 -
5847 - # We use the cache so that the item is saved to the cache,
5848 - # but not read from the cache
5849 - assert dummy_mm_item is not None, "Item should not already be cached"
5844
5845 return next(
5846 mm_kwargs_batch
5847 for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
5854 - [(modality, dummy_mm_item)] * max_items_per_batch,
5848 + dummy_mm_inputs,
5849 device=self.device,
5850 pin_memory=PIN_MEMORY,
5851 )