A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
feat(multimodal): support externally processed mm_kwargs with cache injection (#39502)
Signed-off-by: Krish Hung <krishung5@gmail.com> Signed-off-by: krishung5 <krish@nvidia.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Kris Hung committed
Apr 21, 2026 at 19:31 UTC
766cb65d00438bb1a82b9ff58739bf3bfcc786e1
2 files changed
+232
tests/entrypoints/llm/test_mm_cache_external_injection.py
new
+193
@@ -0,0 +1,193 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
5
+pre-processed mm_kwargs into the processor cache and reports MM cache
6
+hit rate metrics accurately.
7
+
8
+This is used by frameworks like Dynamo that run the HF processor on a
9
+frontend and transfer pre-processed mm_kwargs to the backend, avoiding
10
+redundant processing.
11
+"""
12
+
13
+import logging
14
+
15
+import pytest
16
+import regex as re
17
+
18
+from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
19
+from vllm import LLM, SamplingParams
20
+from vllm.renderers.params import ChatParams
21
+from vllm.v1.metrics import loggers as stat_loggers
22
+from vllm.v1.metrics.reader import Counter, Metric
23
+
24
+
25
+def _make_messages(image_url: str):
26
+ return [
27
+ {
28
+ "role": "user",
29
+ "content": [
30
+ {
31
+ "type": "image_url",
32
+ "image_url": {"url": image_url},
33
+ },
34
+ ],
35
+ }
36
+ ]
37
+
38
+
39
+def _get_counter_value(metrics: list[Metric], name: str):
40
+ metric = next(m for m in metrics if m.name == name)
41
+ assert isinstance(metric, Counter)
42
+ return metric.value
43
+
44
+
45
+def _get_mm_cache_stats(metrics: list[Metric]):
46
+ mm_cache_queries = _get_counter_value(metrics, "vllm:mm_cache_queries")
47
+ mm_cache_hits = _get_counter_value(metrics, "vllm:mm_cache_hits")
48
+ return mm_cache_queries, mm_cache_hits
49
+
50
+
51
+def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
52
+ caplog_vllm.clear()
53
+ with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
54
+ llm.llm_engine.do_log_stats()
55
+
56
+ assert len(caplog_vllm.records) == 1
57
+ msg = caplog_vllm.records[0].getMessage()
58
+
59
+ assert "MM cache hit rate" in msg
60
+ match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
61
+ assert match is not None
62
+ return float(match.group(1))
63
+
64
+
65
+@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
66
+@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
67
+def test_inject_into_mm_cache(
68
+ num_gpus_available,
69
+ image_urls,
70
+ mm_processor_cache_type,
71
+ caplog_vllm,
72
+):
73
+ """Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
74
+ the processor cache and MM cache hit metrics are updated correctly.
75
+
76
+ Steps:
77
+ 1. Two normal requests (same image) -> cache miss then hit (baseline)
78
+ 2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
79
+ then generate with a pre-rendered input -> verifies injection works
80
+ """
81
+ llm = LLM(
82
+ model="llava-hf/llava-1.5-7b-hf",
83
+ max_model_len=4096,
84
+ max_num_seqs=5,
85
+ enforce_eager=True,
86
+ disable_log_stats=False,
87
+ limit_mm_per_prompt={"image": 2},
88
+ mm_processor_cache_type=mm_processor_cache_type,
89
+ )
90
+
91
+ # Step 1: Normal requests to populate the cache
92
+ llm.chat(_make_messages(image_urls[0]))
93
+ assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
94
+
95
+ llm.chat(_make_messages(image_urls[0]))
96
+ assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
97
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
98
+
99
+ # Step 2: Use a second image to get valid expanded tokens and
100
+ # placeholder positions via the renderer.
101
+ llm.chat(_make_messages(image_urls[1]))
102
+ queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
103
+
104
+ renderer = llm.llm_engine.renderer
105
+ cache = renderer.mm_processor_cache
106
+ assert cache is not None, "Processor cache should be enabled"
107
+
108
+ _, eng_prompts = renderer.render_chat(
109
+ [_make_messages(image_urls[1])],
110
+ ChatParams(),
111
+ )
112
+ eng_input = eng_prompts[0]
113
+
114
+ # Inject pre-processed mm_kwargs with a NEW hash via public API
115
+ new_mm_hash = "deadbeef" * 8
116
+ mm_hashes = {"image": [new_mm_hash]}
117
+ mm_kwargs = eng_input["mm_kwargs"]
118
+
119
+ llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
120
+
121
+ # Build pre-rendered input (no externally_processed flag needed)
122
+ pre_rendered_input = {
123
+ "type": "multimodal",
124
+ "prompt_token_ids": eng_input["prompt_token_ids"],
125
+ "mm_kwargs": mm_kwargs,
126
+ "mm_hashes": mm_hashes,
127
+ "mm_placeholders": eng_input["mm_placeholders"],
128
+ }
129
+
130
+ llm.generate(
131
+ pre_rendered_input,
132
+ sampling_params=SamplingParams(max_tokens=1),
133
+ )
134
+
135
+ # Verify cache was queried and injection happened
136
+ queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
137
+ assert queries_after > queries_before, (
138
+ "Cache should have been queried for the injected item"
139
+ )
140
+ mm_rate = _get_mm_cache_log(llm, caplog_vllm)
141
+ assert mm_rate >= 0.0, "MM cache hit rate should be reported"
142
+
143
+
144
+@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
145
+def test_inject_into_mm_cache_without_cache(
146
+ num_gpus_available,
147
+ image_urls,
148
+):
149
+ """Test that inject_into_mm_cache works gracefully when processor cache
150
+ is disabled (mm_processor_cache_gb=0). Should not crash.
151
+ """
152
+ llm = LLM(
153
+ model="llava-hf/llava-1.5-7b-hf",
154
+ max_model_len=4096,
155
+ max_num_seqs=5,
156
+ enforce_eager=True,
157
+ disable_log_stats=False,
158
+ limit_mm_per_prompt={"image": 2},
159
+ mm_processor_cache_gb=0,
160
+ )
161
+
162
+ # Run a normal chat request first to warm up the model.
163
+ llm.chat(_make_messages(image_urls[0]))
164
+
165
+ # Use the renderer to get a proper EngineInput with expanded tokens
166
+ renderer = llm.llm_engine.renderer
167
+ _, eng_prompts = renderer.render_chat(
168
+ [_make_messages(image_urls[0])],
169
+ ChatParams(),
170
+ )
171
+ eng_input = eng_prompts[0]
172
+
173
+ mm_hashes = {"image": ["abcd1234" * 8]}
174
+ mm_kwargs = eng_input["mm_kwargs"]
175
+
176
+ # inject_into_mm_cache should not crash even without cache
177
+ llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
178
+
179
+ # Build and generate with pre-rendered input
180
+ pre_rendered_input = {
181
+ "type": "multimodal",
182
+ "prompt_token_ids": eng_input["prompt_token_ids"],
183
+ "mm_kwargs": mm_kwargs,
184
+ "mm_hashes": mm_hashes,
185
+ "mm_placeholders": eng_input["mm_placeholders"],
186
+ }
187
+
188
+ result = llm.generate(
189
+ pre_rendered_input,
190
+ sampling_params=SamplingParams(max_tokens=1),
191
+ )
192
+ assert len(result) == 1, "Should produce one output"
193
+ assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
vllm/v1/engine/input_processor.py
+39
@@ -172,6 +172,45 @@ class InputProcessor:
172
return mm_hash
173
return f"{lora_request.lora_name}:{mm_hash}"
174
175
+ def inject_into_mm_cache(
176
+ self,
177
+ mm_hashes: dict[str, list[str]],
178
+ mm_kwargs: dict[str, list],
179
+ ) -> None:
180
+ """Inject pre-processed mm_kwargs into the processor cache.
181
+
182
+ Call this when mm_kwargs have already been through the HF processor
183
+ externally (e.g. by a frontend that transfers pre-processed tensors
184
+ to the backend). This ensures MM cache hit rate metrics are reported
185
+ accurately and avoids redundant processing on subsequent requests
186
+ with the same images.
187
+
188
+ Uses ``get_and_update_item()`` with an empty prompt_updates list,
189
+ since token expansion has already been handled externally.
190
+ """
191
+ cache = self.renderer.mm_processor_cache
192
+ if cache is None:
193
+ return
194
+ try:
195
+ for modality, hashes in mm_hashes.items():
196
+ items = mm_kwargs.get(modality, [])
197
+ for i, mm_hash in enumerate(hashes):
198
+ if i < len(items) and items[i] is not None:
199
+ # Insert into cache via get_and_update_item.
200
+ # Use the returned item (may be an address for SHM
201
+ # cache or the original item for LRU cache).
202
+ items[i], _ = cache.get_and_update_item(
203
+ (items[i], []),
204
+ mm_hash,
205
+ )
206
+ # Update cache stats to reflect the externally processed items
207
+ self.renderer.update_mm_cache_stats()
208
+ except Exception:
209
+ logger.warning(
210
+ "Failed to inject mm_kwargs into processor cache",
211
+ exc_info=True,
212
+ )
213
+
214
@staticmethod
215
def assign_request_id(request: EngineCoreRequest):
216
"""Replace the externally supplied request ID with an internal request ID