A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 22:30 UTC
[Bugfix] Forward mm_processor_kwargs in offline generate APIs (#40251)
Signed-off-by: wuyingjun <wuyingjun_yewu@cmss.chinamobile.com>
wuyingjun committed
Apr 20, 2026 at 15:56 UTC
77fd2c863147a01c75e1080523425dea68fa75d7
2 files changed
+279
-4
tests/entrypoints/llm/test_mm_processor_kwargs.py
new
+241
@@ -0,0 +1,241 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from types import SimpleNamespace
5
+from unittest.mock import Mock
6
+
7
+import pytest
8
+
9
+from vllm import LLM, SamplingParams
10
+
11
+
12
+def _make_mock_llm() -> LLM:
13
+ llm = object.__new__(LLM)
14
+ llm.model_config = SimpleNamespace(runner_type="generate")
15
+ return llm
16
+
17
+
18
+def test_generate_forwards_mm_processor_kwargs() -> None:
19
+ llm = _make_mock_llm()
20
+ mm_processor_kwargs = {"num_crops": 4}
21
+ sampling_params = SamplingParams(max_tokens=1)
22
+
23
+ llm._run_completion = Mock(return_value=["ok"])
24
+
25
+ outputs = llm.generate(
26
+ "prompt",
27
+ sampling_params=sampling_params,
28
+ mm_processor_kwargs=mm_processor_kwargs,
29
+ )
30
+
31
+ assert outputs == ["ok"]
32
+ assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
33
+ mm_processor_kwargs
34
+ )
35
+
36
+
37
+def test_enqueue_forwards_mm_processor_kwargs() -> None:
38
+ llm = _make_mock_llm()
39
+ mm_processor_kwargs = {"do_resize": False}
40
+ sampling_params = SamplingParams(max_tokens=1)
41
+
42
+ llm._add_completion_requests = Mock(return_value=["req-0"])
43
+
44
+ request_ids = llm.enqueue(
45
+ "prompt",
46
+ sampling_params=sampling_params,
47
+ use_tqdm=False,
48
+ mm_processor_kwargs=mm_processor_kwargs,
49
+ )
50
+
51
+ assert request_ids == ["req-0"]
52
+ assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
53
+ mm_processor_kwargs
54
+ )
55
+
56
+
57
+def test_chat_forwards_mm_processor_kwargs() -> None:
58
+ llm = _make_mock_llm()
59
+ mm_processor_kwargs = {"do_pan_and_scan": True}
60
+ sampling_params = SamplingParams(max_tokens=1)
61
+ messages = [{"role": "user", "content": "hello"}]
62
+
63
+ llm._run_chat = Mock(return_value=["ok"])
64
+
65
+ outputs = llm.chat(
66
+ messages,
67
+ sampling_params=sampling_params,
68
+ mm_processor_kwargs=mm_processor_kwargs,
69
+ )
70
+
71
+ assert outputs == ["ok"]
72
+ assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
73
+ mm_processor_kwargs
74
+ )
75
+
76
+
77
+def test_run_completion_forwards_mm_processor_kwargs() -> None:
78
+ llm = _make_mock_llm()
79
+ mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
80
+ sampling_params = SamplingParams(max_tokens=1)
81
+ sentinel_output = ["done"]
82
+
83
+ llm._add_completion_requests = Mock()
84
+ llm._run_engine = Mock(return_value=sentinel_output)
85
+
86
+ outputs = llm._run_completion(
87
+ prompts=["prompt"],
88
+ params=sampling_params,
89
+ output_type=object,
90
+ use_tqdm=False,
91
+ mm_processor_kwargs=mm_processor_kwargs,
92
+ )
93
+
94
+ assert outputs == sentinel_output
95
+ assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
96
+ mm_processor_kwargs
97
+ )
98
+
99
+
100
+def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
101
+ llm = _make_mock_llm()
102
+ mm_processor_kwargs = {"max_dynamic_patch": 4}
103
+ sampling_params = SamplingParams(max_tokens=1)
104
+
105
+ llm._params_to_seq = Mock(return_value=[sampling_params])
106
+ llm._lora_request_to_seq = Mock(return_value=[None])
107
+ llm._priority_to_seq = Mock(return_value=[0])
108
+ llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
109
+
110
+ captured_prompts = []
111
+
112
+ def fake_render_and_add_requests(*, prompts, **_kwargs):
113
+ captured_prompts.extend(prompts)
114
+ return ["req-0"]
115
+
116
+ llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
117
+
118
+ request_ids = llm._add_completion_requests(
119
+ prompts=["prompt"],
120
+ params=sampling_params,
121
+ use_tqdm=False,
122
+ mm_processor_kwargs=mm_processor_kwargs,
123
+ )
124
+
125
+ assert request_ids == ["req-0"]
126
+ llm._preprocess_cmpl_one.assert_called_once_with(
127
+ "prompt",
128
+ None,
129
+ mm_processor_kwargs=mm_processor_kwargs,
130
+ )
131
+ assert captured_prompts == [{"prompt_token_ids": [1]}]
132
+
133
+
134
+def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
135
+ monkeypatch: pytest.MonkeyPatch,
136
+) -> None:
137
+ llm = _make_mock_llm()
138
+ mm_processor_kwargs = {"num_crops": 8}
139
+ prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
140
+
141
+ renderer = Mock()
142
+ renderer.default_cmpl_tok_params = Mock()
143
+ renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
144
+ renderer.render_cmpl.return_value = ["engine-input"]
145
+ llm.renderer = renderer
146
+
147
+ monkeypatch.setattr(
148
+ "vllm.entrypoints.llm.parse_model_prompt",
149
+ lambda _model_config, parsed_prompt: parsed_prompt,
150
+ )
151
+
152
+ outputs = llm._preprocess_cmpl(
153
+ [prompt],
154
+ mm_processor_kwargs=mm_processor_kwargs,
155
+ )
156
+
157
+ assert outputs == ["engine-input"]
158
+ renderer.render_cmpl.assert_called_once_with(
159
+ [prompt],
160
+ "tok-params",
161
+ prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
162
+ )
163
+
164
+
165
+def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
166
+ monkeypatch: pytest.MonkeyPatch,
167
+) -> None:
168
+ llm = _make_mock_llm()
169
+ prompt = {
170
+ "prompt": "<image>",
171
+ "multi_modal_data": {"image": object()},
172
+ "mm_processor_kwargs": {"num_crops": 2},
173
+ }
174
+
175
+ renderer = Mock()
176
+ renderer.default_cmpl_tok_params = Mock()
177
+ renderer.default_cmpl_tok_params.with_kwargs.return_value = "tok-params"
178
+ renderer.render_cmpl.return_value = ["engine-input"]
179
+ llm.renderer = renderer
180
+
181
+ monkeypatch.setattr(
182
+ "vllm.entrypoints.llm.parse_model_prompt",
183
+ lambda _model_config, parsed_prompt: parsed_prompt,
184
+ )
185
+
186
+ outputs = llm._preprocess_cmpl([prompt])
187
+
188
+ assert outputs == ["engine-input"]
189
+ renderer.render_cmpl.assert_called_once_with(
190
+ [prompt],
191
+ "tok-params",
192
+ prompt_extras=None,
193
+ )
194
+
195
+
196
+def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
197
+ llm = _make_mock_llm()
198
+ mm_processor_kwargs = {"num_crops": 8}
199
+ messages = [[{"role": "user", "content": "Describe this image."}]]
200
+
201
+ renderer = Mock()
202
+ renderer.tokenizer = object()
203
+ renderer.default_chat_tok_params = Mock()
204
+ renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
205
+ renderer.render_chat.return_value = (messages, ["engine-input"])
206
+ llm.renderer = renderer
207
+
208
+ outputs = llm._preprocess_chat(
209
+ messages,
210
+ mm_processor_kwargs=mm_processor_kwargs,
211
+ )
212
+
213
+ assert outputs == ["engine-input"]
214
+ call_args = renderer.render_chat.call_args
215
+ assert call_args.args[0] == messages
216
+ assert call_args.args[1].mm_processor_kwargs == mm_processor_kwargs
217
+ assert call_args.args[2] == "tok-params"
218
+ assert call_args.kwargs["prompt_extras"] == {
219
+ "mm_processor_kwargs": mm_processor_kwargs
220
+ }
221
+
222
+
223
+def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
224
+ llm = _make_mock_llm()
225
+ messages = [[{"role": "user", "content": "Describe this image."}]]
226
+
227
+ renderer = Mock()
228
+ renderer.tokenizer = object()
229
+ renderer.default_chat_tok_params = Mock()
230
+ renderer.default_chat_tok_params.with_kwargs.return_value = "tok-params"
231
+ renderer.render_chat.return_value = (messages, ["engine-input"])
232
+ llm.renderer = renderer
233
+
234
+ outputs = llm._preprocess_chat(messages)
235
+
236
+ assert outputs == ["engine-input"]
237
+ call_args = renderer.render_chat.call_args
238
+ assert call_args.args[0] == messages
239
+ assert call_args.args[1].mm_processor_kwargs is None
240
+ assert call_args.args[2] == "tok-params"
241
+ assert call_args.kwargs["prompt_extras"] is None
vllm/entrypoints/llm.py
+38
-4
@@ -452,6 +452,7 @@ class LLM:
452
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
453
priority: list[int] | None = None,
454
tokenization_kwargs: dict[str, Any] | None = None,
455
+ mm_processor_kwargs: dict[str, Any] | None = None,
456
) -> list[RequestOutput]:
457
"""Generates the completions for the input prompts.
458
@@ -479,6 +480,7 @@ class LLM:
480
of `prompts`, where each priority value corresponds to the prompt
481
at the same index.
482
tokenization_kwargs: Overrides for `tokenizer.encode`.
483
+ mm_processor_kwargs: Overrides for `processor.__call__`.
484
485
Returns:
486
A list of `RequestOutput` objects containing the
@@ -503,6 +505,7 @@ class LLM:
505
lora_request=lora_request,
506
tokenization_kwargs=tokenization_kwargs,
507
priority=priority,
508
+ mm_processor_kwargs=mm_processor_kwargs,
509
)
510
511
def enqueue(
@@ -513,6 +516,7 @@ class LLM:
516
priority: list[int] | None = None,
517
use_tqdm: bool | Callable[..., tqdm] = True,
518
tokenization_kwargs: dict[str, Any] | None = None,
519
+ mm_processor_kwargs: dict[str, Any] | None = None,
520
) -> list[str]:
521
"""Enqueue prompts for generation without waiting for completion.
522
@@ -527,6 +531,7 @@ class LLM:
531
priority: The priority of the requests, if any.
532
use_tqdm: If True, shows a tqdm progress bar while adding requests.
533
tokenization_kwargs: Overrides for `tokenizer.encode`.
534
+ mm_processor_kwargs: Overrides for `processor.__call__`.
535
536
Returns:
537
A list of request IDs for the enqueued requests.
@@ -545,6 +550,7 @@ class LLM:
550
lora_request=lora_request,
551
priority=priority,
552
tokenization_kwargs=tokenization_kwargs,
553
+ mm_processor_kwargs=mm_processor_kwargs,
554
)
555
556
@overload
@@ -843,6 +849,7 @@ class LLM:
849
self,
850
prompts: Sequence[PromptType],
851
tokenization_kwargs: dict[str, Any] | None = None,
852
+ mm_processor_kwargs: dict[str, Any] | None = None,
853
) -> Sequence[EngineInput]:
854
"""
855
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
@@ -862,15 +869,29 @@ class LLM:
869
tok_params = renderer.default_cmpl_tok_params.with_kwargs(
870
**(tokenization_kwargs or {})
871
)
872
+ prompt_extras = (
873
+ None
874
+ if mm_processor_kwargs is None
875
+ else {"mm_processor_kwargs": mm_processor_kwargs}
876
+ )
877
866
- return renderer.render_cmpl(parsed_prompts, tok_params)
878
+ return renderer.render_cmpl(
879
+ parsed_prompts,
880
+ tok_params,
881
+ prompt_extras=prompt_extras,
882
+ )
883
884
def _preprocess_cmpl_one(
885
self,
886
prompt: PromptType,
887
tokenization_kwargs: dict[str, Any] | None = None,
888
+ mm_processor_kwargs: dict[str, Any] | None = None,
889
) -> EngineInput:
873
- (engine_input,) = self._preprocess_cmpl([prompt], tokenization_kwargs)
890
+ (engine_input,) = self._preprocess_cmpl(
891
+ [prompt],
892
+ tokenization_kwargs,
893
+ mm_processor_kwargs=mm_processor_kwargs,
894
+ )
895
return engine_input
896
897
def _preprocess_chat(
@@ -908,16 +929,22 @@ class LLM:
929
tokenize=is_mistral_tokenizer(renderer.tokenizer),
930
),
931
),
932
+ mm_processor_kwargs=mm_processor_kwargs,
933
)
934
tok_params = renderer.default_chat_tok_params.with_kwargs(
935
**(tokenization_kwargs or {})
936
)
937
+ prompt_extras = (
938
+ None
939
+ if mm_processor_kwargs is None
940
+ else {"mm_processor_kwargs": mm_processor_kwargs}
941
+ )
942
943
_, engine_inputs = renderer.render_chat(
944
conversations,
945
chat_params,
946
tok_params,
920
- prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
947
+ prompt_extras=prompt_extras,
948
)
949
950
return engine_inputs
@@ -1571,6 +1598,7 @@ class LLM:
1598
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1599
priority: list[int] | None = None,
1600
tokenization_kwargs: dict[str, Any] | None = None,
1601
+ mm_processor_kwargs: dict[str, Any] | None = None,
1602
) -> list[str]:
1603
seq_prompts = prompt_to_seq(prompts)
1604
seq_params = self._params_to_seq(params, len(seq_prompts))
@@ -1579,7 +1607,11 @@ class LLM:
1607
1608
return self._render_and_add_requests(
1609
prompts=(
1582
- self._preprocess_cmpl_one(prompt, tokenization_kwargs)
1610
+ self._preprocess_cmpl_one(
1611
+ prompt,
1612
+ tokenization_kwargs,
1613
+ mm_processor_kwargs=mm_processor_kwargs,
1614
+ )
1615
for prompt in maybe_tqdm(
1616
seq_prompts,
1617
use_tqdm=use_tqdm,
@@ -1603,6 +1635,7 @@ class LLM:
1635
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1636
priority: list[int] | None = None,
1637
tokenization_kwargs: dict[str, Any] | None = None,
1638
+ mm_processor_kwargs: dict[str, Any] | None = None,
1639
):
1640
self._add_completion_requests(
1641
prompts=prompts,
@@ -1611,6 +1644,7 @@ class LLM:
1644
lora_request=lora_request,
1645
priority=priority,
1646
tokenization_kwargs=tokenization_kwargs,
1647
+ mm_processor_kwargs=mm_processor_kwargs,
1648
)
1649
return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)
1650