A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Core][Frontend] Add Support for Inference Time mm_processor_kwargs (#9131)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Oct 8, 2024 at 08:12 UTC
a3691b6b5eb7e60039a8ff34550be5a7e8365394
21 files changed
+443
-121
examples/offline_inference_vision_language.py
+1
@@ -105,6 +105,7 @@ def run_phi3v(question: str, modality: str):
105
trust_remote_code=True,
106
max_model_len=4096,
107
max_num_seqs=2,
108
+ # Note - mm_processor_kwargs can also be passed to generate/chat calls
109
mm_processor_kwargs={"num_crops": 16},
110
)
111
stop_token_ids = None
tests/multimodal/test_processor_kwargs.py
+70
-40
@@ -74,11 +74,11 @@ def mm_model_cls():
74
# lambda whose signature matches max token calcs extra & mapper + extra kwargs
75
get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
76
custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
77
- "num_pixels": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
77
+ "pixel_values": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
78
}
79
80
81
-### Test for default processor logic & mm_processor_kwargs wrapping
81
+### Tests for default processor logic & mm_processor_kwargs wrapping
82
def test_default_processor_is_a_noop():
83
"""Ensure that by default, there is no processor override."""
84
dummy_registry = InputRegistry()
@@ -89,23 +89,46 @@ def test_default_processor_is_a_noop():
89
assert proc_inputs is proc_outputs
90
91
92
-@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
93
-def test_processor_default_kwargs(use_processor_mock, num_crops):
94
- """Ensure input processors can use processor kwargs."""
95
- dummy_registry = InputRegistry()
92
+def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
93
+ """Get the init / inference kwargs and expected num_crops for this test."""
94
# If we have a value for num_crops, pass the override value and make
95
# sure we get that value as a return-value from out mock processor,
96
# otherwise fall back to the default value
99
- mm_processor_kwargs = None if num_crops is None else {
100
- "num_crops": num_crops
97
+ init_kwargs = None if init_num_crops is None else {
98
+ "num_crops": init_num_crops
99
}
102
- expected_num_crops = DEFAULT_NUM_CROPS if num_crops is None else num_crops
103
- ctx = build_model_context(DUMMY_MODEL_ID,
104
- mm_processor_kwargs=mm_processor_kwargs)
105
- processor = dummy_registry.create_input_processor(ctx.model_config)
100
+ inference_kwargs = None if inference_num_crops is None else {
101
+ "num_crops": inference_num_crops
102
+ }
103
+ if inference_num_crops is not None:
104
+ expected_seq_count = inference_num_crops
105
+ elif init_num_crops is not None:
106
+ expected_seq_count = init_num_crops
107
+ else:
108
+ expected_seq_count = DEFAULT_NUM_CROPS
109
+ return init_kwargs, inference_kwargs, expected_seq_count
110
+
111
+
112
+@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
113
+ (None, None),
114
+ (NUM_CROPS_OVERRIDE, None),
115
+ (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
116
+])
117
+def test_input_processor_kwargs(use_processor_mock, init_num_crops,
118
+ inference_num_crops):
119
+ """Ensure input processors can use processor kwargs."""
120
+ dummy_registry = InputRegistry()
121
+
122
+ init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
123
+ init_num_crops, inference_num_crops)
124
107
- num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
108
- assert num_crops_val == expected_num_crops
125
+ ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
126
+ processor = dummy_registry.create_input_processor(ctx.model_config)
127
+ num_crops_val = processor(
128
+ LLMInputs(prompt_token_ids=[],
129
+ prompt="",
130
+ mm_processor_kwargs=inference_kwargs))
131
+ assert num_crops_val == expected_seq_count
132
133
134
@pytest.mark.parametrize(
@@ -124,11 +147,16 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
147
mm_processor_kwargs):
148
"""Ensure that input processors filter out invalid mm_processor_kwargs"""
149
dummy_registry = InputRegistry()
150
+ # Should filter out the init time kwargs
151
ctx = build_model_context(DUMMY_MODEL_ID,
152
mm_processor_kwargs=mm_processor_kwargs)
153
154
processor = dummy_registry.create_input_processor(ctx.model_config)
131
- num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
155
+ # Should filter out the inference time kwargs
156
+ num_crops_val = processor(
157
+ LLMInputs(prompt_token_ids=[],
158
+ prompt="",
159
+ mm_processor_kwargs=mm_processor_kwargs))
160
assert num_crops_val == DEFAULT_NUM_CROPS
161
162
@@ -271,32 +299,34 @@ def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
299
assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
300
301
274
-@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
275
-def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
302
+@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
303
+ (None, None),
304
+ (NUM_CROPS_OVERRIDE, None),
305
+ (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
306
+])
307
+def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
308
+ inference_num_crops):
309
"""Ensure custom mappers can use processor kwargs."""
277
- mm_processor_kwargs = None if num_crops is None else {
278
- "num_crops": num_crops
279
- }
280
- expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
310
+ init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
311
+ init_num_crops, inference_num_crops)
312
+
313
ctx = build_model_context(MULTIMODAL_MODEL_ID,
314
trust_remote_code=True,
283
- mm_processor_kwargs=mm_processor_kwargs,
315
+ mm_processor_kwargs=init_kwargs,
316
limit_mm_per_prompt={"image": 1})
317
318
mm_registry = MultiModalRegistry()
319
mm_registry.init_mm_limits_per_prompt(ctx.model_config)
288
- # Patch the image registry for phi3v with our lambda that is compatible
289
- # with overrides, then ensure that calling the method correctly echos
290
- # our num_crops value back from the mm_processor_kwargs.
320
image = image_assets[0].pil_image
321
mm_inputs = {"image": image}
322
294
- with patch.object(
295
- mm_registry._get_plugin("image"),
296
- "_default_input_mapper",
297
- {mm_model_cls(): custom_mapper},
298
- ):
299
- mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
323
+ # Patch the image registry for phi3v with our lambda that is compatible
324
+ # with overrides, then ensure that calling the method correctly echos
325
+ # our num_crops value back from the mm_processor_kwargs.
326
+ mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
327
+ mm_model_cls())
328
+ mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs,
329
+ inference_kwargs)
330
331
assert mapped_inputs["pixel_values"].shape[1] == expected_seq_count + 1
332
@@ -316,6 +346,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
346
def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
347
mm_processor_kwargs):
348
"""Ensure that custom mappers filters out invalid mm_processor_kwargs"""
349
+ # Should filter out the init time kwargs
350
ctx = build_model_context(MULTIMODAL_MODEL_ID,
351
trust_remote_code=True,
352
mm_processor_kwargs=mm_processor_kwargs,
@@ -323,17 +354,16 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
354
355
mm_registry = MultiModalRegistry()
356
mm_registry.init_mm_limits_per_prompt(ctx.model_config)
326
- # Patch the image registry for phi3v with our lambda that is compatible
327
- # with overrides, then ensure that calling the method correctly echos
328
- # our num_crops value back from the mm_processor_kwargs.
357
image = image_assets[0].pil_image
358
mm_inputs = {"image": image}
359
332
- with patch.object(
333
- mm_registry._get_plugin("image"),
334
- "_default_input_mapper",
335
- {mm_model_cls(): custom_mapper},
336
- ):
337
- mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
360
+ # Patch the image registry for phi3v with our lambda that is compatible
361
+ # with overrides, then ensure that calling the method correctly echos
362
+ # our num_crops value back from the mm_processor_kwargs.
363
+ mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
364
+ mm_model_cls())
365
+ # Should filter out the inference time kwargs
366
+ mapped_inputs = mm_registry.map_input(
367
+ ctx.model_config, mm_inputs, mm_processor_kwargs=mm_processor_kwargs)
368
369
assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
tests/test_inputs.py
+26
@@ -2,6 +2,7 @@ from typing import List
2
3
import pytest
4
5
+from vllm.inputs import zip_enc_dec_prompts
6
from vllm.inputs.parse import parse_and_batch_prompt
7
8
STRING_INPUTS = [
@@ -51,3 +52,28 @@ def test_parse_single_batch_token_consistent(token_input: List[int]):
52
def test_parse_single_batch_string_slice(inputs_slice: slice):
53
assert parse_and_batch_prompt(STRING_INPUTS)[inputs_slice] \
54
== parse_and_batch_prompt(STRING_INPUTS[inputs_slice])
55
+
56
+
57
+# yapf: disable
58
+@pytest.mark.parametrize('mm_processor_kwargs,expected_mm_kwargs', [
59
+ (None, [{}, {}]),
60
+ ({}, [{}, {}]),
61
+ ({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
62
+ ([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
63
+])
64
+# yapf: enable
65
+def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
66
+ """Test mm_processor_kwargs init for zipping enc/dec prompts."""
67
+ encoder_prompts = ['An encoder prompt', 'Another encoder prompt']
68
+ decoder_prompts = ['A decoder prompt', 'Another decoder prompt']
69
+ zipped_prompts = zip_enc_dec_prompts(encoder_prompts, decoder_prompts,
70
+ mm_processor_kwargs)
71
+ assert len(zipped_prompts) == len(encoder_prompts) == len(decoder_prompts)
72
+ for enc, dec, exp_kwargs, zipped in zip(encoder_prompts, decoder_prompts,
73
+ expected_mm_kwargs,
74
+ zipped_prompts):
75
+ assert isinstance(zipped, dict)
76
+ assert len(zipped.keys()) == 3
77
+ assert zipped['encoder_prompt'] == enc
78
+ assert zipped['decoder_prompt'] == dec
79
+ assert zipped['mm_processor_kwargs'] == exp_kwargs
tests/test_utils.py
+31
-1
@@ -7,7 +7,7 @@ from typing import AsyncIterator, Tuple
7
import pytest
8
9
from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
10
- get_open_port, merge_async_iterators)
10
+ get_open_port, merge_async_iterators, supports_kw)
11
12
from .utils import error_on_warning
13
@@ -236,3 +236,33 @@ def test_no_model_tag(parser_with_config):
236
with pytest.raises(ValueError):
237
parser_with_config.parse_args(
238
['serve', '--config', './data/test_config.yaml'])
239
+
240
+
241
+# yapf: enable
242
+@pytest.mark.parametrize(
243
+ "callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
244
+ [
245
+ # Tests for positional argument support
246
+ (lambda foo: None, "foo", True, True, False),
247
+ (lambda foo: None, "foo", False, True, True),
248
+ # Tests for positional or keyword / keyword only
249
+ (lambda foo=100: None, "foo", True, True, False),
250
+ (lambda *, foo: None, "foo", False, True, True),
251
+ # Tests to make sure the names of variadic params are NOT supported
252
+ (lambda *args: None, "args", False, True, False),
253
+ (lambda **kwargs: None, "kwargs", False, True, False),
254
+ # Tests for if we allow var kwargs to add support
255
+ (lambda foo: None, "something_else", False, True, False),
256
+ (lambda foo, **kwargs: None, "something_else", False, True, True),
257
+ (lambda foo, **kwargs: None, "kwargs", True, True, False),
258
+ (lambda foo, **kwargs: None, "foo", True, True, False),
259
+ ])
260
+# yapf: disable
261
+def test_supports_kw(callable,kw_name,requires_kw_only,
262
+ allow_var_kwargs,is_supported):
263
+ assert supports_kw(
264
+ callable=callable,
265
+ kw_name=kw_name,
266
+ requires_kw_only=requires_kw_only,
267
+ allow_var_kwargs=allow_var_kwargs
268
+ ) == is_supported
vllm/core/scheduler.py
+1
@@ -1309,6 +1309,7 @@ class Scheduler:
1309
# `multi_modal_data` will be None.
1310
multi_modal_data=seq_group.multi_modal_data
1311
if scheduler_outputs.num_prefill_groups > 0 else None,
1312
+ mm_processor_kwargs=seq_group.mm_processor_kwargs,
1313
prompt_adapter_request=seq_group.prompt_adapter_request,
1314
)
1315
else:
vllm/engine/llm_engine.py
+7
@@ -811,6 +811,13 @@ class LLMEngine:
811
)
812
processed_inputs = self.input_processor(preprocessed_inputs)
813
814
+ # This is a bit of a hack - copy the mm_processor_kwargs that were
815
+ # used in the input processor to the processed output, since these
816
+ # kwargs are presumed to be immutable and the values should be aligned
817
+ # between the input processor (here) and the input mapper.
818
+ processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
819
+ "mm_processor_kwargs")
820
+
821
self._add_processed_request(
822
request_id=request_id,
823
processed_inputs=processed_inputs,
vllm/entrypoints/llm.py
+9
@@ -472,6 +472,7 @@ class LLM:
472
add_generation_prompt: bool = True,
473
continue_final_message: bool = False,
474
tools: Optional[List[Dict[str, Any]]] = None,
475
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
476
) -> List[RequestOutput]:
477
"""
478
Generate responses for a chat conversation.
@@ -501,6 +502,8 @@ class LLM:
502
continue_final_message: If True, continues the final message in
503
the conversation instead of starting a new one. Cannot be `True`
504
if `add_generation_prompt` is also `True`.
505
+ mm_processor_kwargs: Multimodal processor kwarg overrides for this
506
+ chat request. Only used for offline requests.
507
508
Returns:
509
A list of ``RequestOutput`` objects containing the generated
@@ -522,6 +525,9 @@ class LLM:
525
tokenizer = self.get_tokenizer()
526
model_config = self.llm_engine.get_model_config()
527
528
+ # NOTE: _parse_chat_message_content_parts() currently doesn't
529
+ # handle mm_processor_kwargs, since there is no implementation in
530
+ # the chat message parsing for it.
531
conversation, mm_data = parse_chat_messages(
532
msgs, model_config, tokenizer)
533
@@ -554,6 +560,9 @@ class LLM:
560
if mm_data is not None:
561
prompt["multi_modal_data"] = mm_data
562
563
+ if mm_processor_kwargs is not None:
564
+ prompt["mm_processor_kwargs"] = mm_processor_kwargs
565
+
566
prompts.append(prompt)
567
568
return self.generate(
vllm/inputs/data.py
+58
-9
@@ -1,5 +1,5 @@
1
-from typing import (TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple,
2
- Union)
1
+from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
2
+ Optional, Tuple, Union)
3
4
from typing_extensions import NotRequired, TypedDict, TypeVar
5
@@ -19,6 +19,14 @@ class TextPrompt(TypedDict):
19
if the model supports it.
20
"""
21
22
+ mm_processor_kwargs: NotRequired[Dict[str, Any]]
23
+ """
24
+ Optional multi-modal processor kwargs to be forwarded to the
25
+ multimodal input mapper & processor. Note that if multiple modalities
26
+ have registered mappers etc for the model being considered, we attempt
27
+ to pass the mm_processor_kwargs to each of them.
28
+ """
29
+
30
31
class TokensPrompt(TypedDict):
32
"""Schema for a tokenized prompt."""
@@ -32,6 +40,14 @@ class TokensPrompt(TypedDict):
40
if the model supports it.
41
"""
42
43
+ mm_processor_kwargs: NotRequired[Dict[str, Any]]
44
+ """
45
+ Optional multi-modal processor kwargs to be forwarded to the
46
+ multimodal input mapper & processor. Note that if multiple modalities
47
+ have registered mappers etc for the model being considered, we attempt
48
+ to pass the mm_processor_kwargs to each of them.
49
+ """
50
+
51
52
SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
53
"""
@@ -74,7 +90,9 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
90
according to any of the :class:`SingletonPrompt` schemas,
91
and are not required to have the same schema.
92
77
- Only the encoder prompt may have multi-modal data.
93
+ Only the encoder prompt may have multi-modal data. mm_processor_kwargs
94
+ should be at the top-level, and should not be set in the encoder/decoder
95
+ prompts, since they are agnostic to the encoder/decoder.
96
97
Note that an :class:`ExplicitEncoderDecoderPrompt` may not
98
be used as an input to a decoder-only model,
@@ -87,6 +105,8 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
105
106
decoder_prompt: Optional[_T2_co]
107
108
+ mm_processor_kwargs: NotRequired[Dict[str, Any]]
109
+
110
111
PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
112
"""
@@ -121,6 +141,14 @@ class LLMInputs(TypedDict):
141
if the model supports it.
142
"""
143
144
+ mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
145
+ """
146
+ Optional multi-modal processor kwargs to be forwarded to the
147
+ multimodal input mapper & processor. Note that if multiple modalities
148
+ have registered mappers etc for the model being considered, we attempt
149
+ to pass the mm_processor_kwargs to each of them.
150
+ """
151
+
152
153
class EncoderDecoderLLMInputs(LLMInputs):
154
"""
@@ -152,22 +180,43 @@ _T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
180
def build_explicit_enc_dec_prompt(
181
encoder_prompt: _T1,
182
decoder_prompt: Optional[_T2],
183
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
184
) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
156
- return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
157
- decoder_prompt=decoder_prompt)
185
+ if mm_processor_kwargs is None:
186
+ mm_processor_kwargs = {}
187
+ return ExplicitEncoderDecoderPrompt(
188
+ encoder_prompt=encoder_prompt,
189
+ decoder_prompt=decoder_prompt,
190
+ mm_processor_kwargs=mm_processor_kwargs)
191
192
193
def zip_enc_dec_prompts(
194
enc_prompts: Iterable[_T1],
195
dec_prompts: Iterable[Optional[_T2]],
196
+ mm_processor_kwargs: Optional[Union[Iterable[Dict[str, Any]],
197
+ Dict[str, Any]]] = None,
198
) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
199
"""
200
Zip encoder and decoder prompts together into a list of
166
- :class:`ExplicitEncoderDecoderPrompt` instances.
167
- """
201
+ :class:`ExplicitEncoderDecoderPrompt` instances. mm_processor_kwargs
202
+ may also be provided; if a dict is passed, the same dictionary will be
203
+ used for every encoder/decoder prompt. If an iterable is provided, it will
204
+ be zipped with the encoder/decoder prompts.
205
+ """
206
+ if mm_processor_kwargs is None:
207
+ mm_processor_kwargs = {}
208
+ if isinstance(mm_processor_kwargs, Dict):
209
+ return [
210
+ build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
211
+ mm_processor_kwargs)
212
+ for (encoder_prompt,
213
+ decoder_prompt) in zip(enc_prompts, dec_prompts)
214
+ ]
215
return [
169
- build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
170
- for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)
216
+ build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
217
+ mm_proc_kwargs)
218
+ for (encoder_prompt, decoder_prompt, mm_proc_kwargs
219
+ ) in zip(enc_prompts, dec_prompts, mm_processor_kwargs)
220
]
221
222
vllm/inputs/preprocess.py
+51
-19
@@ -1,5 +1,5 @@
1
import asyncio
2
-from typing import TYPE_CHECKING, List, Optional, Tuple, Union
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4
from typing_extensions import assert_never
5
@@ -20,9 +20,11 @@ if TYPE_CHECKING:
20
logger = init_logger(__name__)
21
22
PromptComponents = Tuple[Optional[str], List[int],
23
- Optional["MultiModalDataDict"]]
23
+ Optional["MultiModalDataDict"], Optional[Dict[str,
24
+ Any]]]
25
DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
25
- Optional["MultiModalDataDict"]]
26
+ Optional["MultiModalDataDict"],
27
+ Optional[Dict[str, Any]]]
28
29
30
class InputPreprocessor:
@@ -227,6 +229,7 @@ class InputPreprocessor:
229
* prompt
230
* prompt_token_ids
231
* multi_modal_data
232
+ * mm_processor_kwargs (request-level input processor/mapper overrides)
233
'''
234
235
parsed = parse_singleton_prompt(prompt)
@@ -239,10 +242,12 @@ class InputPreprocessor:
242
lora_request=lora_request,
243
)
244
multi_modal_data = None
245
+ mm_processor_kwargs = None
246
elif parsed["type"] == "tokens":
247
prompt_text = None
248
prompt_token_ids = parsed["content"]["prompt_token_ids"]
249
multi_modal_data = parsed["content"].get("multi_modal_data")
250
+ mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
251
elif parsed["type"] == "text":
252
prompt_text = parsed["content"]["prompt"]
253
prompt_token_ids = self._tokenize_prompt(
@@ -251,10 +256,12 @@ class InputPreprocessor:
256
lora_request=lora_request,
257
)
258
multi_modal_data = parsed["content"].get("multi_modal_data")
259
+ mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
260
else:
261
assert_never(parsed)
262
257
- return prompt_text, prompt_token_ids, multi_modal_data
263
+ return (prompt_text, prompt_token_ids, multi_modal_data,
264
+ mm_processor_kwargs)
265
266
async def _extract_prompt_components_async(
267
self,
@@ -273,10 +280,12 @@ class InputPreprocessor:
280
lora_request=lora_request,
281
)
282
multi_modal_data = None
283
+ mm_processor_kwargs = None
284
elif parsed["type"] == "tokens":
285
prompt_text = None
286
prompt_token_ids = parsed["content"]["prompt_token_ids"]
287
multi_modal_data = parsed["content"].get("multi_modal_data")
288
+ mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
289
elif parsed["type"] == "text":
290
prompt_text = parsed["content"]["prompt"]
291
prompt_token_ids = await self._tokenize_prompt_async(
@@ -285,18 +294,21 @@ class InputPreprocessor:
294
lora_request=lora_request,
295
)
296
multi_modal_data = parsed["content"].get("multi_modal_data")
297
+ mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
298
else:
299
assert_never(parsed)
300
291
- return prompt_text, prompt_token_ids, multi_modal_data
301
+ return (prompt_text, prompt_token_ids, multi_modal_data,
302
+ mm_processor_kwargs)
303
304
def _build_enc_dec_llm_inputs(
305
self,
306
encoder_comps: PromptComponents,
307
decoder_comps: DecoderPromptComponents,
308
+ mm_processor_kwargs: Dict[str, Any],
309
) -> EncoderDecoderLLMInputs:
298
- encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
299
- decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
310
+ encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311
+ decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
313
if decoder_mm_data is not None:
314
raise ValueError(
@@ -314,6 +326,7 @@ class InputPreprocessor:
326
prompt_token_ids=decoder_prompt_ids,
327
prompt=decoder_prompt,
328
multi_modal_data=decoder_mm_data,
329
+ mm_processor_kwargs=mm_processor_kwargs,
330
encoder_prompt_token_ids=encoder_prompt_ids,
331
encoder_prompt=encoder_prompt,
332
encoder_multi_modal_data=encoder_mm_data,
@@ -367,21 +380,30 @@ class InputPreprocessor:
380
)
381
382
if (decoder_input := prompt["decoder_prompt"]) is None:
370
- decoder_comps = None, None, None
383
+ decoder_comps = None, None, None, None
384
else:
385
decoder_comps = self._extract_prompt_components(
386
decoder_input,
387
request_id=request_id,
388
)
389
+ # Handle this carefully in case it was directly initialized by user
390
+ mm_processor_kwargs = prompt.get("mm_processor_kwargs", {})
391
else:
392
encoder_comps = self._extract_prompt_components(
393
prompt,
394
request_id=request_id,
395
)
381
-
382
- decoder_comps = None, None, None
383
-
384
- return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
396
+ # If there are no decoder components, we assume the
397
+ # mm_processor_kwargs are in the encoder prompt
398
+ mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
399
+ -1] is not None else {}
400
+ decoder_comps = None, None, None, None
401
+
402
+ return self._build_enc_dec_llm_inputs(
403
+ encoder_comps,
404
+ decoder_comps,
405
+ mm_processor_kwargs,
406
+ )
407
408
async def _process_encoder_decoder_prompt_async(
409
self,
@@ -400,7 +422,7 @@ class InputPreprocessor:
422
423
if (decoder_input := prompt["decoder_prompt"]) is None:
424
encoder_comps = await encoder_task
403
- decoder_comps = None, None, None
425
+ decoder_comps = None, None, None, None
426
else:
427
decoder_task = self._extract_prompt_components_async(
428
decoder_input,
@@ -409,29 +431,39 @@ class InputPreprocessor:
431
432
encoder_comps, decoder_comps = await asyncio.gather(
433
encoder_task, decoder_task)
434
+ mm_processor_kwargs = prompt["mm_processor_kwargs"]
435
else:
436
encoder_comps = await self._extract_prompt_components_async(
437
prompt,
438
request_id=request_id,
439
)
417
-
418
- decoder_comps = None, None, None
419
-
420
- return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
440
+ # If there are no decoder components, we assume the
441
+ # mm_processor_kwargs are in the encoder prompt
442
+ mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
443
+ -1] is not None else {}
444
+ decoder_comps = None, None, None, None
445
+
446
+ return self._build_enc_dec_llm_inputs(
447
+ encoder_comps,
448
+ decoder_comps,
449
+ mm_processor_kwargs,
450
+ )
451
452
def _build_decoder_only_llm_inputs(
453
self,
454
prompt_comps: PromptComponents,
455
prompt_adapter_request: Optional[PromptAdapterRequest],
456
) -> LLMInputs:
427
- prompt, prompt_token_ids, multi_modal_data = prompt_comps
457
+ (prompt, prompt_token_ids, multi_modal_data,
458
+ mm_processor_kwargs) = prompt_comps
459
460
prompt_token_ids = self._apply_prompt_adapter(
461
prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
462
463
return LLMInputs(prompt_token_ids=prompt_token_ids,
464
prompt=prompt,
434
- multi_modal_data=multi_modal_data)
465
+ multi_modal_data=multi_modal_data,
466
+ mm_processor_kwargs=mm_processor_kwargs)
467
468
def _process_decoder_only_prompt(
469
self,
vllm/inputs/registry.py
+10
-3
@@ -9,7 +9,8 @@ from transformers import PretrainedConfig
9
from typing_extensions import TypeVar
10
11
from vllm.logger import init_logger
12
-from vllm.utils import get_allowed_kwarg_only_overrides, print_warning_once
12
+from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13
+ resolve_mm_processor_kwargs)
14
15
from .data import LLMInputs
16
@@ -293,8 +294,14 @@ class InputRegistry:
294
model_cls, _ = get_model_architecture(model_config)
295
processor = self._get_model_input_processor(model_cls)
296
296
- mm_processor_kwargs = get_allowed_kwarg_only_overrides(
297
- processor, overrides=model_config.mm_processor_kwargs)
297
+ # Handle multimodal processor kwargs with priority:
298
+ # Inference kwargs -> Init kwargs -> {}
299
+ # If it's empty, it'll fall back to the default kwarg values
300
+ mm_processor_kwargs = resolve_mm_processor_kwargs(
301
+ model_config.mm_processor_kwargs,
302
+ inputs.get("mm_processor_kwargs"),
303
+ processor,
304
+ )
305
306
return processor(InputContext(model_config), inputs,
307
**mm_processor_kwargs)
vllm/multimodal/audio.py
+2
-2
@@ -8,8 +8,8 @@ class AudioPlugin(MultiModalPlugin):
8
def get_data_key(self) -> str:
9
return "audio"
10
11
- def _default_input_mapper(self, ctx: InputContext,
12
- data: object) -> MultiModalInputs:
11
+ def _default_input_mapper(self, ctx: InputContext, data: object,
12
+ **mm_processor_kwargs) -> MultiModalInputs:
13
raise NotImplementedError("There is no default audio input mapper")
14
15
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
vllm/multimodal/base.py
+20
-11
@@ -1,7 +1,7 @@
1
import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
-from typing import (Callable, Dict, List, Mapping, Optional, Tuple, Type,
4
+from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type,
5
TypedDict, TypeVar, Union, cast, final)
6
7
import numpy as np
@@ -15,7 +15,7 @@ from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
17
from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18
- json_map_leaves)
18
+ json_map_leaves, resolve_mm_processor_kwargs)
19
20
logger = init_logger(__name__)
21
@@ -200,6 +200,7 @@ class MultiModalPlugin(ABC):
200
self,
201
ctx: InputContext,
202
data: MultiModalData[object],
203
+ **mm_processor_kwargs,
204
) -> MultiModalInputs:
205
"""
206
Return a dictionary to be passed as keyword arguments to
@@ -243,7 +244,8 @@ class MultiModalPlugin(ABC):
244
return wrapper
245
246
def map_input(self, model_config: ModelConfig,
246
- data: MultiModalData[object]) -> MultiModalInputs:
247
+ data: MultiModalData[object],
248
+ mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
249
"""
250
Transform the data into a dictionary of model inputs using the
251
input mapper registered for that model.
@@ -263,19 +265,26 @@ class MultiModalPlugin(ABC):
265
model_cls, _ = get_model_architecture(model_config)
266
267
mapper = self._input_mappers.get(model_cls)
266
- # Only get processor kwargs at mapping time if we are not using the
267
- # input mapper; no overrides are used on the default here because they
268
- # should be passed to the huggingface resource at initialization time.
269
- if mapper is not None and mapper != self._default_input_mapper:
270
- mm_processor_kwargs = get_allowed_kwarg_only_overrides(
271
- mapper, overrides=model_config.mm_processor_kwargs)
272
- else:
273
- mm_processor_kwargs = {}
268
269
if mapper is None:
270
raise KeyError(f"No input mapper in {self} is registered for "
271
f"model class {model_cls.__name__}.")
272
273
+ # In the case of the default mapper, we have to get resource
274
+ # processor through its HuggingFace autoclass; since this goes
275
+ # through **kwargs, we can't inspect it the same way, so we allow
276
+ # drop mm_processor_kwargs based on signature inspection
277
+ # if we're using the default mapper.
278
+ #
279
+ # This should be safe in general due to the sanitation, since the
280
+ # transformers resource should filter unused kwargs anyway.
281
+ uses_default_mapper = mapper == self._default_input_mapper
282
+ mm_processor_kwargs = resolve_mm_processor_kwargs(
283
+ model_config.mm_processor_kwargs,
284
+ mm_processor_kwargs,
285
+ callable=mapper,
286
+ allow_var_kwargs=uses_default_mapper,
287
+ )
288
return mapper(InputContext(model_config), data, **mm_processor_kwargs)
289
290
@abstractmethod
vllm/multimodal/image.py
+18
-6
@@ -1,4 +1,5 @@
1
from functools import lru_cache
2
+from typing import Any, Dict, Optional
3
4
import torch
5
from PIL import Image
@@ -23,11 +24,13 @@ class ImagePlugin(MultiModalPlugin):
24
def get_data_key(self) -> str:
25
return "image"
26
26
- def _get_hf_image_processor(self, model_config: ModelConfig):
27
- mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
28
- else model_config.mm_processor_kwargs)
29
- # We don't explicitly check kwarg overrides to the HF class
30
- # since the automodel just takes kwargs, so we can't inspect it
27
+ def _get_hf_image_processor(
28
+ self,
29
+ model_config: ModelConfig,
30
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
31
+ ):
32
+ if mm_processor_kwargs is None:
33
+ mm_processor_kwargs = {}
34
return cached_get_image_processor(
35
model_config.model,
36
trust_remote_code=model_config.trust_remote_code,
@@ -37,6 +40,7 @@ class ImagePlugin(MultiModalPlugin):
40
self,
41
ctx: InputContext,
42
data: MultiModalData[object],
43
+ **mm_processor_kwargs,
44
) -> MultiModalInputs:
45
model_config = ctx.model_config
46
@@ -46,12 +50,20 @@ class ImagePlugin(MultiModalPlugin):
50
51
# PIL image
52
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
49
- image_processor = self._get_hf_image_processor(model_config)
53
+ image_processor = self._get_hf_image_processor(
54
+ model_config,
55
+ mm_processor_kwargs,
56
+ )
57
58
if image_processor is None:
59
raise RuntimeError("No HuggingFace processor is available "
60
"to process the image object")
61
try:
62
+ # NOTE: It may make sense to forward the mm_processor_kwargs
63
+ # here too. For now, to keep it simple, we only allow it be
64
+ # used for the initialization call though, just in case the
65
+ # signatures of the preprocessor initializer don't match
66
+ # preprocess()
67
batch_data = image_processor \
68
.preprocess(data, return_tensors="pt") \
69
.data
vllm/multimodal/registry.py
+9
-4
@@ -1,6 +1,6 @@
1
import functools
2
from collections import UserDict
3
-from typing import Dict, Mapping, Optional, Sequence
3
+from typing import Any, Dict, Mapping, Optional, Sequence
4
5
from vllm.config import ModelConfig
6
from vllm.logger import init_logger
@@ -96,8 +96,12 @@ class MultiModalRegistry:
96
"""
97
return self.register_input_mapper("image", mapper)
98
99
- def map_input(self, model_config: ModelConfig,
100
- data: MultiModalDataDict) -> MultiModalInputs:
99
+ def map_input(
100
+ self,
101
+ model_config: ModelConfig,
102
+ data: MultiModalDataDict,
103
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
104
+ ) -> MultiModalInputs:
105
"""
106
Apply an input mapper to the data passed to the model.
107
@@ -123,7 +127,8 @@ class MultiModalRegistry:
127
f"`--limit-mm-per-prompt`, but found {num_items} items "
128
"in the same prompt.")
129
126
- input_dict = plugin.map_input(model_config, data_value)
130
+ input_dict = plugin.map_input(model_config, data_value,
131
+ mm_processor_kwargs)
132
for input_key, input_tensor in input_dict.items():
133
if input_key in merged_dict:
134
raise ValueError(f"The input mappers (keys={set(data)}) "
vllm/multimodal/video.py
+17
-7
@@ -1,5 +1,5 @@
1
from functools import lru_cache
2
-from typing import List, Union
2
+from typing import Any, Dict, List, Optional, Union
3
4
import numpy as np
5
@@ -36,11 +36,13 @@ class VideoPlugin(ImagePlugin):
36
def get_data_key(self) -> str:
37
return "video"
38
39
- def _get_hf_video_processor(self, model_config: ModelConfig):
40
- mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
41
- else model_config.mm_processor_kwargs)
42
- # We don't explicitly check kwarg overrides to the HF class
43
- # since the automodel just takes kwargs, so we can't inspect it
39
+ def _get_hf_video_processor(
40
+ self,
41
+ model_config: ModelConfig,
42
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
43
+ ):
44
+ if mm_processor_kwargs is None:
45
+ mm_processor_kwargs = {}
46
return cached_get_video_processor(
47
model_config.model,
48
trust_remote_code=model_config.trust_remote_code,
@@ -50,16 +52,24 @@ class VideoPlugin(ImagePlugin):
52
self,
53
ctx: InputContext,
54
data: MultiModalData[object],
55
+ **mm_processor_kwargs,
56
) -> MultiModalInputs:
57
model_config = ctx.model_config
58
59
# single video input as np.ndarray
60
if isinstance(data, np.ndarray):
58
- video_processor = self._get_hf_video_processor(model_config)
61
+ video_processor = self._get_hf_video_processor(
62
+ model_config,
63
+ mm_processor_kwargs,
64
+ )
65
if video_processor is None:
66
raise RuntimeError("No HuggingFace processor is available "
67
"to process the image object")
68
try:
69
+ # NOTE: Similar to image; it may be a good idea to filter and
70
+ # pass mm_processor_kwargs here too, but for now we don't to
71
+ # avoid extra complexity if the initializer and preprocess
72
+ # signatures of the processor don't align
73
batch_data = video_processor(data, return_tensors="pt").data
74
except Exception:
75
logger.error("Failed to process image (%s)", data)
vllm/sequence.py
+14
@@ -481,6 +481,10 @@ class Sequence:
481
EncoderDecoderLLMInputs,
482
inputs).get("encoder_multi_modal_data")) or {}
483
484
+ @property
485
+ def mm_processor_kwargs(self) -> Dict[str, Any]:
486
+ return self.inputs.get("mm_processor_kwargs") or {}
487
+
488
@property
489
def lora_int_id(self) -> int:
490
return self.lora_request.lora_int_id if self.lora_request else 0
@@ -710,6 +714,14 @@ class SequenceGroup:
714
# We use the multi-modal data of an arbitrary sequence.
715
return self.seqs[0].multi_modal_data
716
717
+ @property
718
+ def mm_processor_kwargs(self) -> Dict[str, Any]:
719
+ # As with multi-modal data, all sequences in the group should have the
720
+ # same processor kwargs (i.e., mm_processor_kwargs are optionally
721
+ # provided per request; note that are independent of whether the model
722
+ # decoder-only or an encoder-decoder).
723
+ return self.seqs[0].mm_processor_kwargs
724
+
725
@property
726
def lora_int_id(self) -> int:
727
return self.lora_request.lora_int_id if self.lora_request else 0
@@ -949,6 +961,7 @@ class SequenceGroupMetadata(
961
used in prefix caching.
962
state: Internal state tied to this sequence group.
963
multi_modal_data: Multi modal data.
964
+ mm_processor_kwargs: Multimodal input processor / mapper overrides.
965
encoder_seq_data: Optional sequence data for encoder prompt
966
(SequenceGroup.encoder_seq). Should be None
967
unless you are working with an encoder/decoder
@@ -975,6 +988,7 @@ class SequenceGroupMetadata(
988
# "MultiModalDataDict" types. We have to use Any due to msgspec
989
# doesn't allow to have union of 2 different dicts.
990
multi_modal_data: Optional[Any] = None
991
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None
992
encoder_seq_data: Optional[SequenceData] = None
993
cross_block_table: Optional[List[int]] = None
994
prompt_adapter_request: Optional[PromptAdapterRequest] = None
vllm/utils.py
+82
-13
@@ -1277,18 +1277,87 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1277
return await task(*args, **kwargs)
1278
1279
1280
-def supports_kw(callable: Callable[..., object], kw_name: str) -> bool:
1280
+def supports_kw(
1281
+ callable: Callable[..., object],
1282
+ kw_name: str,
1283
+ requires_kw_only: bool = False,
1284
+ allow_var_kwargs: bool = True,
1285
+) -> bool:
1286
+ """Check if a keyword is a valid kwarg for a callable; if requires_kw_only
1287
+ disallows kwargs names that can also be positional arguments.
1288
+ """
1289
params = inspect.signature(callable).parameters
1282
- if kw_name in params:
1283
- return True
1290
+ if not params:
1291
+ return False
1292
+
1293
+ param_val = params.get(kw_name)
1294
+
1295
+ # Types where the it may be valid, i.e., explicitly defined & nonvariadic
1296
+ passable_kw_types = set((inspect.Parameter.POSITIONAL_ONLY,
1297
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
1298
+ inspect.Parameter.KEYWORD_ONLY))
1299
+
1300
+ if param_val:
1301
+ is_sig_param = param_val.kind in passable_kw_types
1302
+ # We want kwargs only, but this is passable as a positional arg
1303
+ if (requires_kw_only and is_sig_param
1304
+ and param_val.kind != inspect.Parameter.KEYWORD_ONLY):
1305
+ return False
1306
+ if ((requires_kw_only
1307
+ and param_val.kind == inspect.Parameter.KEYWORD_ONLY)
1308
+ or (not requires_kw_only and is_sig_param)):
1309
+ return True
1310
+
1311
+ # If we're okay with var-kwargs, it's supported as long as
1312
+ # the kw_name isn't something like *args, **kwargs
1313
+ if allow_var_kwargs:
1314
+ # Get the last param; type is ignored here because params is a proxy
1315
+ # mapping, but it wraps an ordered dict, and they appear in order.
1316
+ # Ref: https://docs.python.org/3/library/inspect.html#inspect.Signature.parameters
1317
+ last_param = params[next(reversed(params))] # type: ignore
1318
+ return (last_param.kind == inspect.Parameter.VAR_KEYWORD
1319
+ and last_param.name != kw_name)
1320
+ return False
1321
+
1322
+
1323
+def resolve_mm_processor_kwargs(
1324
+ init_kwargs: Optional[Dict[str, Any]],
1325
+ inference_kwargs: Optional[Dict[str, Any]],
1326
+ callable: Callable[..., object],
1327
+ allow_var_kwargs: bool = False,
1328
+) -> Dict[str, Any]:
1329
+ """Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
1330
+ those who are not explicit keywords to the given callable (of one is
1331
+ given; otherwise no filtering is done), then merges the kwarg dicts,
1332
+ giving priority to inference_kwargs if there are any collisions.
1333
+
1334
+ In the case that no kwarg overrides are provided, returns an empty
1335
+ dict so that it can still be kwarg expanded into the callable later on.
1336
+
1337
+ If allow_var_kwargs=True, allows for things that can be expanded into
1338
+ kwargs as long as they aren't naming collision for var_kwargs or potential
1339
+ positional arguments.
1340
+ """
1341
+ # Filter inference time multimodal processor kwargs provided
1342
+ runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
1343
+ callable,
1344
+ overrides=inference_kwargs,
1345
+ allow_var_kwargs=allow_var_kwargs)
1346
+
1347
+ # Filter init time multimodal processor kwargs provided
1348
+ init_mm_kwargs = get_allowed_kwarg_only_overrides(
1349
+ callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs)
1350
1285
- return any(param.kind == inspect.Parameter.VAR_KEYWORD
1286
- for param in params.values())
1351
+ # Merge the final processor kwargs, prioritizing inference
1352
+ # time values over the initialization time values.
1353
+ mm_processor_kwargs = {**init_mm_kwargs, **runtime_mm_kwargs}
1354
+ return mm_processor_kwargs
1355
1356
1357
def get_allowed_kwarg_only_overrides(
1358
callable: Callable[..., object],
1359
overrides: Optional[Dict[str, Any]],
1360
+ allow_var_kwargs: bool = False,
1361
) -> Dict[str, Any]:
1362
"""
1363
Given a callable which has one or more keyword only params and a dict
@@ -1300,7 +1369,9 @@ def get_allowed_kwarg_only_overrides(
1369
1370
Args:
1371
callable: Callable which takes 0 or more keyword only arguments.
1372
+ If None is provided, all overrides names are allowed.
1373
overrides: Potential overrides to be used when invoking the callable.
1374
+ allow_var_kwargs: Allows overrides that are expandable for var kwargs.
1375
1376
Returns:
1377
Dictionary containing the kwargs to be leveraged which may be used
@@ -1310,17 +1381,15 @@ def get_allowed_kwarg_only_overrides(
1381
if not overrides:
1382
return {}
1383
1313
- allowed_override_names = [
1314
- name for name, param in inspect.signature(callable).parameters.items()
1315
- if param.kind == inspect.Parameter.KEYWORD_ONLY
1316
- ]
1317
-
1318
- # Drop any mm_processor_kwargs provided by the user that are
1319
- # not kwarg names accepted by the provided input processor.
1384
+ # Drop any mm_processor_kwargs provided by the user that
1385
+ # are not kwargs, unless it can fit it var_kwargs param
1386
filtered_overrides = {
1387
kwarg_name: val
1388
for kwarg_name, val in overrides.items()
1323
- if kwarg_name in allowed_override_names
1389
+ if supports_kw(callable,
1390
+ kwarg_name,
1391
+ requires_kw_only=True,
1392
+ allow_var_kwargs=allow_var_kwargs)
1393
}
1394
1395
# If anything is dropped, log a warning
vllm/worker/cpu_model_runner.py
+5
-3
@@ -148,8 +148,9 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
148
)
149
150
def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
151
- computed_len: int):
152
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
151
+ computed_len: int,
152
+ mm_processor_kwargs: Dict[str, Any]):
153
+ mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
154
155
# special processing for mrope position deltas.
156
mrope_positions = None
@@ -210,7 +211,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
211
mrope_positions = None
212
if (mm_data := seq_group_metadata.multi_modal_data):
213
mm_kwargs, mrope_positions = self._compute_multi_modal_input(
213
- seq_data, mm_data, computed_len)
214
+ seq_data, mm_data, computed_len,
215
+ seq_group_metadata.mm_processor_kwargs)
216
multi_modal_inputs_list.append(mm_kwargs)
217
218
# Token position ids
vllm/worker/model_runner.py
+3
-1
@@ -640,7 +640,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
640
if not mm_data:
641
return
642
643
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
643
+ mm_kwargs = self.multi_modal_input_mapper(
644
+ mm_data,
645
+ mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
646
inter_data.multi_modal_inputs = mm_kwargs
647
648
# special processing for mrope position deltas.
vllm/worker/neuron_model_runner.py
+4
-1
@@ -153,7 +153,10 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
153
mm_data = seq_group_metadata.multi_modal_data
154
if mm_data:
155
# Process multi-modal data
156
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
156
+ mm_kwargs = self.multi_modal_input_mapper(
157
+ mm_data,
158
+ mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs,
159
+ )
160
multi_modal_inputs_list.append(mm_kwargs)
161
162
max_seq_len = max(seq_lens)
vllm/worker/openvino_model_runner.py
+5
-1
@@ -172,7 +172,11 @@ class OpenVINOModelRunner:
172
173
mm_data = seq_group_metadata.multi_modal_data
174
if mm_data:
175
- mm_kwargs = self.multi_modal_input_mapper(mm_data)
175
+ mm_kwargs = self.multi_modal_input_mapper(
176
+ mm_data,
177
+ mm_processor_kwargs=seq_group_metadata.
178
+ mm_processor_kwargs,
179
+ )
180
multi_modal_inputs_list.append(mm_kwargs)
181
182
block_table = seq_group_metadata.block_tables[seq_id]