@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3691b6b5e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Core][Frontend] Add Support for Inference Time mm_processor_kwargs (#9131)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Oct 8, 2024 at 08:12 UTC a3691b6b5eb7e60039a8ff34550be5a7e8365394
21 files changed +443 -121
examples/offline_inference_vision_language.py
+1
@@ -105,6 +105,7 @@ def run_phi3v(question: str, modality: str):
105 trust_remote_code=True,
106 max_model_len=4096,
107 max_num_seqs=2,
108 + # Note - mm_processor_kwargs can also be passed to generate/chat calls
109 mm_processor_kwargs={"num_crops": 16},
110 )
111 stop_token_ids = None
tests/multimodal/test_processor_kwargs.py
+70 -40
@@ -74,11 +74,11 @@ def mm_model_cls():
74 # lambda whose signature matches max token calcs extra & mapper + extra kwargs
75 get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
76 custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
77 - "num_pixels": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
77 + "pixel_values": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
78 }
79
80
81 -### Test for default processor logic & mm_processor_kwargs wrapping
81 +### Tests for default processor logic & mm_processor_kwargs wrapping
82 def test_default_processor_is_a_noop():
83 """Ensure that by default, there is no processor override."""
84 dummy_registry = InputRegistry()
@@ -89,23 +89,46 @@ def test_default_processor_is_a_noop():
89 assert proc_inputs is proc_outputs
90
91
92 -@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
93 -def test_processor_default_kwargs(use_processor_mock, num_crops):
94 - """Ensure input processors can use processor kwargs."""
95 - dummy_registry = InputRegistry()
92 +def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
93 + """Get the init / inference kwargs and expected num_crops for this test."""
94 # If we have a value for num_crops, pass the override value and make
95 # sure we get that value as a return-value from out mock processor,
96 # otherwise fall back to the default value
99 - mm_processor_kwargs = None if num_crops is None else {
100 - "num_crops": num_crops
97 + init_kwargs = None if init_num_crops is None else {
98 + "num_crops": init_num_crops
99 }
102 - expected_num_crops = DEFAULT_NUM_CROPS if num_crops is None else num_crops
103 - ctx = build_model_context(DUMMY_MODEL_ID,
104 - mm_processor_kwargs=mm_processor_kwargs)
105 - processor = dummy_registry.create_input_processor(ctx.model_config)
100 + inference_kwargs = None if inference_num_crops is None else {
101 + "num_crops": inference_num_crops
102 + }
103 + if inference_num_crops is not None:
104 + expected_seq_count = inference_num_crops
105 + elif init_num_crops is not None:
106 + expected_seq_count = init_num_crops
107 + else:
108 + expected_seq_count = DEFAULT_NUM_CROPS
109 + return init_kwargs, inference_kwargs, expected_seq_count
110 +
111 +
112 +@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
113 + (None, None),
114 + (NUM_CROPS_OVERRIDE, None),
115 + (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
116 +])
117 +def test_input_processor_kwargs(use_processor_mock, init_num_crops,
118 + inference_num_crops):
119 + """Ensure input processors can use processor kwargs."""
120 + dummy_registry = InputRegistry()
121 +
122 + init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
123 + init_num_crops, inference_num_crops)
124
107 - num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
108 - assert num_crops_val == expected_num_crops
125 + ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
126 + processor = dummy_registry.create_input_processor(ctx.model_config)
127 + num_crops_val = processor(
128 + LLMInputs(prompt_token_ids=[],
129 + prompt="",
130 + mm_processor_kwargs=inference_kwargs))
131 + assert num_crops_val == expected_seq_count
132
133
134 @pytest.mark.parametrize(
@@ -124,11 +147,16 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
147 mm_processor_kwargs):
148 """Ensure that input processors filter out invalid mm_processor_kwargs"""
149 dummy_registry = InputRegistry()
150 + # Should filter out the init time kwargs
151 ctx = build_model_context(DUMMY_MODEL_ID,
152 mm_processor_kwargs=mm_processor_kwargs)
153
154 processor = dummy_registry.create_input_processor(ctx.model_config)
131 - num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
155 + # Should filter out the inference time kwargs
156 + num_crops_val = processor(
157 + LLMInputs(prompt_token_ids=[],
158 + prompt="",
159 + mm_processor_kwargs=mm_processor_kwargs))
160 assert num_crops_val == DEFAULT_NUM_CROPS
161
162
@@ -271,32 +299,34 @@ def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
299 assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
300
301
274 -@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
275 -def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
302 +@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
303 + (None, None),
304 + (NUM_CROPS_OVERRIDE, None),
305 + (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
306 +])
307 +def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
308 + inference_num_crops):
309 """Ensure custom mappers can use processor kwargs."""
277 - mm_processor_kwargs = None if num_crops is None else {
278 - "num_crops": num_crops
279 - }
280 - expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
310 + init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
311 + init_num_crops, inference_num_crops)
312 +
313 ctx = build_model_context(MULTIMODAL_MODEL_ID,
314 trust_remote_code=True,
283 - mm_processor_kwargs=mm_processor_kwargs,
315 + mm_processor_kwargs=init_kwargs,
316 limit_mm_per_prompt={"image": 1})
317
318 mm_registry = MultiModalRegistry()
319 mm_registry.init_mm_limits_per_prompt(ctx.model_config)
288 - # Patch the image registry for phi3v with our lambda that is compatible
289 - # with overrides, then ensure that calling the method correctly echos
290 - # our num_crops value back from the mm_processor_kwargs.
320 image = image_assets[0].pil_image
321 mm_inputs = {"image": image}
322
294 - with patch.object(
295 - mm_registry._get_plugin("image"),
296 - "_default_input_mapper",
297 - {mm_model_cls(): custom_mapper},
298 - ):
299 - mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
323 + # Patch the image registry for phi3v with our lambda that is compatible
324 + # with overrides, then ensure that calling the method correctly echos
325 + # our num_crops value back from the mm_processor_kwargs.
326 + mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
327 + mm_model_cls())
328 + mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs,
329 + inference_kwargs)
330
331 assert mapped_inputs["pixel_values"].shape[1] == expected_seq_count + 1
332
@@ -316,6 +346,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
346 def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
347 mm_processor_kwargs):
348 """Ensure that custom mappers filters out invalid mm_processor_kwargs"""
349 + # Should filter out the init time kwargs
350 ctx = build_model_context(MULTIMODAL_MODEL_ID,
351 trust_remote_code=True,
352 mm_processor_kwargs=mm_processor_kwargs,
@@ -323,17 +354,16 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
354
355 mm_registry = MultiModalRegistry()
356 mm_registry.init_mm_limits_per_prompt(ctx.model_config)
326 - # Patch the image registry for phi3v with our lambda that is compatible
327 - # with overrides, then ensure that calling the method correctly echos
328 - # our num_crops value back from the mm_processor_kwargs.
357 image = image_assets[0].pil_image
358 mm_inputs = {"image": image}
359
332 - with patch.object(
333 - mm_registry._get_plugin("image"),
334 - "_default_input_mapper",
335 - {mm_model_cls(): custom_mapper},
336 - ):
337 - mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
360 + # Patch the image registry for phi3v with our lambda that is compatible
361 + # with overrides, then ensure that calling the method correctly echos
362 + # our num_crops value back from the mm_processor_kwargs.
363 + mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
364 + mm_model_cls())
365 + # Should filter out the inference time kwargs
366 + mapped_inputs = mm_registry.map_input(
367 + ctx.model_config, mm_inputs, mm_processor_kwargs=mm_processor_kwargs)
368
369 assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
tests/test_inputs.py
+26
@@ -2,6 +2,7 @@ from typing import List
2
3 import pytest
4
5 +from vllm.inputs import zip_enc_dec_prompts
6 from vllm.inputs.parse import parse_and_batch_prompt
7
8 STRING_INPUTS = [
@@ -51,3 +52,28 @@ def test_parse_single_batch_token_consistent(token_input: List[int]):
52 def test_parse_single_batch_string_slice(inputs_slice: slice):
53 assert parse_and_batch_prompt(STRING_INPUTS)[inputs_slice] \
54 == parse_and_batch_prompt(STRING_INPUTS[inputs_slice])
55 +
56 +
57 +# yapf: disable
58 +@pytest.mark.parametrize('mm_processor_kwargs,expected_mm_kwargs', [
59 + (None, [{}, {}]),
60 + ({}, [{}, {}]),
61 + ({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
62 + ([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
63 +])
64 +# yapf: enable
65 +def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
66 + """Test mm_processor_kwargs init for zipping enc/dec prompts."""
67 + encoder_prompts = ['An encoder prompt', 'Another encoder prompt']
68 + decoder_prompts = ['A decoder prompt', 'Another decoder prompt']
69 + zipped_prompts = zip_enc_dec_prompts(encoder_prompts, decoder_prompts,
70 + mm_processor_kwargs)
71 + assert len(zipped_prompts) == len(encoder_prompts) == len(decoder_prompts)
72 + for enc, dec, exp_kwargs, zipped in zip(encoder_prompts, decoder_prompts,
73 + expected_mm_kwargs,
74 + zipped_prompts):
75 + assert isinstance(zipped, dict)
76 + assert len(zipped.keys()) == 3
77 + assert zipped['encoder_prompt'] == enc
78 + assert zipped['decoder_prompt'] == dec
79 + assert zipped['mm_processor_kwargs'] == exp_kwargs
tests/test_utils.py
+31 -1
@@ -7,7 +7,7 @@ from typing import AsyncIterator, Tuple
7 import pytest
8
9 from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
10 - get_open_port, merge_async_iterators)
10 + get_open_port, merge_async_iterators, supports_kw)
11
12 from .utils import error_on_warning
13
@@ -236,3 +236,33 @@ def test_no_model_tag(parser_with_config):
236 with pytest.raises(ValueError):
237 parser_with_config.parse_args(
238 ['serve', '--config', './data/test_config.yaml'])
239 +
240 +
241 +# yapf: enable
242 +@pytest.mark.parametrize(
243 + "callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
244 + [
245 + # Tests for positional argument support
246 + (lambda foo: None, "foo", True, True, False),
247 + (lambda foo: None, "foo", False, True, True),
248 + # Tests for positional or keyword / keyword only
249 + (lambda foo=100: None, "foo", True, True, False),
250 + (lambda *, foo: None, "foo", False, True, True),
251 + # Tests to make sure the names of variadic params are NOT supported
252 + (lambda *args: None, "args", False, True, False),
253 + (lambda **kwargs: None, "kwargs", False, True, False),
254 + # Tests for if we allow var kwargs to add support
255 + (lambda foo: None, "something_else", False, True, False),
256 + (lambda foo, **kwargs: None, "something_else", False, True, True),
257 + (lambda foo, **kwargs: None, "kwargs", True, True, False),
258 + (lambda foo, **kwargs: None, "foo", True, True, False),
259 + ])
260 +# yapf: disable
261 +def test_supports_kw(callable,kw_name,requires_kw_only,
262 + allow_var_kwargs,is_supported):
263 + assert supports_kw(
264 + callable=callable,
265 + kw_name=kw_name,
266 + requires_kw_only=requires_kw_only,
267 + allow_var_kwargs=allow_var_kwargs
268 + ) == is_supported
vllm/core/scheduler.py
+1
@@ -1309,6 +1309,7 @@ class Scheduler:
1309 # `multi_modal_data` will be None.
1310 multi_modal_data=seq_group.multi_modal_data
1311 if scheduler_outputs.num_prefill_groups > 0 else None,
1312 + mm_processor_kwargs=seq_group.mm_processor_kwargs,
1313 prompt_adapter_request=seq_group.prompt_adapter_request,
1314 )
1315 else:
vllm/engine/llm_engine.py
+7
@@ -811,6 +811,13 @@ class LLMEngine:
811 )
812 processed_inputs = self.input_processor(preprocessed_inputs)
813
814 + # This is a bit of a hack - copy the mm_processor_kwargs that were
815 + # used in the input processor to the processed output, since these
816 + # kwargs are presumed to be immutable and the values should be aligned
817 + # between the input processor (here) and the input mapper.
818 + processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
819 + "mm_processor_kwargs")
820 +
821 self._add_processed_request(
822 request_id=request_id,
823 processed_inputs=processed_inputs,
vllm/entrypoints/llm.py
+9
@@ -472,6 +472,7 @@ class LLM:
472 add_generation_prompt: bool = True,
473 continue_final_message: bool = False,
474 tools: Optional[List[Dict[str, Any]]] = None,
475 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
476 ) -> List[RequestOutput]:
477 """
478 Generate responses for a chat conversation.
@@ -501,6 +502,8 @@ class LLM:
502 continue_final_message: If True, continues the final message in
503 the conversation instead of starting a new one. Cannot be `True`
504 if `add_generation_prompt` is also `True`.
505 + mm_processor_kwargs: Multimodal processor kwarg overrides for this
506 + chat request. Only used for offline requests.
507
508 Returns:
509 A list of ``RequestOutput`` objects containing the generated
@@ -522,6 +525,9 @@ class LLM:
525 tokenizer = self.get_tokenizer()
526 model_config = self.llm_engine.get_model_config()
527
528 + # NOTE: _parse_chat_message_content_parts() currently doesn't
529 + # handle mm_processor_kwargs, since there is no implementation in
530 + # the chat message parsing for it.
531 conversation, mm_data = parse_chat_messages(
532 msgs, model_config, tokenizer)
533
@@ -554,6 +560,9 @@ class LLM:
560 if mm_data is not None:
561 prompt["multi_modal_data"] = mm_data
562
563 + if mm_processor_kwargs is not None:
564 + prompt["mm_processor_kwargs"] = mm_processor_kwargs
565 +
566 prompts.append(prompt)
567
568 return self.generate(
vllm/inputs/data.py
+58 -9
@@ -1,5 +1,5 @@
1 -from typing import (TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple,
2 - Union)
1 +from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
2 + Optional, Tuple, Union)
3
4 from typing_extensions import NotRequired, TypedDict, TypeVar
5
@@ -19,6 +19,14 @@ class TextPrompt(TypedDict):
19 if the model supports it.
20 """
21
22 + mm_processor_kwargs: NotRequired[Dict[str, Any]]
23 + """
24 + Optional multi-modal processor kwargs to be forwarded to the
25 + multimodal input mapper & processor. Note that if multiple modalities
26 + have registered mappers etc for the model being considered, we attempt
27 + to pass the mm_processor_kwargs to each of them.
28 + """
29 +
30
31 class TokensPrompt(TypedDict):
32 """Schema for a tokenized prompt."""
@@ -32,6 +40,14 @@ class TokensPrompt(TypedDict):
40 if the model supports it.
41 """
42
43 + mm_processor_kwargs: NotRequired[Dict[str, Any]]
44 + """
45 + Optional multi-modal processor kwargs to be forwarded to the
46 + multimodal input mapper & processor. Note that if multiple modalities
47 + have registered mappers etc for the model being considered, we attempt
48 + to pass the mm_processor_kwargs to each of them.
49 + """
50 +
51
52 SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
53 """
@@ -74,7 +90,9 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
90 according to any of the :class:`SingletonPrompt` schemas,
91 and are not required to have the same schema.
92
77 - Only the encoder prompt may have multi-modal data.
93 + Only the encoder prompt may have multi-modal data. mm_processor_kwargs
94 + should be at the top-level, and should not be set in the encoder/decoder
95 + prompts, since they are agnostic to the encoder/decoder.
96
97 Note that an :class:`ExplicitEncoderDecoderPrompt` may not
98 be used as an input to a decoder-only model,
@@ -87,6 +105,8 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
105
106 decoder_prompt: Optional[_T2_co]
107
108 + mm_processor_kwargs: NotRequired[Dict[str, Any]]
109 +
110
111 PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
112 """
@@ -121,6 +141,14 @@ class LLMInputs(TypedDict):
141 if the model supports it.
142 """
143
144 + mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
145 + """
146 + Optional multi-modal processor kwargs to be forwarded to the
147 + multimodal input mapper & processor. Note that if multiple modalities
148 + have registered mappers etc for the model being considered, we attempt
149 + to pass the mm_processor_kwargs to each of them.
150 + """
151 +
152
153 class EncoderDecoderLLMInputs(LLMInputs):
154 """
@@ -152,22 +180,43 @@ _T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
180 def build_explicit_enc_dec_prompt(
181 encoder_prompt: _T1,
182 decoder_prompt: Optional[_T2],
183 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
184 ) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
156 - return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
157 - decoder_prompt=decoder_prompt)
185 + if mm_processor_kwargs is None:
186 + mm_processor_kwargs = {}
187 + return ExplicitEncoderDecoderPrompt(
188 + encoder_prompt=encoder_prompt,
189 + decoder_prompt=decoder_prompt,
190 + mm_processor_kwargs=mm_processor_kwargs)
191
192
193 def zip_enc_dec_prompts(
194 enc_prompts: Iterable[_T1],
195 dec_prompts: Iterable[Optional[_T2]],
196 + mm_processor_kwargs: Optional[Union[Iterable[Dict[str, Any]],
197 + Dict[str, Any]]] = None,
198 ) -> List[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
199 """
200 Zip encoder and decoder prompts together into a list of
166 - :class:`ExplicitEncoderDecoderPrompt` instances.
167 - """
201 + :class:`ExplicitEncoderDecoderPrompt` instances. mm_processor_kwargs
202 + may also be provided; if a dict is passed, the same dictionary will be
203 + used for every encoder/decoder prompt. If an iterable is provided, it will
204 + be zipped with the encoder/decoder prompts.
205 + """
206 + if mm_processor_kwargs is None:
207 + mm_processor_kwargs = {}
208 + if isinstance(mm_processor_kwargs, Dict):
209 + return [
210 + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
211 + mm_processor_kwargs)
212 + for (encoder_prompt,
213 + decoder_prompt) in zip(enc_prompts, dec_prompts)
214 + ]
215 return [
169 - build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
170 - for (encoder_prompt, decoder_prompt) in zip(enc_prompts, dec_prompts)
216 + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
217 + mm_proc_kwargs)
218 + for (encoder_prompt, decoder_prompt, mm_proc_kwargs
219 + ) in zip(enc_prompts, dec_prompts, mm_processor_kwargs)
220 ]
221
222
vllm/inputs/preprocess.py
+51 -19
@@ -1,5 +1,5 @@
1 import asyncio
2 -from typing import TYPE_CHECKING, List, Optional, Tuple, Union
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
3
4 from typing_extensions import assert_never
5
@@ -20,9 +20,11 @@ if TYPE_CHECKING:
20 logger = init_logger(__name__)
21
22 PromptComponents = Tuple[Optional[str], List[int],
23 - Optional["MultiModalDataDict"]]
23 + Optional["MultiModalDataDict"], Optional[Dict[str,
24 + Any]]]
25 DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
25 - Optional["MultiModalDataDict"]]
26 + Optional["MultiModalDataDict"],
27 + Optional[Dict[str, Any]]]
28
29
30 class InputPreprocessor:
@@ -227,6 +229,7 @@ class InputPreprocessor:
229 * prompt
230 * prompt_token_ids
231 * multi_modal_data
232 + * mm_processor_kwargs (request-level input processor/mapper overrides)
233 '''
234
235 parsed = parse_singleton_prompt(prompt)
@@ -239,10 +242,12 @@ class InputPreprocessor:
242 lora_request=lora_request,
243 )
244 multi_modal_data = None
245 + mm_processor_kwargs = None
246 elif parsed["type"] == "tokens":
247 prompt_text = None
248 prompt_token_ids = parsed["content"]["prompt_token_ids"]
249 multi_modal_data = parsed["content"].get("multi_modal_data")
250 + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
251 elif parsed["type"] == "text":
252 prompt_text = parsed["content"]["prompt"]
253 prompt_token_ids = self._tokenize_prompt(
@@ -251,10 +256,12 @@ class InputPreprocessor:
256 lora_request=lora_request,
257 )
258 multi_modal_data = parsed["content"].get("multi_modal_data")
259 + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
260 else:
261 assert_never(parsed)
262
257 - return prompt_text, prompt_token_ids, multi_modal_data
263 + return (prompt_text, prompt_token_ids, multi_modal_data,
264 + mm_processor_kwargs)
265
266 async def _extract_prompt_components_async(
267 self,
@@ -273,10 +280,12 @@ class InputPreprocessor:
280 lora_request=lora_request,
281 )
282 multi_modal_data = None
283 + mm_processor_kwargs = None
284 elif parsed["type"] == "tokens":
285 prompt_text = None
286 prompt_token_ids = parsed["content"]["prompt_token_ids"]
287 multi_modal_data = parsed["content"].get("multi_modal_data")
288 + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
289 elif parsed["type"] == "text":
290 prompt_text = parsed["content"]["prompt"]
291 prompt_token_ids = await self._tokenize_prompt_async(
@@ -285,18 +294,21 @@ class InputPreprocessor:
294 lora_request=lora_request,
295 )
296 multi_modal_data = parsed["content"].get("multi_modal_data")
297 + mm_processor_kwargs = parsed["content"].get("mm_processor_kwargs")
298 else:
299 assert_never(parsed)
300
291 - return prompt_text, prompt_token_ids, multi_modal_data
301 + return (prompt_text, prompt_token_ids, multi_modal_data,
302 + mm_processor_kwargs)
303
304 def _build_enc_dec_llm_inputs(
305 self,
306 encoder_comps: PromptComponents,
307 decoder_comps: DecoderPromptComponents,
308 + mm_processor_kwargs: Dict[str, Any],
309 ) -> EncoderDecoderLLMInputs:
298 - encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
299 - decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
310 + encoder_prompt, encoder_prompt_ids, encoder_mm_data, _ = encoder_comps
311 + decoder_prompt, decoder_prompt_ids, decoder_mm_data, _ = decoder_comps
312
313 if decoder_mm_data is not None:
314 raise ValueError(
@@ -314,6 +326,7 @@ class InputPreprocessor:
326 prompt_token_ids=decoder_prompt_ids,
327 prompt=decoder_prompt,
328 multi_modal_data=decoder_mm_data,
329 + mm_processor_kwargs=mm_processor_kwargs,
330 encoder_prompt_token_ids=encoder_prompt_ids,
331 encoder_prompt=encoder_prompt,
332 encoder_multi_modal_data=encoder_mm_data,
@@ -367,21 +380,30 @@ class InputPreprocessor:
380 )
381
382 if (decoder_input := prompt["decoder_prompt"]) is None:
370 - decoder_comps = None, None, None
383 + decoder_comps = None, None, None, None
384 else:
385 decoder_comps = self._extract_prompt_components(
386 decoder_input,
387 request_id=request_id,
388 )
389 + # Handle this carefully in case it was directly initialized by user
390 + mm_processor_kwargs = prompt.get("mm_processor_kwargs", {})
391 else:
392 encoder_comps = self._extract_prompt_components(
393 prompt,
394 request_id=request_id,
395 )
381 -
382 - decoder_comps = None, None, None
383 -
384 - return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
396 + # If there are no decoder components, we assume the
397 + # mm_processor_kwargs are in the encoder prompt
398 + mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
399 + -1] is not None else {}
400 + decoder_comps = None, None, None, None
401 +
402 + return self._build_enc_dec_llm_inputs(
403 + encoder_comps,
404 + decoder_comps,
405 + mm_processor_kwargs,
406 + )
407
408 async def _process_encoder_decoder_prompt_async(
409 self,
@@ -400,7 +422,7 @@ class InputPreprocessor:
422
423 if (decoder_input := prompt["decoder_prompt"]) is None:
424 encoder_comps = await encoder_task
403 - decoder_comps = None, None, None
425 + decoder_comps = None, None, None, None
426 else:
427 decoder_task = self._extract_prompt_components_async(
428 decoder_input,
@@ -409,29 +431,39 @@ class InputPreprocessor:
431
432 encoder_comps, decoder_comps = await asyncio.gather(
433 encoder_task, decoder_task)
434 + mm_processor_kwargs = prompt["mm_processor_kwargs"]
435 else:
436 encoder_comps = await self._extract_prompt_components_async(
437 prompt,
438 request_id=request_id,
439 )
417 -
418 - decoder_comps = None, None, None
419 -
420 - return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
440 + # If there are no decoder components, we assume the
441 + # mm_processor_kwargs are in the encoder prompt
442 + mm_processor_kwargs = encoder_comps[-1] if encoder_comps[
443 + -1] is not None else {}
444 + decoder_comps = None, None, None, None
445 +
446 + return self._build_enc_dec_llm_inputs(
447 + encoder_comps,
448 + decoder_comps,
449 + mm_processor_kwargs,
450 + )
451
452 def _build_decoder_only_llm_inputs(
453 self,
454 prompt_comps: PromptComponents,
455 prompt_adapter_request: Optional[PromptAdapterRequest],
456 ) -> LLMInputs:
427 - prompt, prompt_token_ids, multi_modal_data = prompt_comps
457 + (prompt, prompt_token_ids, multi_modal_data,
458 + mm_processor_kwargs) = prompt_comps
459
460 prompt_token_ids = self._apply_prompt_adapter(
461 prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
462
463 return LLMInputs(prompt_token_ids=prompt_token_ids,
464 prompt=prompt,
434 - multi_modal_data=multi_modal_data)
465 + multi_modal_data=multi_modal_data,
466 + mm_processor_kwargs=mm_processor_kwargs)
467
468 def _process_decoder_only_prompt(
469 self,
vllm/inputs/registry.py
+10 -3
@@ -9,7 +9,8 @@ from transformers import PretrainedConfig
9 from typing_extensions import TypeVar
10
11 from vllm.logger import init_logger
12 -from vllm.utils import get_allowed_kwarg_only_overrides, print_warning_once
12 +from vllm.utils import (get_allowed_kwarg_only_overrides, print_warning_once,
13 + resolve_mm_processor_kwargs)
14
15 from .data import LLMInputs
16
@@ -293,8 +294,14 @@ class InputRegistry:
294 model_cls, _ = get_model_architecture(model_config)
295 processor = self._get_model_input_processor(model_cls)
296
296 - mm_processor_kwargs = get_allowed_kwarg_only_overrides(
297 - processor, overrides=model_config.mm_processor_kwargs)
297 + # Handle multimodal processor kwargs with priority:
298 + # Inference kwargs -> Init kwargs -> {}
299 + # If it's empty, it'll fall back to the default kwarg values
300 + mm_processor_kwargs = resolve_mm_processor_kwargs(
301 + model_config.mm_processor_kwargs,
302 + inputs.get("mm_processor_kwargs"),
303 + processor,
304 + )
305
306 return processor(InputContext(model_config), inputs,
307 **mm_processor_kwargs)
vllm/multimodal/audio.py
+2 -2
@@ -8,8 +8,8 @@ class AudioPlugin(MultiModalPlugin):
8 def get_data_key(self) -> str:
9 return "audio"
10
11 - def _default_input_mapper(self, ctx: InputContext,
12 - data: object) -> MultiModalInputs:
11 + def _default_input_mapper(self, ctx: InputContext, data: object,
12 + **mm_processor_kwargs) -> MultiModalInputs:
13 raise NotImplementedError("There is no default audio input mapper")
14
15 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
vllm/multimodal/base.py
+20 -11
@@ -1,7 +1,7 @@
1 import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 -from typing import (Callable, Dict, List, Mapping, Optional, Tuple, Type,
4 +from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type,
5 TypedDict, TypeVar, Union, cast, final)
6
7 import numpy as np
@@ -15,7 +15,7 @@ from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
17 from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18 - json_map_leaves)
18 + json_map_leaves, resolve_mm_processor_kwargs)
19
20 logger = init_logger(__name__)
21
@@ -200,6 +200,7 @@ class MultiModalPlugin(ABC):
200 self,
201 ctx: InputContext,
202 data: MultiModalData[object],
203 + **mm_processor_kwargs,
204 ) -> MultiModalInputs:
205 """
206 Return a dictionary to be passed as keyword arguments to
@@ -243,7 +244,8 @@ class MultiModalPlugin(ABC):
244 return wrapper
245
246 def map_input(self, model_config: ModelConfig,
246 - data: MultiModalData[object]) -> MultiModalInputs:
247 + data: MultiModalData[object],
248 + mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
249 """
250 Transform the data into a dictionary of model inputs using the
251 input mapper registered for that model.
@@ -263,19 +265,26 @@ class MultiModalPlugin(ABC):
265 model_cls, _ = get_model_architecture(model_config)
266
267 mapper = self._input_mappers.get(model_cls)
266 - # Only get processor kwargs at mapping time if we are not using the
267 - # input mapper; no overrides are used on the default here because they
268 - # should be passed to the huggingface resource at initialization time.
269 - if mapper is not None and mapper != self._default_input_mapper:
270 - mm_processor_kwargs = get_allowed_kwarg_only_overrides(
271 - mapper, overrides=model_config.mm_processor_kwargs)
272 - else:
273 - mm_processor_kwargs = {}
268
269 if mapper is None:
270 raise KeyError(f"No input mapper in {self} is registered for "
271 f"model class {model_cls.__name__}.")
272
273 + # In the case of the default mapper, we have to get resource
274 + # processor through its HuggingFace autoclass; since this goes
275 + # through **kwargs, we can't inspect it the same way, so we allow
276 + # drop mm_processor_kwargs based on signature inspection
277 + # if we're using the default mapper.
278 + #
279 + # This should be safe in general due to the sanitation, since the
280 + # transformers resource should filter unused kwargs anyway.
281 + uses_default_mapper = mapper == self._default_input_mapper
282 + mm_processor_kwargs = resolve_mm_processor_kwargs(
283 + model_config.mm_processor_kwargs,
284 + mm_processor_kwargs,
285 + callable=mapper,
286 + allow_var_kwargs=uses_default_mapper,
287 + )
288 return mapper(InputContext(model_config), data, **mm_processor_kwargs)
289
290 @abstractmethod
vllm/multimodal/image.py
+18 -6
@@ -1,4 +1,5 @@
1 from functools import lru_cache
2 +from typing import Any, Dict, Optional
3
4 import torch
5 from PIL import Image
@@ -23,11 +24,13 @@ class ImagePlugin(MultiModalPlugin):
24 def get_data_key(self) -> str:
25 return "image"
26
26 - def _get_hf_image_processor(self, model_config: ModelConfig):
27 - mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
28 - else model_config.mm_processor_kwargs)
29 - # We don't explicitly check kwarg overrides to the HF class
30 - # since the automodel just takes kwargs, so we can't inspect it
27 + def _get_hf_image_processor(
28 + self,
29 + model_config: ModelConfig,
30 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
31 + ):
32 + if mm_processor_kwargs is None:
33 + mm_processor_kwargs = {}
34 return cached_get_image_processor(
35 model_config.model,
36 trust_remote_code=model_config.trust_remote_code,
@@ -37,6 +40,7 @@ class ImagePlugin(MultiModalPlugin):
40 self,
41 ctx: InputContext,
42 data: MultiModalData[object],
43 + **mm_processor_kwargs,
44 ) -> MultiModalInputs:
45 model_config = ctx.model_config
46
@@ -46,12 +50,20 @@ class ImagePlugin(MultiModalPlugin):
50
51 # PIL image
52 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
49 - image_processor = self._get_hf_image_processor(model_config)
53 + image_processor = self._get_hf_image_processor(
54 + model_config,
55 + mm_processor_kwargs,
56 + )
57
58 if image_processor is None:
59 raise RuntimeError("No HuggingFace processor is available "
60 "to process the image object")
61 try:
62 + # NOTE: It may make sense to forward the mm_processor_kwargs
63 + # here too. For now, to keep it simple, we only allow it be
64 + # used for the initialization call though, just in case the
65 + # signatures of the preprocessor initializer don't match
66 + # preprocess()
67 batch_data = image_processor \
68 .preprocess(data, return_tensors="pt") \
69 .data
vllm/multimodal/registry.py
+9 -4
@@ -1,6 +1,6 @@
1 import functools
2 from collections import UserDict
3 -from typing import Dict, Mapping, Optional, Sequence
3 +from typing import Any, Dict, Mapping, Optional, Sequence
4
5 from vllm.config import ModelConfig
6 from vllm.logger import init_logger
@@ -96,8 +96,12 @@ class MultiModalRegistry:
96 """
97 return self.register_input_mapper("image", mapper)
98
99 - def map_input(self, model_config: ModelConfig,
100 - data: MultiModalDataDict) -> MultiModalInputs:
99 + def map_input(
100 + self,
101 + model_config: ModelConfig,
102 + data: MultiModalDataDict,
103 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
104 + ) -> MultiModalInputs:
105 """
106 Apply an input mapper to the data passed to the model.
107
@@ -123,7 +127,8 @@ class MultiModalRegistry:
127 f"`--limit-mm-per-prompt`, but found {num_items} items "
128 "in the same prompt.")
129
126 - input_dict = plugin.map_input(model_config, data_value)
130 + input_dict = plugin.map_input(model_config, data_value,
131 + mm_processor_kwargs)
132 for input_key, input_tensor in input_dict.items():
133 if input_key in merged_dict:
134 raise ValueError(f"The input mappers (keys={set(data)}) "
vllm/multimodal/video.py
+17 -7
@@ -1,5 +1,5 @@
1 from functools import lru_cache
2 -from typing import List, Union
2 +from typing import Any, Dict, List, Optional, Union
3
4 import numpy as np
5
@@ -36,11 +36,13 @@ class VideoPlugin(ImagePlugin):
36 def get_data_key(self) -> str:
37 return "video"
38
39 - def _get_hf_video_processor(self, model_config: ModelConfig):
40 - mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
41 - else model_config.mm_processor_kwargs)
42 - # We don't explicitly check kwarg overrides to the HF class
43 - # since the automodel just takes kwargs, so we can't inspect it
39 + def _get_hf_video_processor(
40 + self,
41 + model_config: ModelConfig,
42 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
43 + ):
44 + if mm_processor_kwargs is None:
45 + mm_processor_kwargs = {}
46 return cached_get_video_processor(
47 model_config.model,
48 trust_remote_code=model_config.trust_remote_code,
@@ -50,16 +52,24 @@ class VideoPlugin(ImagePlugin):
52 self,
53 ctx: InputContext,
54 data: MultiModalData[object],
55 + **mm_processor_kwargs,
56 ) -> MultiModalInputs:
57 model_config = ctx.model_config
58
59 # single video input as np.ndarray
60 if isinstance(data, np.ndarray):
58 - video_processor = self._get_hf_video_processor(model_config)
61 + video_processor = self._get_hf_video_processor(
62 + model_config,
63 + mm_processor_kwargs,
64 + )
65 if video_processor is None:
66 raise RuntimeError("No HuggingFace processor is available "
67 "to process the image object")
68 try:
69 + # NOTE: Similar to image; it may be a good idea to filter and
70 + # pass mm_processor_kwargs here too, but for now we don't to
71 + # avoid extra complexity if the initializer and preprocess
72 + # signatures of the processor don't align
73 batch_data = video_processor(data, return_tensors="pt").data
74 except Exception:
75 logger.error("Failed to process image (%s)", data)
vllm/sequence.py
+14
@@ -481,6 +481,10 @@ class Sequence:
481 EncoderDecoderLLMInputs,
482 inputs).get("encoder_multi_modal_data")) or {}
483
484 + @property
485 + def mm_processor_kwargs(self) -> Dict[str, Any]:
486 + return self.inputs.get("mm_processor_kwargs") or {}
487 +
488 @property
489 def lora_int_id(self) -> int:
490 return self.lora_request.lora_int_id if self.lora_request else 0
@@ -710,6 +714,14 @@ class SequenceGroup:
714 # We use the multi-modal data of an arbitrary sequence.
715 return self.seqs[0].multi_modal_data
716
717 + @property
718 + def mm_processor_kwargs(self) -> Dict[str, Any]:
719 + # As with multi-modal data, all sequences in the group should have the
720 + # same processor kwargs (i.e., mm_processor_kwargs are optionally
721 + # provided per request; note that are independent of whether the model
722 + # decoder-only or an encoder-decoder).
723 + return self.seqs[0].mm_processor_kwargs
724 +
725 @property
726 def lora_int_id(self) -> int:
727 return self.lora_request.lora_int_id if self.lora_request else 0
@@ -949,6 +961,7 @@ class SequenceGroupMetadata(
961 used in prefix caching.
962 state: Internal state tied to this sequence group.
963 multi_modal_data: Multi modal data.
964 + mm_processor_kwargs: Multimodal input processor / mapper overrides.
965 encoder_seq_data: Optional sequence data for encoder prompt
966 (SequenceGroup.encoder_seq). Should be None
967 unless you are working with an encoder/decoder
@@ -975,6 +988,7 @@ class SequenceGroupMetadata(
988 # "MultiModalDataDict" types. We have to use Any due to msgspec
989 # doesn't allow to have union of 2 different dicts.
990 multi_modal_data: Optional[Any] = None
991 + mm_processor_kwargs: Optional[Dict[str, Any]] = None
992 encoder_seq_data: Optional[SequenceData] = None
993 cross_block_table: Optional[List[int]] = None
994 prompt_adapter_request: Optional[PromptAdapterRequest] = None
vllm/utils.py
+82 -13
@@ -1277,18 +1277,87 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1277 return await task(*args, **kwargs)
1278
1279
1280 -def supports_kw(callable: Callable[..., object], kw_name: str) -> bool:
1280 +def supports_kw(
1281 + callable: Callable[..., object],
1282 + kw_name: str,
1283 + requires_kw_only: bool = False,
1284 + allow_var_kwargs: bool = True,
1285 +) -> bool:
1286 + """Check if a keyword is a valid kwarg for a callable; if requires_kw_only
1287 + disallows kwargs names that can also be positional arguments.
1288 + """
1289 params = inspect.signature(callable).parameters
1282 - if kw_name in params:
1283 - return True
1290 + if not params:
1291 + return False
1292 +
1293 + param_val = params.get(kw_name)
1294 +
1295 + # Types where the it may be valid, i.e., explicitly defined & nonvariadic
1296 + passable_kw_types = set((inspect.Parameter.POSITIONAL_ONLY,
1297 + inspect.Parameter.POSITIONAL_OR_KEYWORD,
1298 + inspect.Parameter.KEYWORD_ONLY))
1299 +
1300 + if param_val:
1301 + is_sig_param = param_val.kind in passable_kw_types
1302 + # We want kwargs only, but this is passable as a positional arg
1303 + if (requires_kw_only and is_sig_param
1304 + and param_val.kind != inspect.Parameter.KEYWORD_ONLY):
1305 + return False
1306 + if ((requires_kw_only
1307 + and param_val.kind == inspect.Parameter.KEYWORD_ONLY)
1308 + or (not requires_kw_only and is_sig_param)):
1309 + return True
1310 +
1311 + # If we're okay with var-kwargs, it's supported as long as
1312 + # the kw_name isn't something like *args, **kwargs
1313 + if allow_var_kwargs:
1314 + # Get the last param; type is ignored here because params is a proxy
1315 + # mapping, but it wraps an ordered dict, and they appear in order.
1316 + # Ref: https://docs.python.org/3/library/inspect.html#inspect.Signature.parameters
1317 + last_param = params[next(reversed(params))] # type: ignore
1318 + return (last_param.kind == inspect.Parameter.VAR_KEYWORD
1319 + and last_param.name != kw_name)
1320 + return False
1321 +
1322 +
1323 +def resolve_mm_processor_kwargs(
1324 + init_kwargs: Optional[Dict[str, Any]],
1325 + inference_kwargs: Optional[Dict[str, Any]],
1326 + callable: Callable[..., object],
1327 + allow_var_kwargs: bool = False,
1328 +) -> Dict[str, Any]:
1329 + """Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
1330 + those who are not explicit keywords to the given callable (of one is
1331 + given; otherwise no filtering is done), then merges the kwarg dicts,
1332 + giving priority to inference_kwargs if there are any collisions.
1333 +
1334 + In the case that no kwarg overrides are provided, returns an empty
1335 + dict so that it can still be kwarg expanded into the callable later on.
1336 +
1337 + If allow_var_kwargs=True, allows for things that can be expanded into
1338 + kwargs as long as they aren't naming collision for var_kwargs or potential
1339 + positional arguments.
1340 + """
1341 + # Filter inference time multimodal processor kwargs provided
1342 + runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
1343 + callable,
1344 + overrides=inference_kwargs,
1345 + allow_var_kwargs=allow_var_kwargs)
1346 +
1347 + # Filter init time multimodal processor kwargs provided
1348 + init_mm_kwargs = get_allowed_kwarg_only_overrides(
1349 + callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs)
1350
1285 - return any(param.kind == inspect.Parameter.VAR_KEYWORD
1286 - for param in params.values())
1351 + # Merge the final processor kwargs, prioritizing inference
1352 + # time values over the initialization time values.
1353 + mm_processor_kwargs = {**init_mm_kwargs, **runtime_mm_kwargs}
1354 + return mm_processor_kwargs
1355
1356
1357 def get_allowed_kwarg_only_overrides(
1358 callable: Callable[..., object],
1359 overrides: Optional[Dict[str, Any]],
1360 + allow_var_kwargs: bool = False,
1361 ) -> Dict[str, Any]:
1362 """
1363 Given a callable which has one or more keyword only params and a dict
@@ -1300,7 +1369,9 @@ def get_allowed_kwarg_only_overrides(
1369
1370 Args:
1371 callable: Callable which takes 0 or more keyword only arguments.
1372 + If None is provided, all overrides names are allowed.
1373 overrides: Potential overrides to be used when invoking the callable.
1374 + allow_var_kwargs: Allows overrides that are expandable for var kwargs.
1375
1376 Returns:
1377 Dictionary containing the kwargs to be leveraged which may be used
@@ -1310,17 +1381,15 @@ def get_allowed_kwarg_only_overrides(
1381 if not overrides:
1382 return {}
1383
1313 - allowed_override_names = [
1314 - name for name, param in inspect.signature(callable).parameters.items()
1315 - if param.kind == inspect.Parameter.KEYWORD_ONLY
1316 - ]
1317 -
1318 - # Drop any mm_processor_kwargs provided by the user that are
1319 - # not kwarg names accepted by the provided input processor.
1384 + # Drop any mm_processor_kwargs provided by the user that
1385 + # are not kwargs, unless it can fit it var_kwargs param
1386 filtered_overrides = {
1387 kwarg_name: val
1388 for kwarg_name, val in overrides.items()
1323 - if kwarg_name in allowed_override_names
1389 + if supports_kw(callable,
1390 + kwarg_name,
1391 + requires_kw_only=True,
1392 + allow_var_kwargs=allow_var_kwargs)
1393 }
1394
1395 # If anything is dropped, log a warning
vllm/worker/cpu_model_runner.py
+5 -3
@@ -148,8 +148,9 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
148 )
149
150 def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
151 - computed_len: int):
152 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
151 + computed_len: int,
152 + mm_processor_kwargs: Dict[str, Any]):
153 + mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
154
155 # special processing for mrope position deltas.
156 mrope_positions = None
@@ -210,7 +211,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
211 mrope_positions = None
212 if (mm_data := seq_group_metadata.multi_modal_data):
213 mm_kwargs, mrope_positions = self._compute_multi_modal_input(
213 - seq_data, mm_data, computed_len)
214 + seq_data, mm_data, computed_len,
215 + seq_group_metadata.mm_processor_kwargs)
216 multi_modal_inputs_list.append(mm_kwargs)
217
218 # Token position ids
vllm/worker/model_runner.py
+3 -1
@@ -640,7 +640,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
640 if not mm_data:
641 return
642
643 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
643 + mm_kwargs = self.multi_modal_input_mapper(
644 + mm_data,
645 + mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
646 inter_data.multi_modal_inputs = mm_kwargs
647
648 # special processing for mrope position deltas.
vllm/worker/neuron_model_runner.py
+4 -1
@@ -153,7 +153,10 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
153 mm_data = seq_group_metadata.multi_modal_data
154 if mm_data:
155 # Process multi-modal data
156 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
156 + mm_kwargs = self.multi_modal_input_mapper(
157 + mm_data,
158 + mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs,
159 + )
160 multi_modal_inputs_list.append(mm_kwargs)
161
162 max_seq_len = max(seq_lens)
vllm/worker/openvino_model_runner.py
+5 -1
@@ -172,7 +172,11 @@ class OpenVINOModelRunner:
172
173 mm_data = seq_group_metadata.multi_modal_data
174 if mm_data:
175 - mm_kwargs = self.multi_modal_input_mapper(mm_data)
175 + mm_kwargs = self.multi_modal_input_mapper(
176 + mm_data,
177 + mm_processor_kwargs=seq_group_metadata.
178 + mm_processor_kwargs,
179 + )
180 multi_modal_inputs_list.append(mm_kwargs)
181
182 block_table = seq_group_metadata.block_tables[seq_id]