A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core][Frontend] Support Passing Multimodal Processor Kwargs (#8657)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Sep 23, 2024 at 01:44 UTC
9b8c8ba1198cbcd311d28b7647f0f8d5dcdc9212
16 files changed
+589
-116
tests/engine/test_arg_utils.py
+21
@@ -40,3 +40,24 @@ def test_limit_mm_per_prompt_parser(arg, expected):
40
def test_bad_nullable_kvs(arg):
41
with pytest.raises(ArgumentTypeError):
42
nullable_kvs(arg)
43
+
44
+
45
+@pytest.mark.parametrize(("arg", "expected"), [
46
+ (None, None),
47
+ ("{}", {}),
48
+ ('{"num_crops": 4}', {
49
+ "num_crops": 4
50
+ }),
51
+ ('{"foo": {"bar": "baz"}}', {
52
+ "foo": {
53
+ "bar": "baz"
54
+ }
55
+ }),
56
+])
57
+def test_mm_processor_kwargs_prompt_parser(arg, expected):
58
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
59
+ if arg is None:
60
+ args = parser.parse_args([])
61
+ else:
62
+ args = parser.parse_args(["--mm-processor-kwargs", arg])
63
+ assert args.mm_processor_kwargs == expected
tests/models/decoder_only/vision_language/test_qwen.py
+1
-28
@@ -5,14 +5,13 @@ import pytest
5
import torch
6
from PIL.Image import Image
7
8
-from vllm.config import ModelConfig
8
from vllm.inputs import InputContext, LLMInputs
9
from vllm.multimodal.base import MultiModalInputs
10
from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
11
12
from ....conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
13
VllmRunner, _ImageAssets)
15
-from ...utils import check_logprobs_close
14
+from ...utils import build_model_context, check_logprobs_close
15
16
text_only_models = [
17
"Qwen/Qwen-7B-Chat" # Has no visual component
@@ -42,32 +41,6 @@ VIS_ENC_DIM = 4096
41
IMG_SIZE = 448
42
43
45
-def build_model_context(model_name: str,
46
- tokenizer_name: Optional[str] = None,
47
- trust_remote_code: bool = False):
48
- """Creates an InputContext for a given model.
49
-
50
- Args:
51
- model_name: Name of the model being considered.
52
- tokenizer_name: Name of the tokenizer being considered.
53
- trust_remote_code: Whether or not to allow loading remote code.
54
-
55
- Returns:
56
- InputContext for the model being considered.
57
- """
58
- if tokenizer_name is None:
59
- tokenizer_name = model_name
60
- model_config = ModelConfig(
61
- model_name,
62
- tokenizer_name,
63
- tokenizer_mode="auto",
64
- trust_remote_code=trust_remote_code,
65
- dtype="float32",
66
- seed=0,
67
- )
68
- return InputContext(model_config)
69
-
70
-
44
@pytest.fixture()
45
def input_mapper_for_qwen():
46
# Lazy import to avoid initializing CUDA during test collection
tests/models/utils.py
+35
@@ -1,6 +1,8 @@
1
import warnings
2
from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
+from vllm.config import ModelConfig
5
+from vllm.inputs import InputContext
6
from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
7
8
TokensText = Tuple[List[int], str]
@@ -240,3 +242,36 @@ def check_logprobs_close(
242
warnings.simplefilter("always")
243
244
warnings.warn(fail_msg, stacklevel=2)
245
+
246
+
247
+def build_model_context(model_name: str,
248
+ tokenizer_name: Optional[str] = None,
249
+ trust_remote_code: bool = False,
250
+ mm_processor_kwargs: Optional[Dict] = None,
251
+ limit_mm_per_prompt: Optional[Dict] = None):
252
+ """Creates an InputContext for a given model.
253
+
254
+ Args:
255
+ model_name: Name of the model being considered.
256
+ tokenizer_name: Name of the tokenizer being considered.
257
+ trust_remote_code: Whether or not to allow loading remote code.
258
+ mm_processor_kwargs: optional processor kwargs for to be leveraged
259
+ in the input processor, mapper, dummy data creation, etc.
260
+ limit_mm_per_prompt: Multimodal limits.
261
+
262
+ Returns:
263
+ InputContext for the model being considered.
264
+ """
265
+ if tokenizer_name is None:
266
+ tokenizer_name = model_name
267
+ model_config = ModelConfig(
268
+ model_name,
269
+ tokenizer_name,
270
+ tokenizer_mode="auto",
271
+ trust_remote_code=trust_remote_code,
272
+ dtype="float32",
273
+ seed=0,
274
+ mm_processor_kwargs=mm_processor_kwargs,
275
+ limit_mm_per_prompt=limit_mm_per_prompt,
276
+ )
277
+ return InputContext(model_config)
tests/multimodal/test_processor_kwargs.py
new
+339
@@ -0,0 +1,339 @@
1
+from array import array
2
+from typing import Mapping
3
+from unittest.mock import patch
4
+
5
+import pytest
6
+import torch
7
+
8
+from vllm.inputs import InputContext, LLMInputs
9
+from vllm.inputs.registry import InputRegistry
10
+from vllm.multimodal import MultiModalRegistry
11
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12
+
13
+from ..models.utils import build_model_context
14
+
15
+# Used for fast tests where the model doesn't matter
16
+DUMMY_MODEL_ID = "facebook/opt-125m"
17
+# Used for tests that need a multimodal model
18
+MULTIMODAL_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
19
+
20
+# For mm_processor_kwargs - we test overrides by defining mocks for each place
21
+# it is used, and ensuring that we can pass processor kwargs an override value
22
+# to receive the intended result for things like sequence length etc.
23
+DEFAULT_NUM_CROPS = 4
24
+NUM_CROPS_OVERRIDE = 16
25
+
26
+
27
+# Mocks for all of the places that we use the mm_processor_kwargs
28
+# to override values in different callables
29
+@pytest.fixture
30
+def use_processor_mock():
31
+ """Patches the internal model input processor with an override callable."""
32
+
33
+ def custom_processor(ctx: InputContext,
34
+ llm_inputs: LLMInputs,
35
+ *,
36
+ num_crops=DEFAULT_NUM_CROPS):
37
+ # For testing purposes, we don't worry about the llm inputs / return
38
+ # type validation, and just return the value of the kwarg that we
39
+ # clobber.
40
+ return num_crops
41
+
42
+ with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
43
+ return_value=custom_processor):
44
+ yield
45
+
46
+
47
+@pytest.fixture
48
+def use_dummy_data_mock():
49
+ """Patches the internal model input processor with an override callable."""
50
+
51
+ def custom_dummy_data_factory(self,
52
+ ctx: InputContext,
53
+ seq_len: int,
54
+ mm_counts: Mapping[str, int],
55
+ *,
56
+ num_crops=DEFAULT_NUM_CROPS):
57
+ seq_data = SequenceData(
58
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
59
+ return seq_data, None
60
+
61
+ with patch(
62
+ "vllm.inputs.registry.InputRegistry._default_dummy_data_factory",
63
+ custom_dummy_data_factory):
64
+ yield
65
+
66
+
67
+# Lazy import to avoid CUDA reinitialization error
68
+def mm_model_cls():
69
+ from vllm.model_executor.models.phi3v import Phi3VForCausalLM
70
+
71
+ return Phi3VForCausalLM
72
+
73
+
74
+# lambda whose signature matches max token calcs extra & mapper + extra kwargs
75
+get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
76
+custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
77
+ "num_pixels": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
78
+}
79
+
80
+
81
+### Test for default processor logic & mm_processor_kwargs wrapping
82
+def test_default_processor_is_a_noop():
83
+ """Ensure that by default, there is no processor override."""
84
+ dummy_registry = InputRegistry()
85
+ ctx = build_model_context(DUMMY_MODEL_ID)
86
+ processor = dummy_registry.create_input_processor(ctx.model_config)
87
+ proc_inputs = LLMInputs(prompt_token_ids=[], prompt="")
88
+ proc_outputs = processor(inputs=proc_inputs)
89
+ assert proc_inputs is proc_outputs
90
+
91
+
92
+@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
93
+def test_processor_default_kwargs(use_processor_mock, num_crops):
94
+ """Ensure input processors can use processor kwargs."""
95
+ dummy_registry = InputRegistry()
96
+ # If we have a value for num_crops, pass the override value and make
97
+ # sure we get that value as a return-value from out mock processor,
98
+ # otherwise fall back to the default value
99
+ mm_processor_kwargs = None if num_crops is None else {
100
+ "num_crops": num_crops
101
+ }
102
+ expected_num_crops = DEFAULT_NUM_CROPS if num_crops is None else num_crops
103
+ ctx = build_model_context(DUMMY_MODEL_ID,
104
+ mm_processor_kwargs=mm_processor_kwargs)
105
+ processor = dummy_registry.create_input_processor(ctx.model_config)
106
+
107
+ num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
108
+ assert num_crops_val == expected_num_crops
109
+
110
+
111
+@pytest.mark.parametrize(
112
+ "mm_processor_kwargs",
113
+ [
114
+ # Not part of the signature
115
+ {
116
+ "does_not_exist": 100
117
+ },
118
+ # Part of the signature, not keyword only
119
+ {
120
+ "ctx": "something bad"
121
+ }
122
+ ])
123
+def test_processor_with_sad_kwarg_overrides(use_processor_mock,
124
+ mm_processor_kwargs):
125
+ """Ensure that input processors filter out invalid mm_processor_kwargs"""
126
+ dummy_registry = InputRegistry()
127
+ ctx = build_model_context(DUMMY_MODEL_ID,
128
+ mm_processor_kwargs=mm_processor_kwargs)
129
+
130
+ processor = dummy_registry.create_input_processor(ctx.model_config)
131
+ num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
132
+ assert num_crops_val == DEFAULT_NUM_CROPS
133
+
134
+
135
+### Test overrides for the dummy data
136
+@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
137
+def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
138
+ """Ensure dummy data factories can use processor kwargs."""
139
+ mm_processor_kwargs = None if num_crops is None else {
140
+ "num_crops": num_crops
141
+ }
142
+ expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
143
+ dummy_registry = InputRegistry()
144
+ ctx = build_model_context(DUMMY_MODEL_ID,
145
+ mm_processor_kwargs=mm_processor_kwargs)
146
+ mm_registry = MultiModalRegistry()
147
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
148
+
149
+ # NOTE: seq_len is thrown away here since this will leverage the
150
+ # default dummy data factory that we have patched in, whose seq
151
+ # len is solely dependent on the value of the mm_processor_kwargs.
152
+ seq_data, _ = dummy_registry.dummy_data_for_profiling(
153
+ ctx.model_config, seq_len=-1, mm_registry=mm_registry)
154
+ assert len(seq_data.prompt_token_ids) == expected_seq_count
155
+
156
+
157
+@pytest.mark.parametrize(
158
+ "mm_processor_kwargs",
159
+ [
160
+ # Not part of the signature
161
+ {
162
+ "does_not_exist": 100
163
+ },
164
+ # Part of the signature, not keyword only
165
+ {
166
+ "ctx": "something bad"
167
+ }
168
+ ])
169
+def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
170
+ mm_processor_kwargs):
171
+ """Ensure the dummy data factory filters out invalid mm_processor_kwargs"""
172
+ dummy_registry = InputRegistry()
173
+ ctx = build_model_context(DUMMY_MODEL_ID,
174
+ mm_processor_kwargs=mm_processor_kwargs)
175
+ mm_registry = MultiModalRegistry()
176
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
177
+
178
+ # NOTE: seq_len is thrown away here since this will leverage the
179
+ # default dummy data factory that we have patched in, whose seq
180
+ # len is solely dependent on the value of the mm_processor_kwargs.
181
+ seq_data, _ = dummy_registry.dummy_data_for_profiling(
182
+ ctx.model_config, seq_len=-1, mm_registry=mm_registry)
183
+ assert len(seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
184
+
185
+
186
+### Test overrides for the max token count per multimodal instance
187
+@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
188
+def test_max_tokens_kwarg_overrides(num_crops):
189
+ """Ensure max token calcs can use processor kwargs."""
190
+ mm_processor_kwargs = None if num_crops is None else {
191
+ "num_crops": num_crops
192
+ }
193
+ expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
194
+
195
+ ctx = build_model_context(MULTIMODAL_MODEL_ID,
196
+ trust_remote_code=True,
197
+ mm_processor_kwargs=mm_processor_kwargs,
198
+ limit_mm_per_prompt={"image": 1})
199
+
200
+ mm_registry = MultiModalRegistry()
201
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
202
+ # Patch the image registry for phi3v with our lambda that is compatible
203
+ # with overrides, then ensure that calling the method correctly echos
204
+ # our num_crops value back from the mm_processor_kwargs.
205
+ with patch.object(
206
+ mm_registry._get_plugin("image"),
207
+ "_max_mm_tokens",
208
+ {mm_model_cls(): get_num_crops},
209
+ ):
210
+ max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
211
+ ctx.model_config)
212
+
213
+ assert expected_seq_count == max_multimodal_tokens
214
+
215
+
216
+@pytest.mark.parametrize(
217
+ "mm_processor_kwargs",
218
+ [
219
+ # Not part of the signature
220
+ {
221
+ "does_not_exist": 100
222
+ },
223
+ # Part of the signature, not keyword only
224
+ {
225
+ "ctx": "something bad"
226
+ }
227
+ ])
228
+def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
229
+ """Ensure that max token calcs filters out invalid mm_processor_kwargs"""
230
+ ctx = build_model_context(MULTIMODAL_MODEL_ID,
231
+ trust_remote_code=True,
232
+ mm_processor_kwargs=mm_processor_kwargs,
233
+ limit_mm_per_prompt={"image": 1})
234
+
235
+ mm_registry = MultiModalRegistry()
236
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
237
+
238
+ # Similar before, but since these kwargs get filtered,
239
+ # we always get our default value back.
240
+ with patch.object(
241
+ mm_registry._get_plugin("image"),
242
+ "_max_mm_tokens",
243
+ {mm_model_cls(): get_num_crops},
244
+ ):
245
+ max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
246
+ ctx.model_config)
247
+
248
+ assert max_multimodal_tokens == DEFAULT_NUM_CROPS
249
+
250
+
251
+### Test overrides for the mapper
252
+@pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
253
+def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
254
+ """Ensure that the mapper processor kwargs can fall back to HF models."""
255
+ # NOTE - we don't validate bad inputs for the default mapper, because it's
256
+ # through the automodel interface in transformers, so we can't easily
257
+ # inspect what kwargs are or are not allowed.
258
+ ctx = build_model_context(MULTIMODAL_MODEL_ID,
259
+ trust_remote_code=True,
260
+ mm_processor_kwargs={"num_crops": num_crops},
261
+ limit_mm_per_prompt={"image": 1})
262
+
263
+ mm_registry = MultiModalRegistry()
264
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
265
+
266
+ image = image_assets[0].pil_image
267
+ mm_inputs = {"image": image}
268
+
269
+ mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
270
+ # Phi3v pixel vals should have shape: [batch, num_crops+1, 3, 336, 336]
271
+ assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
272
+
273
+
274
+@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
275
+def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
276
+ """Ensure custom mappers can use processor kwargs."""
277
+ mm_processor_kwargs = None if num_crops is None else {
278
+ "num_crops": num_crops
279
+ }
280
+ expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
281
+ ctx = build_model_context(MULTIMODAL_MODEL_ID,
282
+ trust_remote_code=True,
283
+ mm_processor_kwargs=mm_processor_kwargs,
284
+ limit_mm_per_prompt={"image": 1})
285
+
286
+ mm_registry = MultiModalRegistry()
287
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
288
+ # Patch the image registry for phi3v with our lambda that is compatible
289
+ # with overrides, then ensure that calling the method correctly echos
290
+ # our num_crops value back from the mm_processor_kwargs.
291
+ image = image_assets[0].pil_image
292
+ mm_inputs = {"image": image}
293
+
294
+ with patch.object(
295
+ mm_registry._get_plugin("image"),
296
+ "_default_input_mapper",
297
+ {mm_model_cls(): custom_mapper},
298
+ ):
299
+ mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
300
+
301
+ assert mapped_inputs["pixel_values"].shape[1] == expected_seq_count + 1
302
+
303
+
304
+@pytest.mark.parametrize(
305
+ "mm_processor_kwargs",
306
+ [
307
+ # Not part of the signature
308
+ {
309
+ "does_not_exist": 100
310
+ },
311
+ # Part of the signature, not keyword only
312
+ {
313
+ "ctx": "something bad"
314
+ }
315
+ ])
316
+def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
317
+ mm_processor_kwargs):
318
+ """Ensure that custom mappers filters out invalid mm_processor_kwargs"""
319
+ ctx = build_model_context(MULTIMODAL_MODEL_ID,
320
+ trust_remote_code=True,
321
+ mm_processor_kwargs=mm_processor_kwargs,
322
+ limit_mm_per_prompt={"image": 1})
323
+
324
+ mm_registry = MultiModalRegistry()
325
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
326
+ # Patch the image registry for phi3v with our lambda that is compatible
327
+ # with overrides, then ensure that calling the method correctly echos
328
+ # our num_crops value back from the mm_processor_kwargs.
329
+ image = image_assets[0].pil_image
330
+ mm_inputs = {"image": image}
331
+
332
+ with patch.object(
333
+ mm_registry._get_plugin("image"),
334
+ "_default_input_mapper",
335
+ {mm_model_cls(): custom_mapper},
336
+ ):
337
+ mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
338
+
339
+ assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
vllm/config.py
+5
-1
@@ -122,6 +122,8 @@ class ModelConfig:
122
can not be gathered from the vllm arguments.
123
config_format: The config format which shall be loaded.
124
Defaults to 'auto' which defaults to 'hf'.
125
+ mm_processor_kwargs: Arguments to be forwarded to the model's processor
126
+ for multi-modal data, e.g., image processor.
127
"""
128
129
def __init__(self,
@@ -150,7 +152,8 @@ class ModelConfig:
152
limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
153
use_async_output_proc: bool = True,
154
override_neuron_config: Optional[Dict[str, Any]] = None,
153
- config_format: ConfigFormat = ConfigFormat.AUTO) -> None:
155
+ config_format: ConfigFormat = ConfigFormat.AUTO,
156
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None:
157
self.model = model
158
self.tokenizer = tokenizer
159
self.tokenizer_mode = tokenizer_mode
@@ -184,6 +187,7 @@ class ModelConfig:
187
self.model, revision)
188
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
189
self.use_async_output_proc = use_async_output_proc
190
+ self.mm_processor_kwargs = mm_processor_kwargs
191
192
# Set enforce_eager to False if the value is unset.
193
if self.enforce_eager is None:
vllm/engine/arg_utils.py
+8
@@ -175,6 +175,7 @@ class EngineArgs:
175
collect_detailed_traces: Optional[str] = None
176
disable_async_output_proc: bool = False
177
override_neuron_config: Optional[Dict[str, Any]] = None
178
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None
179
180
def __post_init__(self):
181
if self.tokenizer is None:
@@ -513,6 +514,12 @@ class EngineArgs:
514
'e.g.: `image=16,video=2` allows a maximum of 16 '
515
'images and 2 videos per prompt. Defaults to 1 for '
516
'each modality.'))
517
+ parser.add_argument(
518
+ '--mm-processor-kwargs',
519
+ default=None,
520
+ type=json.loads,
521
+ help=('Overrides for the multimodal input mapping/processing,'
522
+ 'e.g., image processor. For example: {"num_crops": 4}.'))
523
524
# LoRA related configs
525
parser.add_argument('--enable-lora',
@@ -822,6 +829,7 @@ class EngineArgs:
829
use_async_output_proc=not self.disable_async_output_proc,
830
override_neuron_config=self.override_neuron_config,
831
config_format=self.config_format,
832
+ mm_processor_kwargs=self.mm_processor_kwargs,
833
)
834
835
def create_load_config(self) -> LoadConfig:
vllm/engine/llm_engine.py
+2
-1
@@ -235,7 +235,7 @@ class LLMEngine:
235
"decoding_config=%r, observability_config=%r, "
236
"seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
237
"num_scheduler_steps=%d, enable_prefix_caching=%s, "
238
- "use_async_output_proc=%s)",
238
+ "use_async_output_proc=%s, mm_processor_kwargs=%s)",
239
VLLM_VERSION,
240
model_config.model,
241
speculative_config,
@@ -268,6 +268,7 @@ class LLMEngine:
268
scheduler_config.num_scheduler_steps,
269
cache_config.enable_prefix_caching,
270
model_config.use_async_output_proc,
271
+ model_config.mm_processor_kwargs,
272
)
273
# TODO(woosuk): Print more configs in debug mode.
274
from vllm.plugins import load_general_plugins
vllm/entrypoints/llm.py
+2
@@ -134,6 +134,7 @@ class LLM:
134
max_seq_len_to_capture: int = 8192,
135
disable_custom_all_reduce: bool = False,
136
disable_async_output_proc: bool = False,
137
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
138
**kwargs,
139
) -> None:
140
'''
@@ -174,6 +175,7 @@ class LLM:
175
max_seq_len_to_capture=max_seq_len_to_capture,
176
disable_custom_all_reduce=disable_custom_all_reduce,
177
disable_async_output_proc=disable_async_output_proc,
178
+ mm_processor_kwargs=mm_processor_kwargs,
179
**kwargs,
180
)
181
self.llm_engine = LLMEngine.from_engine_args(
vllm/inputs/registry.py
+27
-11
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
9
from typing_extensions import TypeVar
10
11
from vllm.logger import init_logger
12
+from vllm.utils import get_allowed_kwarg_only_overrides
13
14
from .data import LLMInputs
15
@@ -68,12 +69,17 @@ class DummyDataFactory(Protocol):
69
ctx: InputContext,
70
seq_len: int,
71
mm_counts: Mapping[str, int],
72
+ **mm_processor_kwargs: Any,
73
) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
74
"""
75
Create dummy data to be inputted into the model.
76
77
Note:
78
:data:`InputProcessor` is not applied to the dummy data.
79
+
80
+ The :code:`mm_processor_kwargs` are overrides provided at
81
+ initialization time to values in the config whose values
82
+ may affect the number of tokens per instance.
83
"""
84
...
85
@@ -152,6 +158,10 @@ class InputRegistry:
158
159
return wrapper
160
161
+ def _get_dummy_data_factory(self, model_cls: Type[nn.Module]):
162
+ return self._dummy_factories_by_model_type \
163
+ .get(model_cls, self._default_dummy_data_factory)
164
+
165
def dummy_data_for_profiling(
166
self,
167
model_config: "ModelConfig",
@@ -174,15 +184,15 @@ class InputRegistry:
184
from vllm.model_executor.model_loader import get_model_architecture
185
186
model_cls, _ = get_model_architecture(model_config)
177
- dummy_factory = self._dummy_factories_by_model_type \
178
- .get(model_cls, self._default_dummy_data_factory)
187
+ dummy_factory = self._get_dummy_data_factory(model_cls)
188
+
189
mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
190
+ mm_processor_kwargs = get_allowed_kwarg_only_overrides(
191
+ dummy_factory, overrides=model_config.mm_processor_kwargs)
192
181
- seq_data, mm_data = dummy_factory(
182
- InputContext(model_config),
183
- seq_len,
184
- _MultiModalCounts(mm_counts),
185
- )
193
+ seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len,
194
+ _MultiModalCounts(mm_counts),
195
+ **mm_processor_kwargs)
196
197
# Having more tokens is over-conservative but otherwise fine
198
num_tokens = seq_data.prompt_token_ids
@@ -229,6 +239,10 @@ class InputRegistry:
239
240
return wrapper
241
242
+ def _get_model_input_processor(self, model_cls: Type[nn.Module]):
243
+ return self._input_processors_by_model_type \
244
+ .get(model_cls, self._default_input_processor)
245
+
246
def process_input(self, model_config: "ModelConfig",
247
inputs: LLMInputs) -> LLMInputs:
248
"""
@@ -243,15 +257,17 @@ class InputRegistry:
257
from vllm.model_executor.model_loader import get_model_architecture
258
259
model_cls, _ = get_model_architecture(model_config)
260
+ processor = self._get_model_input_processor(model_cls)
261
247
- processor = self._input_processors_by_model_type \
248
- .get(model_cls, self._default_input_processor)
262
+ mm_processor_kwargs = get_allowed_kwarg_only_overrides(
263
+ processor, overrides=model_config.mm_processor_kwargs)
264
250
- return processor(InputContext(model_config), inputs)
265
+ return processor(InputContext(model_config), inputs,
266
+ **mm_processor_kwargs)
267
268
def create_input_processor(self, model_config: "ModelConfig"):
269
"""
254
- Create an input processor (see :meth:`process_input`) for a
270
+ Create an input processor (see :meth:`_process_input`) for a
271
specific model.
272
"""
273
return functools.partial(self.process_input, model_config)
vllm/multimodal/base.py
+16
-3
@@ -14,7 +14,8 @@ from typing_extensions import TypeAlias
14
from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
17
-from vllm.utils import JSONTree, is_list_of, json_map_leaves
17
+from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18
+ json_map_leaves)
19
20
logger = init_logger(__name__)
21
@@ -256,11 +257,20 @@ class MultiModalPlugin(ABC):
257
model_cls, _ = get_model_architecture(model_config)
258
259
mapper = self._input_mappers.get(model_cls)
260
+ # Only get processor kwargs at mapping time if we are not using the
261
+ # input mapper; no overrides are used on the default here because they
262
+ # should be passed to the huggingface resource at initialization time.
263
+ if mapper is not None and mapper != self._default_input_mapper:
264
+ mm_processor_kwargs = get_allowed_kwarg_only_overrides(
265
+ mapper, overrides=model_config.mm_processor_kwargs)
266
+ else:
267
+ mm_processor_kwargs = {}
268
+
269
if mapper is None:
270
raise KeyError(f"No input mapper in {self} is registered for "
271
f"model class {model_cls.__name__}.")
272
263
- return mapper(InputContext(model_config), data)
273
+ return mapper(InputContext(model_config), data, **mm_processor_kwargs)
274
275
@abstractmethod
276
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
@@ -333,7 +343,10 @@ class MultiModalPlugin(ABC):
343
f"for model class {model_cls.__name__} in {self}.")
344
345
if callable(max_mm_tokens):
336
- max_mm_tokens = max_mm_tokens(InputContext(model_config))
346
+ mm_processor_kwargs = get_allowed_kwarg_only_overrides(
347
+ max_mm_tokens, overrides=model_config.mm_processor_kwargs)
348
+ max_mm_tokens = max_mm_tokens(InputContext(model_config),
349
+ **mm_processor_kwargs)
350
351
self._validate_max_multimodal_tokens(max_mm_tokens)
352
vllm/multimodal/image.py
+8
-2
@@ -6,7 +6,7 @@ from PIL import Image
6
from vllm.config import ModelConfig
7
from vllm.inputs.registry import InputContext
8
from vllm.logger import init_logger
9
-from vllm.transformers_utils.image_processor import get_image_processor
9
+from vllm.transformers_utils.processor import get_image_processor
10
from vllm.utils import is_list_of
11
12
from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
@@ -23,9 +23,14 @@ class ImagePlugin(MultiModalPlugin):
23
return "image"
24
25
def _get_hf_image_processor(self, model_config: ModelConfig):
26
+ mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
27
+ else model_config.mm_processor_kwargs)
28
+ # We don't explicitly check kwarg overrides to the HF class
29
+ # since the automodel just takes kwargs, so we can't inspect it
30
return cached_get_image_processor(
31
model_config.model,
28
- trust_remote_code=model_config.trust_remote_code)
32
+ trust_remote_code=model_config.trust_remote_code,
33
+ **mm_processor_kwargs)
34
35
def _default_input_mapper(
36
self,
@@ -37,6 +42,7 @@ class ImagePlugin(MultiModalPlugin):
42
# PIL image
43
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
44
image_processor = self._get_hf_image_processor(model_config)
45
+
46
if image_processor is None:
47
raise RuntimeError("No HuggingFace processor is available "
48
"to process the image object")
vllm/multimodal/registry.py
+9
@@ -138,6 +138,15 @@ class MultiModalRegistry:
138
"""
139
Create an input mapper (see :meth:`map_input`) for a specific model.
140
"""
141
+ # NOTE - we currently make the assumption that if a model has multiple
142
+ # supported modalities, they take the same kwargs. For the default,
143
+ # this could be an issue in the future if it falls back to two HF
144
+ # resources and we can't inspect the signature easily since it's
145
+ # getting initialized through the autoclass.
146
+ #
147
+ # If this is a problem in the future, we should revisit it, but since
148
+ # it potentially introduces a lot of complexity for a currently
149
+ # uncommon case, we do not for simplicity of both use & implementation
150
return functools.partial(self.map_input, model_config)
151
152
def register_max_multimodal_tokens(
vllm/multimodal/video.py
+7
-2
@@ -6,7 +6,7 @@ import numpy as np
6
from vllm.config import ModelConfig
7
from vllm.inputs.registry import InputContext
8
from vllm.logger import init_logger
9
-from vllm.transformers_utils.image_processor import get_video_processor
9
+from vllm.transformers_utils.processor import get_video_processor
10
from vllm.transformers_utils.tokenizer import get_tokenizer
11
from vllm.utils import is_list_of
12
@@ -37,9 +37,14 @@ class VideoPlugin(ImagePlugin):
37
return "video"
38
39
def _get_hf_video_processor(self, model_config: ModelConfig):
40
+ mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
41
+ else model_config.mm_processor_kwargs)
42
+ # We don't explicitly check kwarg overrides to the HF class
43
+ # since the automodel just takes kwargs, so we can't inspect it
44
return cached_get_video_processor(
45
model_config.model,
42
- trust_remote_code=model_config.trust_remote_code)
46
+ trust_remote_code=model_config.trust_remote_code,
47
+ **mm_processor_kwargs)
48
49
def _default_input_mapper(
50
self,
vllm/transformers_utils/image_processor.py
deleted
-64
@@ -1,64 +0,0 @@
1
-from typing import cast
2
-
3
-
4
-def get_video_processor(
5
- processor_name: str,
6
- trust_remote_code: bool = False,
7
-):
8
- """
9
- Gets a processor for the given model name via HuggingFace.
10
- """
11
- from transformers import AutoProcessor
12
-
13
- try:
14
- processor = AutoProcessor.from_pretrained(processor_name)
15
- video_processor = processor.video_processor
16
-
17
- except ValueError as e:
18
- if not trust_remote_code:
19
- err_msg = (
20
- "Failed to load the processor. If the processor is "
21
- "a custom processor not yet available in the HuggingFace "
22
- "transformers library, consider setting "
23
- "`trust_remote_code=True` in LLM or using the "
24
- "`--trust-remote-code` flag in the CLI.")
25
- raise RuntimeError(err_msg) from e
26
- else:
27
- raise e
28
- return video_processor
29
-
30
-
31
-def get_image_processor(
32
- processor_name: str,
33
- *args,
34
- trust_remote_code: bool = False,
35
- **kwargs,
36
-):
37
- """Gets an image processor for the given model name via HuggingFace."""
38
- # don't put this import at the top level
39
- # it will call torch.cuda.device_count()
40
- from transformers import AutoImageProcessor
41
- from transformers.image_processing_utils import BaseImageProcessor
42
-
43
- try:
44
- processor = AutoImageProcessor.from_pretrained(
45
- processor_name,
46
- *args,
47
- trust_remote_code=trust_remote_code,
48
- **kwargs)
49
- except ValueError as e:
50
- # If the error pertains to the processor class not existing or not
51
- # currently being imported, suggest using the --trust-remote-code flag.
52
- # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
53
- if not trust_remote_code:
54
- err_msg = (
55
- "Failed to load the image processor. If the image processor is "
56
- "a custom processor not yet available in the HuggingFace "
57
- "transformers library, consider setting "
58
- "`trust_remote_code=True` in LLM or using the "
59
- "`--trust-remote-code` flag in the CLI.")
60
- raise RuntimeError(err_msg) from e
61
- else:
62
- raise e
63
-
64
- return cast(BaseImageProcessor, processor)
vllm/transformers_utils/processor.py
+61
-4
@@ -1,13 +1,13 @@
1
-from typing import cast
1
+from typing import Any, cast
2
3
4
def get_processor(
5
processor_name: str,
6
- *args,
6
+ *args: Any,
7
trust_remote_code: bool = False,
8
- **kwargs,
8
+ **kwargs: Any,
9
):
10
- """Gets a processor for the given model name via HuggingFace."""
10
+ """Load a processor for the given model name via HuggingFace."""
11
# don't put this import at the top level
12
# it will call torch.cuda.device_count()
13
from transformers import AutoProcessor
@@ -35,3 +35,60 @@ def get_processor(
35
raise e
36
37
return cast(ProcessorMixin, processor)
38
+
39
+
40
+def get_image_processor(
41
+ processor_name: str,
42
+ *args: Any,
43
+ trust_remote_code: bool = False,
44
+ **kwargs: Any,
45
+):
46
+ """Load an image processor for the given model name via HuggingFace."""
47
+ # don't put this import at the top level
48
+ # it will call torch.cuda.device_count()
49
+ from transformers import AutoImageProcessor
50
+ from transformers.image_processing_utils import BaseImageProcessor
51
+
52
+ try:
53
+ processor = AutoImageProcessor.from_pretrained(
54
+ processor_name,
55
+ *args,
56
+ trust_remote_code=trust_remote_code,
57
+ **kwargs)
58
+ except ValueError as e:
59
+ # If the error pertains to the processor class not existing or not
60
+ # currently being imported, suggest using the --trust-remote-code flag.
61
+ # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
62
+ if not trust_remote_code:
63
+ err_msg = (
64
+ "Failed to load the image processor. If the image processor is "
65
+ "a custom processor not yet available in the HuggingFace "
66
+ "transformers library, consider setting "
67
+ "`trust_remote_code=True` in LLM or using the "
68
+ "`--trust-remote-code` flag in the CLI.")
69
+ raise RuntimeError(err_msg) from e
70
+ else:
71
+ raise e
72
+
73
+ return cast(BaseImageProcessor, processor)
74
+
75
+
76
+def get_video_processor(
77
+ processor_name: str,
78
+ *args: Any,
79
+ trust_remote_code: bool = False,
80
+ **kwargs: Any,
81
+):
82
+ """Load a video processor for the given model name via HuggingFace."""
83
+ # don't put this import at the top level
84
+ # it will call torch.cuda.device_count()
85
+ from transformers.image_processing_utils import BaseImageProcessor
86
+
87
+ processor = get_processor(
88
+ processor_name,
89
+ *args,
90
+ trust_remote_code=trust_remote_code,
91
+ **kwargs,
92
+ )
93
+
94
+ return cast(BaseImageProcessor, processor.video_processor)
vllm/utils.py
+48
@@ -4,6 +4,7 @@ import contextlib
4
import datetime
5
import enum
6
import gc
7
+import inspect
8
import os
9
import random
10
import socket
@@ -1237,6 +1238,53 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1238
return await task(*args, **kwargs)
1239
1240
1241
+def get_allowed_kwarg_only_overrides(
1242
+ callable: Callable[..., object],
1243
+ overrides: Optional[Dict[str, Any]],
1244
+) -> Dict[str, Any]:
1245
+ """
1246
+ Given a callable which has one or more keyword only params and a dict
1247
+ mapping param names to values, drop values that can be not be kwarg
1248
+ expanded to overwrite one or more keyword-only args. This is used in a
1249
+ few places to handle custom processor overrides for multimodal models,
1250
+ e.g., for profiling when processor options provided by the user
1251
+ may affect the number of mm tokens per instance.
1252
+
1253
+ Args:
1254
+ callable: Callable which takes 0 or more keyword only arguments.
1255
+ overrides: Potential overrides to be used when invoking the callable.
1256
+
1257
+ Returns:
1258
+ Dictionary containing the kwargs to be leveraged which may be used
1259
+ to overwrite one or more keyword only arguments when invoking the
1260
+ callable.
1261
+ """
1262
+ if not overrides:
1263
+ return {}
1264
+
1265
+ allowed_override_names = [
1266
+ name for name, param in inspect.signature(callable).parameters.items()
1267
+ if param.kind == inspect.Parameter.KEYWORD_ONLY
1268
+ ]
1269
+
1270
+ # Drop any mm_processor_kwargs provided by the user that are
1271
+ # not kwarg names accepted by the provided input processor.
1272
+ filtered_overrides = {
1273
+ kwarg_name: val
1274
+ for kwarg_name, val in overrides.items()
1275
+ if kwarg_name in allowed_override_names
1276
+ }
1277
+
1278
+ # If anything is dropped, log a warning
1279
+ dropped_keys = overrides.keys() - filtered_overrides.keys()
1280
+ if dropped_keys:
1281
+ logger.warning(
1282
+ "The following intended overrides are not keyword-only args "
1283
+ "and and will be dropped: %s", dropped_keys)
1284
+
1285
+ return filtered_overrides
1286
+
1287
+
1288
# Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0.
1289
# In particular, the FakeScalarType is not supported for earlier versions of
1290
# PyTorch which breaks dynamo for any ops registered using ScalarType.