@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b8c8ba119
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core][Frontend] Support Passing Multimodal Processor Kwargs (#8657)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Sep 23, 2024 at 01:44 UTC 9b8c8ba1198cbcd311d28b7647f0f8d5dcdc9212
16 files changed +589 -116
tests/engine/test_arg_utils.py
+21
@@ -40,3 +40,24 @@ def test_limit_mm_per_prompt_parser(arg, expected):
40 def test_bad_nullable_kvs(arg):
41 with pytest.raises(ArgumentTypeError):
42 nullable_kvs(arg)
43 +
44 +
45 +@pytest.mark.parametrize(("arg", "expected"), [
46 + (None, None),
47 + ("{}", {}),
48 + ('{"num_crops": 4}', {
49 + "num_crops": 4
50 + }),
51 + ('{"foo": {"bar": "baz"}}', {
52 + "foo": {
53 + "bar": "baz"
54 + }
55 + }),
56 +])
57 +def test_mm_processor_kwargs_prompt_parser(arg, expected):
58 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
59 + if arg is None:
60 + args = parser.parse_args([])
61 + else:
62 + args = parser.parse_args(["--mm-processor-kwargs", arg])
63 + assert args.mm_processor_kwargs == expected
tests/models/decoder_only/vision_language/test_qwen.py
+1 -28
@@ -5,14 +5,13 @@ import pytest
5 import torch
6 from PIL.Image import Image
7
8 -from vllm.config import ModelConfig
8 from vllm.inputs import InputContext, LLMInputs
9 from vllm.multimodal.base import MultiModalInputs
10 from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
11
12 from ....conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
13 VllmRunner, _ImageAssets)
15 -from ...utils import check_logprobs_close
14 +from ...utils import build_model_context, check_logprobs_close
15
16 text_only_models = [
17 "Qwen/Qwen-7B-Chat" # Has no visual component
@@ -42,32 +41,6 @@ VIS_ENC_DIM = 4096
41 IMG_SIZE = 448
42
43
45 -def build_model_context(model_name: str,
46 - tokenizer_name: Optional[str] = None,
47 - trust_remote_code: bool = False):
48 - """Creates an InputContext for a given model.
49 -
50 - Args:
51 - model_name: Name of the model being considered.
52 - tokenizer_name: Name of the tokenizer being considered.
53 - trust_remote_code: Whether or not to allow loading remote code.
54 -
55 - Returns:
56 - InputContext for the model being considered.
57 - """
58 - if tokenizer_name is None:
59 - tokenizer_name = model_name
60 - model_config = ModelConfig(
61 - model_name,
62 - tokenizer_name,
63 - tokenizer_mode="auto",
64 - trust_remote_code=trust_remote_code,
65 - dtype="float32",
66 - seed=0,
67 - )
68 - return InputContext(model_config)
69 -
70 -
44 @pytest.fixture()
45 def input_mapper_for_qwen():
46 # Lazy import to avoid initializing CUDA during test collection
tests/models/utils.py
+35
@@ -1,6 +1,8 @@
1 import warnings
2 from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 +from vllm.config import ModelConfig
5 +from vllm.inputs import InputContext
6 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
7
8 TokensText = Tuple[List[int], str]
@@ -240,3 +242,36 @@ def check_logprobs_close(
242 warnings.simplefilter("always")
243
244 warnings.warn(fail_msg, stacklevel=2)
245 +
246 +
247 +def build_model_context(model_name: str,
248 + tokenizer_name: Optional[str] = None,
249 + trust_remote_code: bool = False,
250 + mm_processor_kwargs: Optional[Dict] = None,
251 + limit_mm_per_prompt: Optional[Dict] = None):
252 + """Creates an InputContext for a given model.
253 +
254 + Args:
255 + model_name: Name of the model being considered.
256 + tokenizer_name: Name of the tokenizer being considered.
257 + trust_remote_code: Whether or not to allow loading remote code.
258 + mm_processor_kwargs: optional processor kwargs for to be leveraged
259 + in the input processor, mapper, dummy data creation, etc.
260 + limit_mm_per_prompt: Multimodal limits.
261 +
262 + Returns:
263 + InputContext for the model being considered.
264 + """
265 + if tokenizer_name is None:
266 + tokenizer_name = model_name
267 + model_config = ModelConfig(
268 + model_name,
269 + tokenizer_name,
270 + tokenizer_mode="auto",
271 + trust_remote_code=trust_remote_code,
272 + dtype="float32",
273 + seed=0,
274 + mm_processor_kwargs=mm_processor_kwargs,
275 + limit_mm_per_prompt=limit_mm_per_prompt,
276 + )
277 + return InputContext(model_config)
tests/multimodal/test_processor_kwargs.py new
+339
@@ -0,0 +1,339 @@
1 +from array import array
2 +from typing import Mapping
3 +from unittest.mock import patch
4 +
5 +import pytest
6 +import torch
7 +
8 +from vllm.inputs import InputContext, LLMInputs
9 +from vllm.inputs.registry import InputRegistry
10 +from vllm.multimodal import MultiModalRegistry
11 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12 +
13 +from ..models.utils import build_model_context
14 +
15 +# Used for fast tests where the model doesn't matter
16 +DUMMY_MODEL_ID = "facebook/opt-125m"
17 +# Used for tests that need a multimodal model
18 +MULTIMODAL_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
19 +
20 +# For mm_processor_kwargs - we test overrides by defining mocks for each place
21 +# it is used, and ensuring that we can pass processor kwargs an override value
22 +# to receive the intended result for things like sequence length etc.
23 +DEFAULT_NUM_CROPS = 4
24 +NUM_CROPS_OVERRIDE = 16
25 +
26 +
27 +# Mocks for all of the places that we use the mm_processor_kwargs
28 +# to override values in different callables
29 +@pytest.fixture
30 +def use_processor_mock():
31 + """Patches the internal model input processor with an override callable."""
32 +
33 + def custom_processor(ctx: InputContext,
34 + llm_inputs: LLMInputs,
35 + *,
36 + num_crops=DEFAULT_NUM_CROPS):
37 + # For testing purposes, we don't worry about the llm inputs / return
38 + # type validation, and just return the value of the kwarg that we
39 + # clobber.
40 + return num_crops
41 +
42 + with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
43 + return_value=custom_processor):
44 + yield
45 +
46 +
47 +@pytest.fixture
48 +def use_dummy_data_mock():
49 + """Patches the internal model input processor with an override callable."""
50 +
51 + def custom_dummy_data_factory(self,
52 + ctx: InputContext,
53 + seq_len: int,
54 + mm_counts: Mapping[str, int],
55 + *,
56 + num_crops=DEFAULT_NUM_CROPS):
57 + seq_data = SequenceData(
58 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
59 + return seq_data, None
60 +
61 + with patch(
62 + "vllm.inputs.registry.InputRegistry._default_dummy_data_factory",
63 + custom_dummy_data_factory):
64 + yield
65 +
66 +
67 +# Lazy import to avoid CUDA reinitialization error
68 +def mm_model_cls():
69 + from vllm.model_executor.models.phi3v import Phi3VForCausalLM
70 +
71 + return Phi3VForCausalLM
72 +
73 +
74 +# lambda whose signature matches max token calcs extra & mapper + extra kwargs
75 +get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
76 +custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
77 + "num_pixels": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
78 +}
79 +
80 +
81 +### Test for default processor logic & mm_processor_kwargs wrapping
82 +def test_default_processor_is_a_noop():
83 + """Ensure that by default, there is no processor override."""
84 + dummy_registry = InputRegistry()
85 + ctx = build_model_context(DUMMY_MODEL_ID)
86 + processor = dummy_registry.create_input_processor(ctx.model_config)
87 + proc_inputs = LLMInputs(prompt_token_ids=[], prompt="")
88 + proc_outputs = processor(inputs=proc_inputs)
89 + assert proc_inputs is proc_outputs
90 +
91 +
92 +@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
93 +def test_processor_default_kwargs(use_processor_mock, num_crops):
94 + """Ensure input processors can use processor kwargs."""
95 + dummy_registry = InputRegistry()
96 + # If we have a value for num_crops, pass the override value and make
97 + # sure we get that value as a return-value from out mock processor,
98 + # otherwise fall back to the default value
99 + mm_processor_kwargs = None if num_crops is None else {
100 + "num_crops": num_crops
101 + }
102 + expected_num_crops = DEFAULT_NUM_CROPS if num_crops is None else num_crops
103 + ctx = build_model_context(DUMMY_MODEL_ID,
104 + mm_processor_kwargs=mm_processor_kwargs)
105 + processor = dummy_registry.create_input_processor(ctx.model_config)
106 +
107 + num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
108 + assert num_crops_val == expected_num_crops
109 +
110 +
111 +@pytest.mark.parametrize(
112 + "mm_processor_kwargs",
113 + [
114 + # Not part of the signature
115 + {
116 + "does_not_exist": 100
117 + },
118 + # Part of the signature, not keyword only
119 + {
120 + "ctx": "something bad"
121 + }
122 + ])
123 +def test_processor_with_sad_kwarg_overrides(use_processor_mock,
124 + mm_processor_kwargs):
125 + """Ensure that input processors filter out invalid mm_processor_kwargs"""
126 + dummy_registry = InputRegistry()
127 + ctx = build_model_context(DUMMY_MODEL_ID,
128 + mm_processor_kwargs=mm_processor_kwargs)
129 +
130 + processor = dummy_registry.create_input_processor(ctx.model_config)
131 + num_crops_val = processor(LLMInputs(prompt_token_ids=[], prompt=""))
132 + assert num_crops_val == DEFAULT_NUM_CROPS
133 +
134 +
135 +### Test overrides for the dummy data
136 +@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
137 +def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
138 + """Ensure dummy data factories can use processor kwargs."""
139 + mm_processor_kwargs = None if num_crops is None else {
140 + "num_crops": num_crops
141 + }
142 + expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
143 + dummy_registry = InputRegistry()
144 + ctx = build_model_context(DUMMY_MODEL_ID,
145 + mm_processor_kwargs=mm_processor_kwargs)
146 + mm_registry = MultiModalRegistry()
147 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
148 +
149 + # NOTE: seq_len is thrown away here since this will leverage the
150 + # default dummy data factory that we have patched in, whose seq
151 + # len is solely dependent on the value of the mm_processor_kwargs.
152 + seq_data, _ = dummy_registry.dummy_data_for_profiling(
153 + ctx.model_config, seq_len=-1, mm_registry=mm_registry)
154 + assert len(seq_data.prompt_token_ids) == expected_seq_count
155 +
156 +
157 +@pytest.mark.parametrize(
158 + "mm_processor_kwargs",
159 + [
160 + # Not part of the signature
161 + {
162 + "does_not_exist": 100
163 + },
164 + # Part of the signature, not keyword only
165 + {
166 + "ctx": "something bad"
167 + }
168 + ])
169 +def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
170 + mm_processor_kwargs):
171 + """Ensure the dummy data factory filters out invalid mm_processor_kwargs"""
172 + dummy_registry = InputRegistry()
173 + ctx = build_model_context(DUMMY_MODEL_ID,
174 + mm_processor_kwargs=mm_processor_kwargs)
175 + mm_registry = MultiModalRegistry()
176 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
177 +
178 + # NOTE: seq_len is thrown away here since this will leverage the
179 + # default dummy data factory that we have patched in, whose seq
180 + # len is solely dependent on the value of the mm_processor_kwargs.
181 + seq_data, _ = dummy_registry.dummy_data_for_profiling(
182 + ctx.model_config, seq_len=-1, mm_registry=mm_registry)
183 + assert len(seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
184 +
185 +
186 +### Test overrides for the max token count per multimodal instance
187 +@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
188 +def test_max_tokens_kwarg_overrides(num_crops):
189 + """Ensure max token calcs can use processor kwargs."""
190 + mm_processor_kwargs = None if num_crops is None else {
191 + "num_crops": num_crops
192 + }
193 + expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
194 +
195 + ctx = build_model_context(MULTIMODAL_MODEL_ID,
196 + trust_remote_code=True,
197 + mm_processor_kwargs=mm_processor_kwargs,
198 + limit_mm_per_prompt={"image": 1})
199 +
200 + mm_registry = MultiModalRegistry()
201 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
202 + # Patch the image registry for phi3v with our lambda that is compatible
203 + # with overrides, then ensure that calling the method correctly echos
204 + # our num_crops value back from the mm_processor_kwargs.
205 + with patch.object(
206 + mm_registry._get_plugin("image"),
207 + "_max_mm_tokens",
208 + {mm_model_cls(): get_num_crops},
209 + ):
210 + max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
211 + ctx.model_config)
212 +
213 + assert expected_seq_count == max_multimodal_tokens
214 +
215 +
216 +@pytest.mark.parametrize(
217 + "mm_processor_kwargs",
218 + [
219 + # Not part of the signature
220 + {
221 + "does_not_exist": 100
222 + },
223 + # Part of the signature, not keyword only
224 + {
225 + "ctx": "something bad"
226 + }
227 + ])
228 +def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
229 + """Ensure that max token calcs filters out invalid mm_processor_kwargs"""
230 + ctx = build_model_context(MULTIMODAL_MODEL_ID,
231 + trust_remote_code=True,
232 + mm_processor_kwargs=mm_processor_kwargs,
233 + limit_mm_per_prompt={"image": 1})
234 +
235 + mm_registry = MultiModalRegistry()
236 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
237 +
238 + # Similar before, but since these kwargs get filtered,
239 + # we always get our default value back.
240 + with patch.object(
241 + mm_registry._get_plugin("image"),
242 + "_max_mm_tokens",
243 + {mm_model_cls(): get_num_crops},
244 + ):
245 + max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
246 + ctx.model_config)
247 +
248 + assert max_multimodal_tokens == DEFAULT_NUM_CROPS
249 +
250 +
251 +### Test overrides for the mapper
252 +@pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
253 +def test_default_mapper_with_processer_kwargs(image_assets, num_crops):
254 + """Ensure that the mapper processor kwargs can fall back to HF models."""
255 + # NOTE - we don't validate bad inputs for the default mapper, because it's
256 + # through the automodel interface in transformers, so we can't easily
257 + # inspect what kwargs are or are not allowed.
258 + ctx = build_model_context(MULTIMODAL_MODEL_ID,
259 + trust_remote_code=True,
260 + mm_processor_kwargs={"num_crops": num_crops},
261 + limit_mm_per_prompt={"image": 1})
262 +
263 + mm_registry = MultiModalRegistry()
264 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
265 +
266 + image = image_assets[0].pil_image
267 + mm_inputs = {"image": image}
268 +
269 + mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
270 + # Phi3v pixel vals should have shape: [batch, num_crops+1, 3, 336, 336]
271 + assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
272 +
273 +
274 +@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
275 +def test_custom_mapper_kwarg_overrides(image_assets, num_crops):
276 + """Ensure custom mappers can use processor kwargs."""
277 + mm_processor_kwargs = None if num_crops is None else {
278 + "num_crops": num_crops
279 + }
280 + expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
281 + ctx = build_model_context(MULTIMODAL_MODEL_ID,
282 + trust_remote_code=True,
283 + mm_processor_kwargs=mm_processor_kwargs,
284 + limit_mm_per_prompt={"image": 1})
285 +
286 + mm_registry = MultiModalRegistry()
287 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
288 + # Patch the image registry for phi3v with our lambda that is compatible
289 + # with overrides, then ensure that calling the method correctly echos
290 + # our num_crops value back from the mm_processor_kwargs.
291 + image = image_assets[0].pil_image
292 + mm_inputs = {"image": image}
293 +
294 + with patch.object(
295 + mm_registry._get_plugin("image"),
296 + "_default_input_mapper",
297 + {mm_model_cls(): custom_mapper},
298 + ):
299 + mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
300 +
301 + assert mapped_inputs["pixel_values"].shape[1] == expected_seq_count + 1
302 +
303 +
304 +@pytest.mark.parametrize(
305 + "mm_processor_kwargs",
306 + [
307 + # Not part of the signature
308 + {
309 + "does_not_exist": 100
310 + },
311 + # Part of the signature, not keyword only
312 + {
313 + "ctx": "something bad"
314 + }
315 + ])
316 +def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
317 + mm_processor_kwargs):
318 + """Ensure that custom mappers filters out invalid mm_processor_kwargs"""
319 + ctx = build_model_context(MULTIMODAL_MODEL_ID,
320 + trust_remote_code=True,
321 + mm_processor_kwargs=mm_processor_kwargs,
322 + limit_mm_per_prompt={"image": 1})
323 +
324 + mm_registry = MultiModalRegistry()
325 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
326 + # Patch the image registry for phi3v with our lambda that is compatible
327 + # with overrides, then ensure that calling the method correctly echos
328 + # our num_crops value back from the mm_processor_kwargs.
329 + image = image_assets[0].pil_image
330 + mm_inputs = {"image": image}
331 +
332 + with patch.object(
333 + mm_registry._get_plugin("image"),
334 + "_default_input_mapper",
335 + {mm_model_cls(): custom_mapper},
336 + ):
337 + mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
338 +
339 + assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
vllm/config.py
+5 -1
@@ -122,6 +122,8 @@ class ModelConfig:
122 can not be gathered from the vllm arguments.
123 config_format: The config format which shall be loaded.
124 Defaults to 'auto' which defaults to 'hf'.
125 + mm_processor_kwargs: Arguments to be forwarded to the model's processor
126 + for multi-modal data, e.g., image processor.
127 """
128
129 def __init__(self,
@@ -150,7 +152,8 @@ class ModelConfig:
152 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
153 use_async_output_proc: bool = True,
154 override_neuron_config: Optional[Dict[str, Any]] = None,
153 - config_format: ConfigFormat = ConfigFormat.AUTO) -> None:
155 + config_format: ConfigFormat = ConfigFormat.AUTO,
156 + mm_processor_kwargs: Optional[Dict[str, Any]] = None) -> None:
157 self.model = model
158 self.tokenizer = tokenizer
159 self.tokenizer_mode = tokenizer_mode
@@ -184,6 +187,7 @@ class ModelConfig:
187 self.model, revision)
188 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
189 self.use_async_output_proc = use_async_output_proc
190 + self.mm_processor_kwargs = mm_processor_kwargs
191
192 # Set enforce_eager to False if the value is unset.
193 if self.enforce_eager is None:
vllm/engine/arg_utils.py
+8
@@ -175,6 +175,7 @@ class EngineArgs:
175 collect_detailed_traces: Optional[str] = None
176 disable_async_output_proc: bool = False
177 override_neuron_config: Optional[Dict[str, Any]] = None
178 + mm_processor_kwargs: Optional[Dict[str, Any]] = None
179
180 def __post_init__(self):
181 if self.tokenizer is None:
@@ -513,6 +514,12 @@ class EngineArgs:
514 'e.g.: `image=16,video=2` allows a maximum of 16 '
515 'images and 2 videos per prompt. Defaults to 1 for '
516 'each modality.'))
517 + parser.add_argument(
518 + '--mm-processor-kwargs',
519 + default=None,
520 + type=json.loads,
521 + help=('Overrides for the multimodal input mapping/processing,'
522 + 'e.g., image processor. For example: {"num_crops": 4}.'))
523
524 # LoRA related configs
525 parser.add_argument('--enable-lora',
@@ -822,6 +829,7 @@ class EngineArgs:
829 use_async_output_proc=not self.disable_async_output_proc,
830 override_neuron_config=self.override_neuron_config,
831 config_format=self.config_format,
832 + mm_processor_kwargs=self.mm_processor_kwargs,
833 )
834
835 def create_load_config(self) -> LoadConfig:
vllm/engine/llm_engine.py
+2 -1
@@ -235,7 +235,7 @@ class LLMEngine:
235 "decoding_config=%r, observability_config=%r, "
236 "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
237 "num_scheduler_steps=%d, enable_prefix_caching=%s, "
238 - "use_async_output_proc=%s)",
238 + "use_async_output_proc=%s, mm_processor_kwargs=%s)",
239 VLLM_VERSION,
240 model_config.model,
241 speculative_config,
@@ -268,6 +268,7 @@ class LLMEngine:
268 scheduler_config.num_scheduler_steps,
269 cache_config.enable_prefix_caching,
270 model_config.use_async_output_proc,
271 + model_config.mm_processor_kwargs,
272 )
273 # TODO(woosuk): Print more configs in debug mode.
274 from vllm.plugins import load_general_plugins
vllm/entrypoints/llm.py
+2
@@ -134,6 +134,7 @@ class LLM:
134 max_seq_len_to_capture: int = 8192,
135 disable_custom_all_reduce: bool = False,
136 disable_async_output_proc: bool = False,
137 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
138 **kwargs,
139 ) -> None:
140 '''
@@ -174,6 +175,7 @@ class LLM:
175 max_seq_len_to_capture=max_seq_len_to_capture,
176 disable_custom_all_reduce=disable_custom_all_reduce,
177 disable_async_output_proc=disable_async_output_proc,
178 + mm_processor_kwargs=mm_processor_kwargs,
179 **kwargs,
180 )
181 self.llm_engine = LLMEngine.from_engine_args(
vllm/inputs/registry.py
+27 -11
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
9 from typing_extensions import TypeVar
10
11 from vllm.logger import init_logger
12 +from vllm.utils import get_allowed_kwarg_only_overrides
13
14 from .data import LLMInputs
15
@@ -68,12 +69,17 @@ class DummyDataFactory(Protocol):
69 ctx: InputContext,
70 seq_len: int,
71 mm_counts: Mapping[str, int],
72 + **mm_processor_kwargs: Any,
73 ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
74 """
75 Create dummy data to be inputted into the model.
76
77 Note:
78 :data:`InputProcessor` is not applied to the dummy data.
79 +
80 + The :code:`mm_processor_kwargs` are overrides provided at
81 + initialization time to values in the config whose values
82 + may affect the number of tokens per instance.
83 """
84 ...
85
@@ -152,6 +158,10 @@ class InputRegistry:
158
159 return wrapper
160
161 + def _get_dummy_data_factory(self, model_cls: Type[nn.Module]):
162 + return self._dummy_factories_by_model_type \
163 + .get(model_cls, self._default_dummy_data_factory)
164 +
165 def dummy_data_for_profiling(
166 self,
167 model_config: "ModelConfig",
@@ -174,15 +184,15 @@ class InputRegistry:
184 from vllm.model_executor.model_loader import get_model_architecture
185
186 model_cls, _ = get_model_architecture(model_config)
177 - dummy_factory = self._dummy_factories_by_model_type \
178 - .get(model_cls, self._default_dummy_data_factory)
187 + dummy_factory = self._get_dummy_data_factory(model_cls)
188 +
189 mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
190 + mm_processor_kwargs = get_allowed_kwarg_only_overrides(
191 + dummy_factory, overrides=model_config.mm_processor_kwargs)
192
181 - seq_data, mm_data = dummy_factory(
182 - InputContext(model_config),
183 - seq_len,
184 - _MultiModalCounts(mm_counts),
185 - )
193 + seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len,
194 + _MultiModalCounts(mm_counts),
195 + **mm_processor_kwargs)
196
197 # Having more tokens is over-conservative but otherwise fine
198 num_tokens = seq_data.prompt_token_ids
@@ -229,6 +239,10 @@ class InputRegistry:
239
240 return wrapper
241
242 + def _get_model_input_processor(self, model_cls: Type[nn.Module]):
243 + return self._input_processors_by_model_type \
244 + .get(model_cls, self._default_input_processor)
245 +
246 def process_input(self, model_config: "ModelConfig",
247 inputs: LLMInputs) -> LLMInputs:
248 """
@@ -243,15 +257,17 @@ class InputRegistry:
257 from vllm.model_executor.model_loader import get_model_architecture
258
259 model_cls, _ = get_model_architecture(model_config)
260 + processor = self._get_model_input_processor(model_cls)
261
247 - processor = self._input_processors_by_model_type \
248 - .get(model_cls, self._default_input_processor)
262 + mm_processor_kwargs = get_allowed_kwarg_only_overrides(
263 + processor, overrides=model_config.mm_processor_kwargs)
264
250 - return processor(InputContext(model_config), inputs)
265 + return processor(InputContext(model_config), inputs,
266 + **mm_processor_kwargs)
267
268 def create_input_processor(self, model_config: "ModelConfig"):
269 """
254 - Create an input processor (see :meth:`process_input`) for a
270 + Create an input processor (see :meth:`_process_input`) for a
271 specific model.
272 """
273 return functools.partial(self.process_input, model_config)
vllm/multimodal/base.py
+16 -3
@@ -14,7 +14,8 @@ from typing_extensions import TypeAlias
14 from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
17 -from vllm.utils import JSONTree, is_list_of, json_map_leaves
17 +from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18 + json_map_leaves)
19
20 logger = init_logger(__name__)
21
@@ -256,11 +257,20 @@ class MultiModalPlugin(ABC):
257 model_cls, _ = get_model_architecture(model_config)
258
259 mapper = self._input_mappers.get(model_cls)
260 + # Only get processor kwargs at mapping time if we are not using the
261 + # input mapper; no overrides are used on the default here because they
262 + # should be passed to the huggingface resource at initialization time.
263 + if mapper is not None and mapper != self._default_input_mapper:
264 + mm_processor_kwargs = get_allowed_kwarg_only_overrides(
265 + mapper, overrides=model_config.mm_processor_kwargs)
266 + else:
267 + mm_processor_kwargs = {}
268 +
269 if mapper is None:
270 raise KeyError(f"No input mapper in {self} is registered for "
271 f"model class {model_cls.__name__}.")
272
263 - return mapper(InputContext(model_config), data)
273 + return mapper(InputContext(model_config), data, **mm_processor_kwargs)
274
275 @abstractmethod
276 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
@@ -333,7 +343,10 @@ class MultiModalPlugin(ABC):
343 f"for model class {model_cls.__name__} in {self}.")
344
345 if callable(max_mm_tokens):
336 - max_mm_tokens = max_mm_tokens(InputContext(model_config))
346 + mm_processor_kwargs = get_allowed_kwarg_only_overrides(
347 + max_mm_tokens, overrides=model_config.mm_processor_kwargs)
348 + max_mm_tokens = max_mm_tokens(InputContext(model_config),
349 + **mm_processor_kwargs)
350
351 self._validate_max_multimodal_tokens(max_mm_tokens)
352
vllm/multimodal/image.py
+8 -2
@@ -6,7 +6,7 @@ from PIL import Image
6 from vllm.config import ModelConfig
7 from vllm.inputs.registry import InputContext
8 from vllm.logger import init_logger
9 -from vllm.transformers_utils.image_processor import get_image_processor
9 +from vllm.transformers_utils.processor import get_image_processor
10 from vllm.utils import is_list_of
11
12 from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
@@ -23,9 +23,14 @@ class ImagePlugin(MultiModalPlugin):
23 return "image"
24
25 def _get_hf_image_processor(self, model_config: ModelConfig):
26 + mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
27 + else model_config.mm_processor_kwargs)
28 + # We don't explicitly check kwarg overrides to the HF class
29 + # since the automodel just takes kwargs, so we can't inspect it
30 return cached_get_image_processor(
31 model_config.model,
28 - trust_remote_code=model_config.trust_remote_code)
32 + trust_remote_code=model_config.trust_remote_code,
33 + **mm_processor_kwargs)
34
35 def _default_input_mapper(
36 self,
@@ -37,6 +42,7 @@ class ImagePlugin(MultiModalPlugin):
42 # PIL image
43 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
44 image_processor = self._get_hf_image_processor(model_config)
45 +
46 if image_processor is None:
47 raise RuntimeError("No HuggingFace processor is available "
48 "to process the image object")
vllm/multimodal/registry.py
+9
@@ -138,6 +138,15 @@ class MultiModalRegistry:
138 """
139 Create an input mapper (see :meth:`map_input`) for a specific model.
140 """
141 + # NOTE - we currently make the assumption that if a model has multiple
142 + # supported modalities, they take the same kwargs. For the default,
143 + # this could be an issue in the future if it falls back to two HF
144 + # resources and we can't inspect the signature easily since it's
145 + # getting initialized through the autoclass.
146 + #
147 + # If this is a problem in the future, we should revisit it, but since
148 + # it potentially introduces a lot of complexity for a currently
149 + # uncommon case, we do not for simplicity of both use & implementation
150 return functools.partial(self.map_input, model_config)
151
152 def register_max_multimodal_tokens(
vllm/multimodal/video.py
+7 -2
@@ -6,7 +6,7 @@ import numpy as np
6 from vllm.config import ModelConfig
7 from vllm.inputs.registry import InputContext
8 from vllm.logger import init_logger
9 -from vllm.transformers_utils.image_processor import get_video_processor
9 +from vllm.transformers_utils.processor import get_video_processor
10 from vllm.transformers_utils.tokenizer import get_tokenizer
11 from vllm.utils import is_list_of
12
@@ -37,9 +37,14 @@ class VideoPlugin(ImagePlugin):
37 return "video"
38
39 def _get_hf_video_processor(self, model_config: ModelConfig):
40 + mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
41 + else model_config.mm_processor_kwargs)
42 + # We don't explicitly check kwarg overrides to the HF class
43 + # since the automodel just takes kwargs, so we can't inspect it
44 return cached_get_video_processor(
45 model_config.model,
42 - trust_remote_code=model_config.trust_remote_code)
46 + trust_remote_code=model_config.trust_remote_code,
47 + **mm_processor_kwargs)
48
49 def _default_input_mapper(
50 self,
vllm/transformers_utils/image_processor.py deleted
-64
@@ -1,64 +0,0 @@
1 -from typing import cast
2 -
3 -
4 -def get_video_processor(
5 - processor_name: str,
6 - trust_remote_code: bool = False,
7 -):
8 - """
9 - Gets a processor for the given model name via HuggingFace.
10 - """
11 - from transformers import AutoProcessor
12 -
13 - try:
14 - processor = AutoProcessor.from_pretrained(processor_name)
15 - video_processor = processor.video_processor
16 -
17 - except ValueError as e:
18 - if not trust_remote_code:
19 - err_msg = (
20 - "Failed to load the processor. If the processor is "
21 - "a custom processor not yet available in the HuggingFace "
22 - "transformers library, consider setting "
23 - "`trust_remote_code=True` in LLM or using the "
24 - "`--trust-remote-code` flag in the CLI.")
25 - raise RuntimeError(err_msg) from e
26 - else:
27 - raise e
28 - return video_processor
29 -
30 -
31 -def get_image_processor(
32 - processor_name: str,
33 - *args,
34 - trust_remote_code: bool = False,
35 - **kwargs,
36 -):
37 - """Gets an image processor for the given model name via HuggingFace."""
38 - # don't put this import at the top level
39 - # it will call torch.cuda.device_count()
40 - from transformers import AutoImageProcessor
41 - from transformers.image_processing_utils import BaseImageProcessor
42 -
43 - try:
44 - processor = AutoImageProcessor.from_pretrained(
45 - processor_name,
46 - *args,
47 - trust_remote_code=trust_remote_code,
48 - **kwargs)
49 - except ValueError as e:
50 - # If the error pertains to the processor class not existing or not
51 - # currently being imported, suggest using the --trust-remote-code flag.
52 - # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
53 - if not trust_remote_code:
54 - err_msg = (
55 - "Failed to load the image processor. If the image processor is "
56 - "a custom processor not yet available in the HuggingFace "
57 - "transformers library, consider setting "
58 - "`trust_remote_code=True` in LLM or using the "
59 - "`--trust-remote-code` flag in the CLI.")
60 - raise RuntimeError(err_msg) from e
61 - else:
62 - raise e
63 -
64 - return cast(BaseImageProcessor, processor)
vllm/transformers_utils/processor.py
+61 -4
@@ -1,13 +1,13 @@
1 -from typing import cast
1 +from typing import Any, cast
2
3
4 def get_processor(
5 processor_name: str,
6 - *args,
6 + *args: Any,
7 trust_remote_code: bool = False,
8 - **kwargs,
8 + **kwargs: Any,
9 ):
10 - """Gets a processor for the given model name via HuggingFace."""
10 + """Load a processor for the given model name via HuggingFace."""
11 # don't put this import at the top level
12 # it will call torch.cuda.device_count()
13 from transformers import AutoProcessor
@@ -35,3 +35,60 @@ def get_processor(
35 raise e
36
37 return cast(ProcessorMixin, processor)
38 +
39 +
40 +def get_image_processor(
41 + processor_name: str,
42 + *args: Any,
43 + trust_remote_code: bool = False,
44 + **kwargs: Any,
45 +):
46 + """Load an image processor for the given model name via HuggingFace."""
47 + # don't put this import at the top level
48 + # it will call torch.cuda.device_count()
49 + from transformers import AutoImageProcessor
50 + from transformers.image_processing_utils import BaseImageProcessor
51 +
52 + try:
53 + processor = AutoImageProcessor.from_pretrained(
54 + processor_name,
55 + *args,
56 + trust_remote_code=trust_remote_code,
57 + **kwargs)
58 + except ValueError as e:
59 + # If the error pertains to the processor class not existing or not
60 + # currently being imported, suggest using the --trust-remote-code flag.
61 + # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
62 + if not trust_remote_code:
63 + err_msg = (
64 + "Failed to load the image processor. If the image processor is "
65 + "a custom processor not yet available in the HuggingFace "
66 + "transformers library, consider setting "
67 + "`trust_remote_code=True` in LLM or using the "
68 + "`--trust-remote-code` flag in the CLI.")
69 + raise RuntimeError(err_msg) from e
70 + else:
71 + raise e
72 +
73 + return cast(BaseImageProcessor, processor)
74 +
75 +
76 +def get_video_processor(
77 + processor_name: str,
78 + *args: Any,
79 + trust_remote_code: bool = False,
80 + **kwargs: Any,
81 +):
82 + """Load a video processor for the given model name via HuggingFace."""
83 + # don't put this import at the top level
84 + # it will call torch.cuda.device_count()
85 + from transformers.image_processing_utils import BaseImageProcessor
86 +
87 + processor = get_processor(
88 + processor_name,
89 + *args,
90 + trust_remote_code=trust_remote_code,
91 + **kwargs,
92 + )
93 +
94 + return cast(BaseImageProcessor, processor.video_processor)
vllm/utils.py
+48
@@ -4,6 +4,7 @@ import contextlib
4 import datetime
5 import enum
6 import gc
7 +import inspect
8 import os
9 import random
10 import socket
@@ -1237,6 +1238,53 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1238 return await task(*args, **kwargs)
1239
1240
1241 +def get_allowed_kwarg_only_overrides(
1242 + callable: Callable[..., object],
1243 + overrides: Optional[Dict[str, Any]],
1244 +) -> Dict[str, Any]:
1245 + """
1246 + Given a callable which has one or more keyword only params and a dict
1247 + mapping param names to values, drop values that can be not be kwarg
1248 + expanded to overwrite one or more keyword-only args. This is used in a
1249 + few places to handle custom processor overrides for multimodal models,
1250 + e.g., for profiling when processor options provided by the user
1251 + may affect the number of mm tokens per instance.
1252 +
1253 + Args:
1254 + callable: Callable which takes 0 or more keyword only arguments.
1255 + overrides: Potential overrides to be used when invoking the callable.
1256 +
1257 + Returns:
1258 + Dictionary containing the kwargs to be leveraged which may be used
1259 + to overwrite one or more keyword only arguments when invoking the
1260 + callable.
1261 + """
1262 + if not overrides:
1263 + return {}
1264 +
1265 + allowed_override_names = [
1266 + name for name, param in inspect.signature(callable).parameters.items()
1267 + if param.kind == inspect.Parameter.KEYWORD_ONLY
1268 + ]
1269 +
1270 + # Drop any mm_processor_kwargs provided by the user that are
1271 + # not kwarg names accepted by the provided input processor.
1272 + filtered_overrides = {
1273 + kwarg_name: val
1274 + for kwarg_name, val in overrides.items()
1275 + if kwarg_name in allowed_override_names
1276 + }
1277 +
1278 + # If anything is dropped, log a warning
1279 + dropped_keys = overrides.keys() - filtered_overrides.keys()
1280 + if dropped_keys:
1281 + logger.warning(
1282 + "The following intended overrides are not keyword-only args "
1283 + "and and will be dropped: %s", dropped_keys)
1284 +
1285 + return filtered_overrides
1286 +
1287 +
1288 # Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0.
1289 # In particular, the FakeScalarType is not supported for earlier versions of
1290 # PyTorch which breaks dynamo for any ops registered using ScalarType.