@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 955fa9533a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[3/N] Support and implement merged input processor for LLaVA model (#10676)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Dec 7, 2024 at 16:50 UTC 955fa9533afde0d232e73f079d72239c8a87c636
10 files changed +626 -421
tests/multimodal/test_mapper.py
+3 -46
@@ -2,7 +2,7 @@ from contextlib import nullcontext
2
3 import numpy as np
4 import pytest
5 -from transformers import CLIPImageProcessor, LlavaNextImageProcessor
5 +from transformers import LlavaNextImageProcessor
6
7 from vllm.config import ModelConfig
8 from vllm.multimodal import MultiModalRegistry
@@ -14,49 +14,6 @@ def mm_registry():
14 return MultiModalRegistry()
15
16
17 -@pytest.mark.parametrize("dtype", ["half", "float"])
18 -@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19 -def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
20 - MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
21 -
22 - hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
23 - assert isinstance(hf_processor, CLIPImageProcessor)
24 -
25 - model_config = ModelConfig(
26 - model=MODEL_NAME,
27 - task="auto",
28 - tokenizer=MODEL_NAME,
29 - tokenizer_mode="auto",
30 - trust_remote_code=False,
31 - seed=0,
32 - dtype=dtype,
33 - revision=None,
34 - limit_mm_per_prompt={"image": 1},
35 - )
36 -
37 - mm_registry.init_mm_limits_per_prompt(model_config)
38 -
39 - for asset in image_assets:
40 - image = rescale_image_size(asset.pil_image, size_factor)
41 -
42 - hf_result = hf_processor.preprocess(
43 - image,
44 - return_tensors="pt",
45 - )
46 - vllm_result = mm_registry.map_input(
47 - model_config,
48 - {"image": image},
49 - )
50 -
51 - assert hf_result.keys() == vllm_result.keys()
52 - for key, hf_tensor in hf_result.items():
53 - hf_arr: np.ndarray = hf_tensor.numpy()
54 - vllm_arr: np.ndarray = vllm_result[key].numpy()
55 -
56 - assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
57 - assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
58 -
59 -
17 @pytest.mark.parametrize("dtype", ["half", "float"])
18 @pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19 def test_llava_next_image_processor(image_assets, mm_registry, dtype,
@@ -107,7 +64,7 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
64 (2, 1, False), (2, 2, True)],
65 )
66 def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
110 - MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
67 + MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
68
69 model_config = ModelConfig(
70 model=MODEL_NAME,
@@ -138,7 +95,7 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
95 # NOTE: We don't test zero images since the HF processor doesn't support it
96 @pytest.mark.parametrize("num_images", [1, 2])
97 def test_image_mapper_multi(image_assets, mm_registry, num_images):
141 - MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
98 + MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
99
100 model_config = ModelConfig(
101 model=MODEL_NAME,
tests/multimodal/test_processing.py
+195 -82
@@ -3,50 +3,15 @@ from typing import cast
3 import pytest
4 from transformers import BatchFeature
5
6 -from vllm.multimodal.processing import (PromptReplacement, find_text_matches,
7 - find_token_matches, iter_token_matches,
8 - iter_token_runs, replace_text_matches)
6 +from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
7 + find_text_matches, find_token_matches,
8 + iter_placeholders, iter_token_matches,
9 + replace_text_matches,
10 + replace_token_matches)
11 from vllm.transformers_utils.tokenizer import AnyTokenizer
12 from vllm.utils import full_groupby
13
14
13 -# yapf: disable
14 -@pytest.mark.parametrize(
15 - ("token_ids", "expected"),
16 - [
17 - ([], []),
18 - (
19 - [32000, 32000, 32000],
20 - [{ "token_id": 32000, "start_idx": 0, "length": 3 }],
21 - ),
22 - (
23 - [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
24 - [
25 - { "token_id": 9833, "start_idx": 0, "length": 1 },
26 - { "token_id": 28747, "start_idx": 1, "length": 1 },
27 - { "token_id": 32000, "start_idx": 2, "length": 3 },
28 - { "token_id": 9833, "start_idx": 5, "length": 1 },
29 - { "token_id": 28747, "start_idx": 6, "length": 1 },
30 - { "token_id": 32000, "start_idx": 7, "length": 2 },
31 - { "token_id": 918, "start_idx": 9, "length": 1 },
32 - ],
33 - ),
34 - ],
35 -)
36 -# yapf: enable
37 -def test_iter_token_runs(token_ids, expected):
38 - result = list(iter_token_runs(token_ids))
39 -
40 - # Only displayed on error
41 - print("result:", result)
42 -
43 - # Manually constructed results
44 - assert [item._asdict() for item in result] == expected
45 -
46 - # Invariants
47 - assert sum(run_info.length for run_info in result) == len(token_ids)
48 -
49 -
15 # yapf: disable
16 @pytest.mark.parametrize(
17 ("token_ids", "match_ids", "expected"),
@@ -170,13 +135,11 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
135 # Should not be used since there is nothing to convert to token IDs
136 mock_tokenizer = cast(AnyTokenizer, object())
137
173 - result = find_token_matches(
174 - prompt,
175 - [
176 - PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
177 - for key, target in target_by_key.items()
178 - ],
179 - )
138 + prompt_repls = [
139 + PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
140 + for key, target in target_by_key.items()
141 + ]
142 + result = find_token_matches(prompt, prompt_repls)
143
144 # Only displayed on error
145 print("result:", result)
@@ -279,13 +242,11 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
242 # Should not be used since there is nothing to convert to text
243 mock_tokenizer = cast(AnyTokenizer, object())
244
282 - result = find_text_matches(
283 - prompt,
284 - [
285 - PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
286 - for key, target in target_by_key.items()
287 - ],
288 - )
245 + prompt_repls = [
246 + PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
247 + for key, target in target_by_key.items()
248 + ]
249 + result = find_text_matches(prompt, prompt_repls)
250
251 # Only displayed on error
252 print("result:", result)
@@ -303,7 +264,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
264
265 # yapf: disable
266 @pytest.mark.parametrize(
306 - ("prompt", "target_by_key", "repl_by_key", "expected_by_mm_count"),
267 + ("prompt", "target_by_key", "repl_by_key"),
268 [
269 (
270 "Image:<image>Image:<image><image>!",
@@ -322,49 +283,201 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
283 # Test multiple repl_count
284 "pattern_3": ("?", 2),
285 },
325 - {
326 - # Test no replacement
327 - 0: "Image:<image>Image:<image><image>!",
328 - # Test single replacement
329 - 1: "<image><image>Image:<image><image>??",
330 - # Test repeated replacement
331 - 2: "<image><image><image><image><image>??",
332 - },
286 ),
287 ]
288 )
289 +@pytest.mark.parametrize(
290 + ("mm_count", "expected"),
291 + [
292 + (0, "Image:<image>Image:<image><image>!"),
293 + (1, "<image><image>Image:<image><image>??"),
294 + (2, "<image><image><image><image><image>??"),
295 + ]
296 +)
297 # yapf: enable
298 def test_find_replace_text(
299 prompt,
300 target_by_key,
301 repl_by_key,
341 - expected_by_mm_count,
302 + mm_count,
303 + expected,
304 ):
305 # Should not be used since there is nothing to convert to text
306 mock_tokenizer = cast(AnyTokenizer, object())
307
346 - matches = find_text_matches(
308 + prompt_repls = [
309 + PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
310 + for key, target in target_by_key.items()
311 + ]
312 + matches = find_text_matches(prompt, prompt_repls)
313 +
314 + result = replace_text_matches(
315 prompt,
348 - [
349 - PromptReplacement(target, *repl_by_key[key]) \
350 - .bind(key, mock_tokenizer)
351 - for key, target in target_by_key.items()
352 - ],
316 + matches,
317 + {key: list(range(mm_count))
318 + for key in repl_by_key},
319 + BatchFeature(),
320 )
354 - result_by_mm_count = {
355 - mm_count: replace_text_matches(
356 - prompt,
357 - matches,
358 - {key: list(range(mm_count))
359 - for key in repl_by_key},
360 - BatchFeature(),
361 - )
362 - for mm_count in expected_by_mm_count
363 - }
321
322 # Only displayed on error
323 print("matches:", matches)
367 - print("result_by_mm_count:", result_by_mm_count)
324 + print("result:", result)
325 +
326 + # Manually constructed results
327 + assert result == expected
328 +
329 +
330 +# yapf: disable
331 +@pytest.mark.parametrize(
332 + ("prompt", "target_by_key", "repl_by_key"),
333 + [
334 + # Tokenized test cases of `test_find_replace_text`
335 + # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
336 + (
337 + [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
338 + {
339 + # We use `<image>` before `Image:` to test matches that
340 + # occur out of order
341 + "pattern_1": [32000],
342 + "pattern_2": [9833, 28747],
343 + "pattern_3": [918],
344 + },
345 + {
346 + # Test whether target is confused with repl_unit
347 + "pattern_1": ([32000, 32000], 1),
348 + # Test empty repl_unit
349 + "pattern_2": ([], 1),
350 + # Test multiple repl_count
351 + "pattern_3": ([1550], 2),
352 + },
353 + ),
354 + ]
355 +)
356 +@pytest.mark.parametrize(
357 + ("mm_count", "expected"),
358 + [
359 + (0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
360 + (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550]),
361 + (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550]),
362 + ]
363 +)
364 +# yapf: enable
365 +def test_find_replace_tokens(
366 + prompt,
367 + target_by_key,
368 + repl_by_key,
369 + mm_count,
370 + expected,
371 +):
372 + # Should not be used since there is nothing to convert to tokens
373 + mock_tokenizer = cast(AnyTokenizer, object())
374 +
375 + prompt_repls = [
376 + PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
377 + for key, target in target_by_key.items()
378 + ]
379 + matches = find_token_matches(prompt, prompt_repls)
380 +
381 + result = replace_token_matches(
382 + prompt,
383 + matches,
384 + {key: list(range(mm_count))
385 + for key in repl_by_key},
386 + BatchFeature(),
387 + )
388 +
389 + # Only displayed on error
390 + print("matches:", matches)
391 + print("result:", result)
392 +
393 + # Manually constructed results
394 + assert result == expected
395 +
396 +
397 +# yapf: disable
398 +@pytest.mark.parametrize(
399 + "repl_by_key",
400 + [
401 + {
402 + "pattern_1": ([32000, 32000], 1),
403 + "pattern_2": ([], 1),
404 + "pattern_3": ([1550], 2),
405 + },
406 + ],
407 +)
408 +@pytest.mark.parametrize(
409 + ("prompt", "expected"),
410 + [
411 + (
412 + [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
413 + [
414 + _PlaceholderInfo(
415 + modality="pattern_1",
416 + start_idx=6,
417 + unit=[32000, 32000],
418 + unit_count=1,
419 + ),
420 + ],
421 + ),
422 + (
423 + [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550],
424 + [
425 + _PlaceholderInfo(
426 + modality="pattern_1",
427 + start_idx=1,
428 + unit=[32000, 32000],
429 + unit_count=1,
430 + ),
431 + _PlaceholderInfo(
432 + modality="pattern_1",
433 + start_idx=5,
434 + unit=[32000, 32000],
435 + unit_count=1,
436 + ),
437 + _PlaceholderInfo(
438 + modality="pattern_3",
439 + start_idx=7,
440 + unit=[1550],
441 + unit_count=2,
442 + ),
443 + ],
444 + ),
445 + (
446 + [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550],
447 + [
448 + _PlaceholderInfo(
449 + modality="pattern_1",
450 + start_idx=1,
451 + unit=[32000, 32000],
452 + unit_count=2,
453 + ),
454 + _PlaceholderInfo(
455 + modality="pattern_3",
456 + start_idx=6,
457 + unit=[1550],
458 + unit_count=2,
459 + ),
460 + ],
461 + ),
462 + ]
463 +)
464 +def test_iter_placeholders(
465 + repl_by_key,
466 + prompt,
467 + expected,
468 +):
469 + # Should not be used since there is nothing to convert to tokens
470 + mock_tokenizer = cast(AnyTokenizer, object())
471 +
472 + prompt_repls = [
473 + PromptReplacement([], *repl).bind(key, mock_tokenizer)
474 + for key, repl in repl_by_key.items()
475 + ]
476 +
477 + result = list(iter_placeholders(prompt_repls, prompt))
478 +
479 + # Only displayed on error
480 + print("result:", result)
481
482 # Manually constructed results
370 - assert result_by_mm_count == expected_by_mm_count
483 + assert result == expected
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+5 -7
@@ -2,19 +2,17 @@ from typing import Optional
2
3 import torch
4
5 -from vllm.inputs import INPUT_REGISTRY
5 from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
7 - dummy_data_for_llava,
8 - get_max_llava_image_tokens,
9 - input_processor_for_llava)
6 + create_metadata_for_llava,
7 + dummy_mm_kwargs_for_llava,
8 + get_max_llava_image_tokens)
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.multimodal import MULTIMODAL_REGISTRY
11
12
14 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
13 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
16 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
17 -@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
14 +@MULTIMODAL_REGISTRY.register_processor_by_metadata(create_metadata_for_llava,
15 + dummy_mm_kwargs_for_llava)
16 class MyLlava(LlavaForConditionalGeneration):
17
18 def compute_logits(
vllm/inputs/registry.py
+30 -12
@@ -232,19 +232,35 @@ class InputRegistry:
232 """
233 # Avoid circular import
234 from vllm.model_executor.model_loader import get_model_architecture
235 -
236 - model_cls, _ = get_model_architecture(model_config)
237 - if is_encoder_data:
238 - dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
235 + from vllm.multimodal import MultiModalKwargs
236 + from vllm.multimodal.utils import cached_get_tokenizer
237 +
238 + if mm_registry.has_processor(model_config):
239 + tokenizer = cached_get_tokenizer(
240 + model_config.tokenizer,
241 + trust_remote_code=model_config.trust_remote_code,
242 + )
243 + processor = mm_registry.create_processor(model_config, tokenizer)
244 +
245 + mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
246 + mm_max_tokens = mm_registry.get_max_tokens_by_modality(
247 + model_config)
248 +
249 + dummy_data = processor.get_dummy_data(seq_len, mm_counts,
250 + mm_max_tokens)
251 else:
240 - dummy_factory = self._get_dummy_data_factory(model_cls)
241 - mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
242 - mm_processor_kwargs = get_allowed_kwarg_only_overrides(
243 - dummy_factory, overrides=model_config.mm_processor_kwargs)
252 + model_cls, _ = get_model_architecture(model_config)
253 + if is_encoder_data:
254 + dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
255 + else:
256 + dummy_factory = self._get_dummy_data_factory(model_cls)
257 + mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
258 + mm_processor_kwargs = get_allowed_kwarg_only_overrides(
259 + dummy_factory, overrides=model_config.mm_processor_kwargs)
260
245 - dummy_data = dummy_factory(InputContext(model_config), seq_len,
246 - _MultiModalCounts(mm_counts),
247 - **mm_processor_kwargs)
261 + dummy_data = dummy_factory(InputContext(model_config), seq_len,
262 + _MultiModalCounts(mm_counts),
263 + **mm_processor_kwargs)
264
265 # Having more tokens is over-conservative but otherwise fine
266 num_tokens = dummy_data.seq_data.prompt_token_ids
@@ -257,7 +273,9 @@ class InputRegistry:
273 raise AssertionError(
274 f"Expected at least {seq_len} dummy tokens for profiling, "
275 f"but found {len(num_tokens)} tokens instead.")
260 - if dummy_data.multi_modal_data is not None:
276 +
277 + if (dummy_data.multi_modal_data is not None and
278 + not isinstance(dummy_data.multi_modal_data, MultiModalKwargs)):
279 for k, v in dummy_data.multi_modal_data.items():
280 num_items = len(v) if isinstance(v, list) else 1
281 num_expected = mm_counts[k]
vllm/model_executor/models/llava.py
+88 -131
@@ -1,17 +1,19 @@
1 from functools import cached_property
2 +from types import MethodType
3 from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
4 Tuple, TypedDict, Union)
5
6 import torch
7 import torch.nn as nn
7 -from PIL import Image
8 -from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9 - PretrainedConfig, SiglipVisionConfig)
8 +from PIL.Image import Image
9 +from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
10 + PixtralVisionConfig, PretrainedConfig,
11 + ProcessorMixin, SiglipVisionConfig)
12 +from transformers.models.pixtral import PixtralProcessor
13
14 from vllm.attention import AttentionMetadata
15 from vllm.config import VllmConfig
13 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14 - InputContext)
16 +from vllm.inputs import InputContext
17 from vllm.model_executor.layers.activation import get_act_fn
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 RowParallelLinear)
@@ -19,21 +21,20 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
22 -from vllm.multimodal.inputs import NestedTensors
24 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
25 +from vllm.multimodal.processing import (InputProcessingContext,
26 + ModalityProcessingMetadata,
27 + MultiModalProcessingMetadata,
28 + MultiModalProcessor, PromptReplacement)
29 from vllm.sequence import IntermediateTensors
24 -from vllm.utils import is_list_of
30
31 from .clip import (CLIPVisionModel, dummy_image_for_clip,
27 - dummy_seq_data_for_clip, get_max_clip_image_tokens,
28 - input_processor_for_clip)
32 + get_max_clip_image_tokens)
33 from .interfaces import SupportsMultiModal, SupportsPP
34 from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
31 - dummy_seq_data_for_pixtral_hf,
32 - get_max_pixtral_hf_image_tokens,
33 - input_processor_for_pixtral_hf)
35 + get_max_pixtral_hf_image_tokens)
36 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
35 - dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
36 - input_processor_for_siglip)
37 + get_max_siglip_image_tokens)
38 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39 maybe_prefix, merge_multimodal_embeddings)
40
@@ -113,102 +114,86 @@ def get_max_llava_image_tokens(ctx: InputContext):
114 raise ValueError(f"Unexpected select feature strategy: {strategy}")
115
116
116 -def dummy_data_for_llava(ctx: InputContext, seq_len: int,
117 - mm_counts: Mapping[str, int]):
117 +def dummy_mm_kwargs_for_llava(ctx: InputProcessingContext,
118 + mm_counts: Mapping[str, int]):
119 hf_config = ctx.get_hf_config(LlavaConfig)
120 vision_config = hf_config.vision_config
121 num_images = mm_counts["image"]
122
122 - image_feature_size = get_max_llava_image_tokens(ctx)
123 -
123 if isinstance(vision_config, CLIPVisionConfig):
125 - seq_data, ranges = dummy_seq_data_for_clip(
126 - vision_config,
127 - seq_len,
128 - num_images,
129 - image_token_id=hf_config.image_token_index,
130 - image_feature_size_override=image_feature_size,
131 - )
132 -
133 - mm_data = dummy_image_for_clip(vision_config, num_images)
134 - return DummyData(seq_data, mm_data, ranges)
124 + data = dummy_image_for_clip(vision_config, num_images)
125 elif isinstance(vision_config, SiglipVisionConfig):
136 - seq_data, ranges = dummy_seq_data_for_siglip(
137 - vision_config,
138 - seq_len,
139 - num_images,
140 - image_token_id=hf_config.image_token_index,
141 - image_feature_size_override=image_feature_size,
142 - )
143 -
144 - mm_data = dummy_image_for_siglip(vision_config, num_images)
145 - return DummyData(seq_data, mm_data, ranges)
126 + data = dummy_image_for_siglip(vision_config, num_images)
127 elif isinstance(vision_config, PixtralVisionConfig):
147 - seq_data, ranges = dummy_seq_data_for_pixtral_hf(
148 - vision_config,
149 - seq_len,
150 - num_images,
151 - image_token_id=hf_config.image_token_index,
152 - image_feature_size_override=image_feature_size,
153 - )
154 -
155 - mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
156 - return DummyData(seq_data, mm_data, ranges)
128 + data = dummy_image_for_pixtral_hf(vision_config, num_images)
129 + else:
130 + msg = f"Unsupported vision config: {type(vision_config)}"
131 + raise NotImplementedError(msg)
132
158 - msg = f"Unsupported vision config: {type(vision_config)}"
159 - raise NotImplementedError(msg)
133 + hf_processor = ctx.get_hf_processor()
134 + image_processor = hf_processor.image_processor # type: ignore
135 + hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
136 + is_pixtral = isinstance(hf_processor, PixtralProcessor)
137
138 + return MultiModalKwargs(
139 + **hf_inputs,
140 + is_pixtral=torch.tensor(is_pixtral),
141 + )
142
162 -def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
163 - multi_modal_data = inputs.get("multi_modal_data")
164 - if multi_modal_data is None or "image" not in multi_modal_data:
165 - return inputs
143
167 - model_config = ctx.model_config
144 +def create_metadata_for_llava(
145 + ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
146 hf_config = ctx.get_hf_config(LlavaConfig)
169 - vision_config = hf_config.vision_config
147 + image_token_id = hf_config.image_token_index
148 +
149 + def get_repl_count(
150 + mm_items: list[Image],
151 + hf_inputs: BatchFeature,
152 + item_idx: int,
153 + ) -> int:
154 + return get_max_llava_image_tokens(ctx)
155 +
156 + return {
157 + "image":
158 + ModalityProcessingMetadata(prompt_repls=[
159 + PromptReplacement(target=[image_token_id],
160 + repl_unit=[image_token_id],
161 + repl_count=get_repl_count),
162 + ]),
163 + }
164
171 - image_data = multi_modal_data["image"]
172 - if isinstance(image_data, Image.Image):
173 - image_feature_size = get_max_llava_image_tokens(ctx)
174 - elif is_list_of(image_data, Image.Image):
175 - image_feature_size = [get_max_llava_image_tokens(ctx)
176 - ] * len(image_data)
177 - elif isinstance(image_data, torch.Tensor):
178 - num_images, image_feature_size, hidden_size = image_data.shape
179 - elif is_list_of(image_data, torch.Tensor):
180 - image_feature_size = [item.shape[1] for item in image_data]
181 - else:
182 - raise TypeError(f"Invalid image type: {type(image_data)}")
165
184 - if isinstance(vision_config, CLIPVisionConfig):
185 - return input_processor_for_clip(
186 - model_config,
187 - vision_config,
188 - inputs,
189 - image_token_id=hf_config.image_token_index,
190 - image_feature_size_override=image_feature_size,
191 - )
192 - elif isinstance(vision_config, SiglipVisionConfig):
193 - return input_processor_for_siglip(
194 - model_config,
195 - vision_config,
196 - inputs,
197 - image_token_id=hf_config.image_token_index,
198 - image_feature_size_override=image_feature_size,
199 - )
200 - elif isinstance(vision_config, PixtralVisionConfig):
201 - # We ignore image_feature_size_override since we have non-uniform
202 - # image sizes for Pixtral
203 - return input_processor_for_pixtral_hf(
204 - model_config,
205 - vision_config,
206 - inputs,
207 - image_token_id=hf_config.image_token_index,
208 - )
166 +class LlavaProcessor(MultiModalProcessor):
167
210 - msg = f"Unsupported vision config: {type(vision_config)}"
211 - raise NotImplementedError(msg)
168 + def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
169 + if getattr(hf_processor, "__is_patched__", False):
170 + return # Already patched
171 +
172 + image_processor = hf_processor.image_processor # type: ignore
173 + orig_preprocess = image_processor.preprocess
174 +
175 + def preprocess(__self, *args, **kwargs):
176 + hf_inputs = orig_preprocess(*args, **kwargs)
177 + hf_inputs["is_pixtral"] = torch.tensor(True)
178 + return hf_inputs
179 +
180 + image_processor.preprocess = MethodType(preprocess, image_processor)
181 +
182 + hf_processor.__is_patched__ = True # type: ignore
183 +
184 + def _get_hf_processor(self) -> ProcessorMixin:
185 + hf_processor = self.ctx.get_hf_processor()
186 +
187 + if isinstance(hf_processor, PixtralProcessor):
188 + self._patch_pixtral_processor(hf_processor)
189 +
190 + return hf_processor
191 +
192 + def _get_dummy_mm_kwargs(
193 + self,
194 + mm_counts: Mapping[str, int],
195 + ) -> MultiModalKwargs:
196 + return dummy_mm_kwargs_for_llava(self.ctx, mm_counts)
197
198
199 class LlavaLikeConfig(Protocol):
@@ -291,10 +276,11 @@ def init_vision_tower_for_llava(
276 raise NotImplementedError(msg)
277
278
294 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
279 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
296 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
297 -@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
280 +@MULTIMODAL_REGISTRY.register_processor(lambda ctx: LlavaProcessor(
281 + ctx=ctx,
282 + metadata=create_metadata_for_llava(ctx),
283 +))
284 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
285 # BitandBytes specific attributes
286 bitsandbytes_stacked_params_mapping = {
@@ -367,38 +353,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
353
354 return data
355
370 - def _validate_image_sizes(self, images: List[torch.Tensor],
371 - sizes: List[torch.Tensor]) -> List[torch.Tensor]:
372 - if not isinstance(sizes, list):
373 - sizes = [sizes]
374 -
375 - total_images = sum(size.numel() // 2 for size in sizes)
376 - if total_images != len(images):
377 - raise ValueError("Mismatch in number of images. "
378 - f"Expected {total_images}, got {len(images)}")
379 - img_idx = 0
380 - for size in sizes:
381 - # Flatten the size tensor to a list of (height, width) pairs
382 - size = size.view(-1, 2).tolist()
383 - for expected_h, expected_w in size:
384 - if img_idx >= len(images):
385 - raise ValueError("Ran out of images before sizes. "
386 - f"{img_idx} >= {len(images)}")
387 - img = images[img_idx]
388 - if img.shape[-2:] != (expected_h, expected_w):
389 - raise ValueError(
390 - "Image size mismatch. Expected "
391 - f"{(expected_h, expected_w)}, got {img.shape[-2:]}")
392 - if img.shape[-3] != 3:
393 - raise ValueError("Image channel mismatch. Expected 3, "
394 - f"got {img.shape[-3]}")
395 - img_idx += 1
396 - return images
397 -
356 def _parse_and_validate_image_input(
357 self, **kwargs: object) -> Optional[LlavaImageInputs]:
358 pixel_values = kwargs.pop("pixel_values", None)
401 - image_sizes = kwargs.pop("image_sizes", None)
359 + is_pixtral = kwargs.pop("is_pixtral", torch.tensor([False]))
360 image_embeds = kwargs.pop("image_embeds", None)
361
362 if pixel_values is None and image_embeds is None:
@@ -409,9 +367,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
367 raise ValueError("Incorrect type of pixel values. "
368 f"Got type: {type(pixel_values)}")
369
412 - # Case for models like PixtralHF that have dynamic image sizes
413 - # so we need to produce a list of tensors
414 - if image_sizes is not None:
370 + assert isinstance(is_pixtral, torch.Tensor)
371 + if is_pixtral.any():
372 images = pixel_values
373
374 def flatten_to_3d_tensors(item):
@@ -434,7 +391,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
391
392 return LlavaImagePixelInputs(
393 type="pixel_values",
437 - data=self._validate_image_sizes(images, image_sizes),
394 + data=images,
395 )
396
397 return LlavaImagePixelInputs(
vllm/multimodal/base.py
+36 -15
@@ -226,16 +226,16 @@ class MultiModalPlugin(ABC):
226 """
227 # Avoid circular import
228 from vllm.model_executor.model_loader import get_model_architecture
229 + from vllm.model_executor.models import supports_multimodal
230
231 model_cls, _ = get_model_architecture(model_config)
232
232 - if model_cls not in self._input_mappers:
233 + if not supports_multimodal(model_cls):
234 return 0
235
236 max_mm_tokens = self._max_mm_tokens.get(model_cls)
237 if max_mm_tokens is None:
237 - raise KeyError(f"No maximum number of multi-modal tokens is given "
238 - f"for model class {model_cls.__name__} in {self}.")
238 + return 0
239
240 if callable(max_mm_tokens):
241 mm_processor_kwargs = get_allowed_kwarg_only_overrides(
@@ -326,26 +326,47 @@ class MultiModalPlaceholderMap:
326 src_ranges = []
327 dest_ranges = []
328 """
329 - if (not seq_group.multi_modal_data
330 - or not seq_group.multi_modal_placeholders):
331 - return seq_group.multi_modal_data, {}
329 + seq_mm_data = seq_group.multi_modal_data
330 + seq_mm_placeholders = seq_group.multi_modal_placeholders
331 +
332 + if not seq_mm_data or not seq_mm_placeholders:
333 + return seq_mm_data, {}
334 +
335 + # For merged processor, we directly use mm_kwargs as mm_data
336 + if isinstance(seq_mm_data, MultiModalKwargs):
337 + placeholder_maps = dict[str, MultiModalPlaceholderMap]()
338 +
339 + for modality, placeholders in seq_mm_placeholders.items():
340 + placeholder_map = MultiModalPlaceholderMap()
341 +
342 + if positions:
343 + placeholder_map.append_items_from_seq_group(
344 + positions,
345 + # Dummy, since we don't care about intersecting items
346 + [None] * len(placeholders),
347 + placeholders,
348 + )
349 +
350 + placeholder_maps[modality] = placeholder_map
351 +
352 + return seq_mm_data, placeholder_maps
353
333 - mm_data = {**seq_group.multi_modal_data}
334 - placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
354 + mm_data = {**seq_mm_data}
355 + placeholder_maps = defaultdict[str, MultiModalPlaceholderMap](
356 MultiModalPlaceholderMap)
357
337 - for (
338 - modality,
339 - placeholders,
340 - ) in seq_group.multi_modal_placeholders.items():
358 + for modality, placeholders in seq_mm_placeholders.items():
359 mm_items = mm_data.pop(modality)
360 if not isinstance(mm_items, list):
361 mm_items = [mm_items]
362
363 if positions:
346 - intersecting_items = placeholder_maps[
347 - modality].append_items_from_seq_group(
348 - positions, mm_items, placeholders)
364 + intersecting_items = placeholder_maps[modality] \
365 + .append_items_from_seq_group(
366 + positions,
367 + mm_items,
368 + placeholders,
369 + )
370
371 if intersecting_items:
372 mm_data[modality] = intersecting_items
vllm/multimodal/processing.py
+197 -116
@@ -3,14 +3,13 @@ from abc import ABC, abstractmethod
3 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4 from dataclasses import dataclass
5 from functools import lru_cache
6 -from itertools import groupby
6 from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
7
9 -import numpy as np
10 -from transformers import BatchFeature
8 +import torch
9 +from transformers import BatchFeature, ProcessorMixin
10 from typing_extensions import TypeAlias, TypedDict
11
13 -from vllm.inputs import InputProcessingContext
12 +from vllm.inputs import DummyData, InputProcessingContext
13 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
14 from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
15
@@ -256,63 +255,6 @@ def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
255 return multi_data
256
257
259 -class _TokenRun(NamedTuple):
260 - token_id: int
261 -
262 - start_idx: int
263 - length: int
264 -
265 -
266 -def iter_token_runs(token_ids: list[int]) -> Iterable[_TokenRun]:
267 - """
268 - Yield the starting index and length of each run of tokens that are the same.
269 - """
270 - start_idx = 0
271 -
272 - for token_id, it in groupby(token_ids):
273 - length = sum(1 for _ in it)
274 - yield _TokenRun(token_id=token_id, start_idx=start_idx, length=length)
275 -
276 - start_idx += length
277 -
278 -
279 -class _PlaceholderInfo(NamedTuple):
280 - modality: str
281 - offset: int
282 - length: int
283 -
284 - def to_range(self) -> PlaceholderRange:
285 - return PlaceholderRange(offset=self.offset, length=self.length)
286 -
287 -
288 -def iter_placeholders(
289 - prompt_repls: Sequence[_BoundPromptReplacement[Any]],
290 - token_ids: list[int],
291 - *,
292 - min_placeholder_count: int,
293 -) -> Iterable[_PlaceholderInfo]:
294 - """Yield each set of placeholder tokens found in :code:`token_ids`."""
295 - placeholder_ids_by_modality = {
296 - modality: {
297 - token_id
298 - for prompt_repl in repls
299 - for token_id in prompt_repl.repl_unit.token_ids
300 - }
301 - for modality, repls in full_groupby_modality(prompt_repls)
302 - }
303 -
304 - for run_info in iter_token_runs(token_ids):
305 - if run_info.length > min_placeholder_count:
306 - for (modality,
307 - placeholder_ids) in placeholder_ids_by_modality.items():
308 - if run_info.token_id in placeholder_ids:
309 - yield _PlaceholderInfo(
310 - modality=modality,
311 - offset=run_info.start_idx,
312 - length=run_info.length,
313 - )
314 -
315 -
258 class _TokenMatch(NamedTuple):
259 start_idx: int
260 end_idx: int
@@ -353,13 +295,9 @@ class _PromptReplacementMatch(ABC, Generic[_T, _S]):
295 def end_idx(self) -> int:
296 raise NotImplementedError
297
298 + @property
299 @abstractmethod
357 - def get_repl(
358 - self,
359 - mm_items: list[_T],
360 - hf_inputs: BatchFeature,
361 - item_idx: int,
362 - ) -> _S:
300 + def repl_unit(self) -> _S:
301 raise NotImplementedError
302
303 def __repr__(self) -> str:
@@ -380,15 +318,9 @@ class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
318 def end_idx(self) -> int:
319 return self.match.end_idx
320
383 - def get_repl(
384 - self,
385 - mm_items: list[_T],
386 - hf_inputs: BatchFeature,
387 - item_idx: int,
388 - ) -> list[int]:
389 - prompt_repl = self.prompt_repl
390 - count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
391 - return prompt_repl.repl_unit.token_ids * count
321 + @property
322 + def repl_unit(self) -> list[int]:
323 + return self.prompt_repl.repl_unit.token_ids
324
325
326 @dataclass(repr=False)
@@ -404,15 +336,26 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
336 def end_idx(self) -> int:
337 return self.match.end()
338
407 - def get_repl(
408 - self,
409 - mm_items: list[_T],
410 - hf_inputs: BatchFeature,
411 - item_idx: int,
412 - ) -> str:
413 - prompt_repl = self.prompt_repl
414 - count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
415 - return prompt_repl.repl_unit.text * count
339 + @property
340 + def repl_unit(self) -> str:
341 + return self.prompt_repl.repl_unit.text
342 +
343 +
344 +class _PlaceholderInfo(NamedTuple):
345 + modality: str
346 + start_idx: int
347 + unit: list[int]
348 + unit_count: int
349 +
350 + @property
351 + def length(self) -> int:
352 + return len(self.unit) * self.unit_count
353 +
354 + def to_range(self) -> PlaceholderRange:
355 + return PlaceholderRange(
356 + offset=self.start_idx,
357 + length=self.length,
358 + )
359
360
361 def find_token_matches(
@@ -447,15 +390,17 @@ def _resolve_matches(
390 Resolve :code:`matches` to ensure that there are no overlapping matches,
391 and sort them such that earlier matches take priority over later ones.
392 """
450 - num_matches_by_idx = np.zeros(len(prompt), dtype=int)
393 + seen_matches: list[Optional[_PromptReplacementMatch[_T, _S]]] \
394 + = [None] * len(prompt)
395 +
396 for match in matches:
452 - num_matches_by_idx[match.start_idx:match.end_idx] += 1
397 + for idx in range(match.start_idx, match.end_idx):
398 + if seen_matches[idx] is not None:
399 + raise ValueError("Found overlapping matches "
400 + f"({seen_matches[idx]} and {match}) "
401 + f"at index={idx} of prompt={prompt}")
402
454 - duplicate_matches_idxs, = np.nonzero(num_matches_by_idx > 1)
455 - if len(duplicate_matches_idxs) > 0:
456 - raise ValueError("Unable to find a unique replacement "
457 - f"at indices={duplicate_matches_idxs} "
458 - f"of prompt={prompt}")
403 + seen_matches[idx] = match
404
405 return sorted(matches, key=lambda x: x.start_idx)
406
@@ -480,9 +425,12 @@ def _replace_matches(
425
426 start_idx = match.start_idx
427 end_idx = match.end_idx
483 - repl_ids = match.get_repl(mm_items, hf_inputs, item_idx)
428 + repl_unit = match.repl_unit
429 + repl_info = match.prompt_repl
430 + repl_count = repl_info.get_count(mm_items, hf_inputs, item_idx)
431
485 - out_seqs.append(prompt[prev_end_idx:start_idx] + repl_ids)
432 + out_seqs.append(prompt[prev_end_idx:start_idx] +
433 + repl_unit * repl_count)
434 prev_end_idx = end_idx
435 next_idx_by_modality[modality] += 1
436
@@ -531,7 +479,57 @@ def replace_text_matches(
479 return "".join(texts)
480
481
534 -class MultiModalProcessor:
482 +def _merge_placeholder_matches(
483 + matches: Iterable[_PromptReplacementTokenMatch],
484 +) -> Iterable[_PromptReplacementTokenMatch]:
485 + current_match = None
486 +
487 + for match in sorted(matches, key=lambda x: x.start_idx):
488 + if current_match is None:
489 + current_match = match
490 + elif (current_match.prompt_repl == match.prompt_repl
491 + and current_match.end_idx == match.start_idx):
492 + current_match = _PromptReplacementTokenMatch(
493 + current_match.prompt_repl,
494 + match=_TokenMatch(current_match.start_idx, match.end_idx),
495 + )
496 + else:
497 + yield current_match
498 + current_match = match
499 +
500 + if current_match is not None:
501 + yield current_match
502 +
503 +
504 +def iter_placeholders(
505 + prompt_repls: Sequence[_BoundPromptReplacement[Any]],
506 + prompt: list[int],
507 + *,
508 + min_unit_count: int = 1,
509 +) -> Iterable[_PlaceholderInfo]:
510 + """Yield each set of placeholder tokens found in :code:`token_ids`."""
511 + if min_unit_count <= 0:
512 + raise ValueError("`min_unit_count` must be a positive integer")
513 +
514 + matches = (_PromptReplacementTokenMatch(prompt_repl, match)
515 + for prompt_repl in prompt_repls
516 + if len(repl_unit := prompt_repl.repl_unit.token_ids) > 0
517 + for match in iter_token_matches(prompt, repl_unit))
518 +
519 + for match in _merge_placeholder_matches(matches):
520 + unit = match.repl_unit
521 + placeholder = _PlaceholderInfo(
522 + modality=match.modality,
523 + start_idx=match.start_idx,
524 + unit=unit,
525 + unit_count=(match.end_idx - match.start_idx) // len(unit),
526 + )
527 +
528 + if placeholder.unit_count >= min_unit_count:
529 + yield placeholder
530 +
531 +
532 +class MultiModalProcessor(ABC):
533 """
534 Helper class to process multi-modal inputs to be used in vLLM.
535 """
@@ -546,6 +544,12 @@ class MultiModalProcessor:
544 self.ctx = ctx
545 self.metadata = metadata
546
547 + def _get_hf_processor(self) -> ProcessorMixin:
548 + return self.ctx.get_hf_processor()
549 +
550 + def _get_tokenizer(self) -> AnyTokenizer:
551 + return self.ctx.tokenizer
552 +
553 def __call__(
554 self,
555 prompt: str,
@@ -562,13 +566,13 @@ class MultiModalProcessor:
566 # To avoid false positives from multi-input when detecting
567 # whether placeholder tokens have been inserted, in case
568 # the target sequence is a subset of the replacement tokens
565 - min_placeholder_count: int = 16,
569 + min_unit_count: int = 16,
570 ) -> list[_PlaceholderInfo]:
571 return list(
572 iter_placeholders(
573 all_prompt_repls,
574 new_token_ids,
571 - min_placeholder_count=min_placeholder_count,
575 + min_unit_count=min_unit_count,
576 ))
577
578 def _apply_hf_processor(
@@ -577,19 +581,49 @@ class MultiModalProcessor:
581 mm_data: MultiModalDataDict,
582 mm_processor_kwargs: Mapping[str, object],
583 ) -> BatchFeature:
580 - hf_processor = self.ctx.get_hf_processor()
584 + hf_processor = self._get_hf_processor()
585 +
586 + processor_data = dict[str, Any]()
587 + passthrough_data = dict[str, Any]()
588 + for k, v in mm_data.items():
589 + # TODO: Make a separate modality for embedding inputs
590 + # to avoid confusion
591 + if k in ("image", "video", "audio"):
592 + if isinstance(v, torch.Tensor) and v.ndim == 3:
593 + # Pass through embedding inputs (single)
594 + passthrough_data[f"{k}_embeds"] = [v]
595 + elif is_list_of(v, torch.Tensor) and v[0].ndim == 2:
596 + # Pass through embedding inputs (multi)
597 + passthrough_data[f"{k}_embeds"] = v
598 + else:
599 + # Map keys to plural form, e.g.: image -> images
600 + processor_data[f"{k}s"] = v
601 + else:
602 + processor_data[k] = v
603 +
604 + try:
605 + hf_inputs = hf_processor(
606 + text=prompt, # type: ignore
607 + **processor_data,
608 + **mm_processor_kwargs,
609 + return_tensors="pt",
610 + )
611 + except Exception as exc:
612 + data = dict(text=prompt, **processor_data)
613
582 - return hf_processor(
583 - text=prompt, # type: ignore
584 - **mm_data,
585 - **mm_processor_kwargs,
586 - )
614 + raise RuntimeError(
615 + f"Failed to apply {type(hf_processor).__name__} "
616 + f"on data={data} with kwargs={mm_processor_kwargs}") from exc
617 +
618 + hf_inputs.update(passthrough_data)
619 +
620 + return hf_inputs
621
622 def _bind_prompt_replacements(
623 self,
624 mm_data: MultiModalDataDict,
625 ) -> list[_BoundPromptReplacement[Any]]:
592 - tokenizer = self.ctx.tokenizer
626 + tokenizer = self._get_tokenizer()
627
628 return [
629 prompt_repl.bind(modality, tokenizer)
@@ -604,7 +638,7 @@ class MultiModalProcessor:
638 token_ids: list[int],
639 prompt_repls: Sequence[_BoundPromptReplacement[Any]],
640 ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
607 - tokenizer = self.ctx.tokenizer
641 + tokenizer = self._get_tokenizer()
642
643 mm_items = to_multi_format(mm_data)
644 token_matches = find_token_matches(token_ids, prompt_repls)
@@ -620,7 +654,7 @@ class MultiModalProcessor:
654 # of the search text in the prompt, we instead perform string
655 # replacement on the decoded token IDs, then encode them back.
656 if all(
623 - len(matches) >= len(mm_data[modality])
657 + len(matches) >= len(mm_items[modality])
658 for modality, matches in full_groupby_modality(token_matches)
659 ): # yapf: disable
660 token_ids = replace_token_matches(
@@ -648,15 +682,6 @@ class MultiModalProcessor:
682
683 placeholders = self._find_placeholders(matched_repls, token_ids)
684
651 - # Sanity check
652 - assert len(placeholders) == len(matched_repls), dict(
653 - # Log this information for easier debugging
654 - text=text,
655 - token_ids=token_ids,
656 - placeholders=placeholders,
657 - matched_repls=matched_repls,
658 - )
659 -
685 return token_ids, text, placeholders
686
687 def apply(
@@ -678,7 +703,7 @@ class MultiModalProcessor:
703 3. Extract information about the placeholder tokens from the
704 processed token IDs.
705 """
681 - tokenizer = self.ctx.tokenizer
706 + tokenizer = self._get_tokenizer()
707
708 hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
709 mm_processor_kwargs)
@@ -717,3 +742,59 @@ class MultiModalProcessor:
742 mm_kwargs=mm_kwargs,
743 mm_placeholders=mm_placeholders,
744 )
745 +
746 + @abstractmethod
747 + def _get_dummy_mm_kwargs(
748 + self,
749 + mm_counts: Mapping[str, int],
750 + ) -> MultiModalKwargs:
751 + """
752 + Build the input that corresponds to `mm_max_tokens` in
753 + :meth:`get_dummy_data`.
754 + """
755 + raise NotImplementedError
756 +
757 + def get_dummy_data(
758 + self,
759 + seq_len: int,
760 + mm_counts: Mapping[str, int],
761 + mm_max_tokens: Mapping[str, int],
762 + ) -> DummyData:
763 + # Avoid circular import
764 + from vllm.sequence import SequenceData
765 +
766 + tokenizer = self._get_tokenizer()
767 +
768 + mm_placeholders = dict[str, _PlaceholderInfo]()
769 + offset = 0
770 +
771 + for modality, max_tokens in mm_max_tokens.items():
772 + if max_tokens == 0:
773 + continue
774 +
775 + metadata = self.metadata[modality]
776 + repl = metadata.prompt_repls[0].bind(modality, tokenizer)
777 + repl_token_ids = repl.repl_unit.token_ids
778 +
779 + placeholders = _PlaceholderInfo(
780 + modality=modality,
781 + start_idx=offset,
782 + unit=repl_token_ids,
783 + unit_count=max_tokens // len(repl_token_ids),
784 + )
785 +
786 + mm_placeholders[modality] = placeholders
787 + offset += placeholders.length
788 +
789 + prompt_token_ids = flatten_2d_lists(
790 + [p.unit * p.unit_count for p in mm_placeholders.values()])
791 + prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
792 +
793 + return DummyData(
794 + seq_data=SequenceData.from_seqs(prompt_token_ids),
795 + multi_modal_data=self._get_dummy_mm_kwargs(mm_counts),
796 + multi_modal_placeholders={
797 + modality: [p.to_range()]
798 + for modality, p in mm_placeholders.items()
799 + },
800 + )
vllm/multimodal/registry.py
+60 -7
@@ -15,7 +15,7 @@ from .audio import AudioPlugin
15 from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16 from .image import ImagePlugin
17 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18 -from .processing import MultiModalProcessor
18 +from .processing import MultiModalProcessingMetadata, MultiModalProcessor
19 from .video import VideoPlugin
20
21 if TYPE_CHECKING:
@@ -200,9 +200,12 @@ class MultiModalRegistry:
200 """
201 return self.register_max_multimodal_tokens("image", max_mm_tokens)
202
203 - def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
203 + def get_max_tokens_by_modality(
204 + self,
205 + model_config: "ModelConfig",
206 + ) -> Mapping[str, int]:
207 """
205 - Get the maximum number of multi-modal tokens
208 + Get the maximum number of tokens from each modality
209 for profiling the memory usage of a model.
210
211 See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
@@ -212,9 +215,23 @@ class MultiModalRegistry:
215 """
216 limits_per_plugin = self._limits_by_model[model_config]
217
215 - return sum((limits_per_plugin[key] *
216 - plugin.get_max_multimodal_tokens(model_config))
217 - for key, plugin in self._plugins.items())
218 + return {
219 + key: (limits_per_plugin[key] *
220 + plugin.get_max_multimodal_tokens(model_config))
221 + for key, plugin in self._plugins.items()
222 + }
223 +
224 + def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
225 + """
226 + Get the maximum number of multi-modal tokens
227 + for profiling the memory usage of a model.
228 +
229 + See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
230 +
231 + Note:
232 + This should be called after :meth:`init_mm_limits_per_prompt`.
233 + """
234 + return sum(self.get_max_tokens_by_modality(model_config).values())
235
236 def init_mm_limits_per_prompt(
237 self,
@@ -270,7 +287,8 @@ class MultiModalRegistry:
287 factory: MultiModalProcessorFactory,
288 ):
289 """
273 - Register a multi-modal processor to a model class.
290 + Register a multi-modal processor to a model class. The processor
291 + is constructed lazily, hence a factory method should be passed.
292
293 When the model receives multi-modal data, the provided function is
294 invoked to transform the data into a dictionary of model inputs.
@@ -293,6 +311,41 @@ class MultiModalRegistry:
311
312 return wrapper
313
314 + def register_processor_by_metadata(
315 + self,
316 + metadata_factory: Callable[[InputProcessingContext],
317 + MultiModalProcessingMetadata],
318 + get_dummy_mm_kwargs: Callable[
319 + [InputProcessingContext, Mapping[str, int]], MultiModalKwargs],
320 + ):
321 + """
322 + Convenience method to register a multi-modal processor to a model class
323 + according to a function that constructs its metadata.
324 +
325 + When the model receives multi-modal data, the provided function is
326 + invoked to transform the data into a dictionary of model inputs.
327 +
328 + See also:
329 + - :ref:`input_processing_pipeline`
330 + - :ref:`enabling_multimodal_inputs`
331 + """
332 +
333 + class ConcreteMultiModalProcessor(MultiModalProcessor):
334 +
335 + def _get_dummy_mm_kwargs(
336 + self,
337 + mm_counts: Mapping[str, int],
338 + ) -> MultiModalKwargs:
339 + return get_dummy_mm_kwargs(self.ctx, mm_counts)
340 +
341 + def factory(ctx: InputProcessingContext):
342 + return ConcreteMultiModalProcessor(
343 + ctx=ctx,
344 + metadata=metadata_factory(ctx),
345 + )
346 +
347 + return self.register_processor(factory)
348 +
349 def has_processor(self, model_config: "ModelConfig") -> bool:
350 """
351 Test whether a multi-modal processor is defined for a specific model.
vllm/v1/engine/mm_input_mapper.py
+1
@@ -12,6 +12,7 @@ class MMInputMapper:
12 model_config: ModelConfig,
13 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
14 ):
15 + self.model_config = model_config
16 self.mm_registry = mm_registry
17 self.multi_modal_input_mapper = mm_registry.create_input_mapper(
18 model_config)
vllm/v1/engine/processor.py
+11 -5
@@ -7,7 +7,8 @@ from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
7 from vllm.inputs.parse import is_encoder_decoder_inputs
8 from vllm.inputs.preprocess import InputPreprocessor
9 from vllm.lora.request import LoRARequest
10 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
11 + MultiModalRegistry)
12 from vllm.pooling_params import PoolingParams
13 from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sampling_params import SamplingParams
@@ -101,10 +102,15 @@ class Processor:
102 self.generation_config_fields, eos_token_id)
103
104 # Preprocess multi-modal data
104 - mm_inputs = self.mm_input_mapper.process_inputs(
105 - decoder_inputs.multi_modal_data,
106 - decoder_inputs.mm_processor_kwargs) if len(
107 - decoder_inputs.multi_modal_data) > 0 else None
105 + if len(decoder_inputs.multi_modal_data) == 0:
106 + mm_inputs = None
107 + elif isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
108 + mm_inputs = [decoder_inputs.multi_modal_data]
109 + else:
110 + mm_inputs = self.mm_input_mapper.process_inputs(
111 + decoder_inputs.multi_modal_data,
112 + decoder_inputs.mm_processor_kwargs,
113 + )
114
115 # Make Request for Detokenizer.
116 detokenizer_request = DetokenizerRequest(