A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[3/N] Support and implement merged input processor for LLaVA model (#10676)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Dec 7, 2024 at 16:50 UTC
955fa9533afde0d232e73f079d72239c8a87c636
10 files changed
+626
-421
tests/multimodal/test_mapper.py
+3
-46
@@ -2,7 +2,7 @@ from contextlib import nullcontext
2
3
import numpy as np
4
import pytest
5
-from transformers import CLIPImageProcessor, LlavaNextImageProcessor
5
+from transformers import LlavaNextImageProcessor
6
7
from vllm.config import ModelConfig
8
from vllm.multimodal import MultiModalRegistry
@@ -14,49 +14,6 @@ def mm_registry():
14
return MultiModalRegistry()
15
16
17
-@pytest.mark.parametrize("dtype", ["half", "float"])
18
-@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19
-def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
20
- MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
21
-
22
- hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
23
- assert isinstance(hf_processor, CLIPImageProcessor)
24
-
25
- model_config = ModelConfig(
26
- model=MODEL_NAME,
27
- task="auto",
28
- tokenizer=MODEL_NAME,
29
- tokenizer_mode="auto",
30
- trust_remote_code=False,
31
- seed=0,
32
- dtype=dtype,
33
- revision=None,
34
- limit_mm_per_prompt={"image": 1},
35
- )
36
-
37
- mm_registry.init_mm_limits_per_prompt(model_config)
38
-
39
- for asset in image_assets:
40
- image = rescale_image_size(asset.pil_image, size_factor)
41
-
42
- hf_result = hf_processor.preprocess(
43
- image,
44
- return_tensors="pt",
45
- )
46
- vllm_result = mm_registry.map_input(
47
- model_config,
48
- {"image": image},
49
- )
50
-
51
- assert hf_result.keys() == vllm_result.keys()
52
- for key, hf_tensor in hf_result.items():
53
- hf_arr: np.ndarray = hf_tensor.numpy()
54
- vllm_arr: np.ndarray = vllm_result[key].numpy()
55
-
56
- assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
57
- assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
58
-
59
-
17
@pytest.mark.parametrize("dtype", ["half", "float"])
18
@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
19
def test_llava_next_image_processor(image_assets, mm_registry, dtype,
@@ -107,7 +64,7 @@ def test_llava_next_image_processor(image_assets, mm_registry, dtype,
64
(2, 1, False), (2, 2, True)],
65
)
66
def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
110
- MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
67
+ MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
68
69
model_config = ModelConfig(
70
model=MODEL_NAME,
@@ -138,7 +95,7 @@ def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
95
# NOTE: We don't test zero images since the HF processor doesn't support it
96
@pytest.mark.parametrize("num_images", [1, 2])
97
def test_image_mapper_multi(image_assets, mm_registry, num_images):
141
- MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
98
+ MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
99
100
model_config = ModelConfig(
101
model=MODEL_NAME,
tests/multimodal/test_processing.py
+195
-82
@@ -3,50 +3,15 @@ from typing import cast
3
import pytest
4
from transformers import BatchFeature
5
6
-from vllm.multimodal.processing import (PromptReplacement, find_text_matches,
7
- find_token_matches, iter_token_matches,
8
- iter_token_runs, replace_text_matches)
6
+from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
7
+ find_text_matches, find_token_matches,
8
+ iter_placeholders, iter_token_matches,
9
+ replace_text_matches,
10
+ replace_token_matches)
11
from vllm.transformers_utils.tokenizer import AnyTokenizer
12
from vllm.utils import full_groupby
13
14
13
-# yapf: disable
14
-@pytest.mark.parametrize(
15
- ("token_ids", "expected"),
16
- [
17
- ([], []),
18
- (
19
- [32000, 32000, 32000],
20
- [{ "token_id": 32000, "start_idx": 0, "length": 3 }],
21
- ),
22
- (
23
- [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
24
- [
25
- { "token_id": 9833, "start_idx": 0, "length": 1 },
26
- { "token_id": 28747, "start_idx": 1, "length": 1 },
27
- { "token_id": 32000, "start_idx": 2, "length": 3 },
28
- { "token_id": 9833, "start_idx": 5, "length": 1 },
29
- { "token_id": 28747, "start_idx": 6, "length": 1 },
30
- { "token_id": 32000, "start_idx": 7, "length": 2 },
31
- { "token_id": 918, "start_idx": 9, "length": 1 },
32
- ],
33
- ),
34
- ],
35
-)
36
-# yapf: enable
37
-def test_iter_token_runs(token_ids, expected):
38
- result = list(iter_token_runs(token_ids))
39
-
40
- # Only displayed on error
41
- print("result:", result)
42
-
43
- # Manually constructed results
44
- assert [item._asdict() for item in result] == expected
45
-
46
- # Invariants
47
- assert sum(run_info.length for run_info in result) == len(token_ids)
48
-
49
-
15
# yapf: disable
16
@pytest.mark.parametrize(
17
("token_ids", "match_ids", "expected"),
@@ -170,13 +135,11 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
135
# Should not be used since there is nothing to convert to token IDs
136
mock_tokenizer = cast(AnyTokenizer, object())
137
173
- result = find_token_matches(
174
- prompt,
175
- [
176
- PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
177
- for key, target in target_by_key.items()
178
- ],
179
- )
138
+ prompt_repls = [
139
+ PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
140
+ for key, target in target_by_key.items()
141
+ ]
142
+ result = find_token_matches(prompt, prompt_repls)
143
144
# Only displayed on error
145
print("result:", result)
@@ -279,13 +242,11 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
242
# Should not be used since there is nothing to convert to text
243
mock_tokenizer = cast(AnyTokenizer, object())
244
282
- result = find_text_matches(
283
- prompt,
284
- [
285
- PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
286
- for key, target in target_by_key.items()
287
- ],
288
- )
245
+ prompt_repls = [
246
+ PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
247
+ for key, target in target_by_key.items()
248
+ ]
249
+ result = find_text_matches(prompt, prompt_repls)
250
251
# Only displayed on error
252
print("result:", result)
@@ -303,7 +264,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
264
265
# yapf: disable
266
@pytest.mark.parametrize(
306
- ("prompt", "target_by_key", "repl_by_key", "expected_by_mm_count"),
267
+ ("prompt", "target_by_key", "repl_by_key"),
268
[
269
(
270
"Image:<image>Image:<image><image>!",
@@ -322,49 +283,201 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
283
# Test multiple repl_count
284
"pattern_3": ("?", 2),
285
},
325
- {
326
- # Test no replacement
327
- 0: "Image:<image>Image:<image><image>!",
328
- # Test single replacement
329
- 1: "<image><image>Image:<image><image>??",
330
- # Test repeated replacement
331
- 2: "<image><image><image><image><image>??",
332
- },
286
),
287
]
288
)
289
+@pytest.mark.parametrize(
290
+ ("mm_count", "expected"),
291
+ [
292
+ (0, "Image:<image>Image:<image><image>!"),
293
+ (1, "<image><image>Image:<image><image>??"),
294
+ (2, "<image><image><image><image><image>??"),
295
+ ]
296
+)
297
# yapf: enable
298
def test_find_replace_text(
299
prompt,
300
target_by_key,
301
repl_by_key,
341
- expected_by_mm_count,
302
+ mm_count,
303
+ expected,
304
):
305
# Should not be used since there is nothing to convert to text
306
mock_tokenizer = cast(AnyTokenizer, object())
307
346
- matches = find_text_matches(
308
+ prompt_repls = [
309
+ PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
310
+ for key, target in target_by_key.items()
311
+ ]
312
+ matches = find_text_matches(prompt, prompt_repls)
313
+
314
+ result = replace_text_matches(
315
prompt,
348
- [
349
- PromptReplacement(target, *repl_by_key[key]) \
350
- .bind(key, mock_tokenizer)
351
- for key, target in target_by_key.items()
352
- ],
316
+ matches,
317
+ {key: list(range(mm_count))
318
+ for key in repl_by_key},
319
+ BatchFeature(),
320
)
354
- result_by_mm_count = {
355
- mm_count: replace_text_matches(
356
- prompt,
357
- matches,
358
- {key: list(range(mm_count))
359
- for key in repl_by_key},
360
- BatchFeature(),
361
- )
362
- for mm_count in expected_by_mm_count
363
- }
321
322
# Only displayed on error
323
print("matches:", matches)
367
- print("result_by_mm_count:", result_by_mm_count)
324
+ print("result:", result)
325
+
326
+ # Manually constructed results
327
+ assert result == expected
328
+
329
+
330
+# yapf: disable
331
+@pytest.mark.parametrize(
332
+ ("prompt", "target_by_key", "repl_by_key"),
333
+ [
334
+ # Tokenized test cases of `test_find_replace_text`
335
+ # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
336
+ (
337
+ [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
338
+ {
339
+ # We use `<image>` before `Image:` to test matches that
340
+ # occur out of order
341
+ "pattern_1": [32000],
342
+ "pattern_2": [9833, 28747],
343
+ "pattern_3": [918],
344
+ },
345
+ {
346
+ # Test whether target is confused with repl_unit
347
+ "pattern_1": ([32000, 32000], 1),
348
+ # Test empty repl_unit
349
+ "pattern_2": ([], 1),
350
+ # Test multiple repl_count
351
+ "pattern_3": ([1550], 2),
352
+ },
353
+ ),
354
+ ]
355
+)
356
+@pytest.mark.parametrize(
357
+ ("mm_count", "expected"),
358
+ [
359
+ (0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
360
+ (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550]),
361
+ (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550]),
362
+ ]
363
+)
364
+# yapf: enable
365
+def test_find_replace_tokens(
366
+ prompt,
367
+ target_by_key,
368
+ repl_by_key,
369
+ mm_count,
370
+ expected,
371
+):
372
+ # Should not be used since there is nothing to convert to tokens
373
+ mock_tokenizer = cast(AnyTokenizer, object())
374
+
375
+ prompt_repls = [
376
+ PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
377
+ for key, target in target_by_key.items()
378
+ ]
379
+ matches = find_token_matches(prompt, prompt_repls)
380
+
381
+ result = replace_token_matches(
382
+ prompt,
383
+ matches,
384
+ {key: list(range(mm_count))
385
+ for key in repl_by_key},
386
+ BatchFeature(),
387
+ )
388
+
389
+ # Only displayed on error
390
+ print("matches:", matches)
391
+ print("result:", result)
392
+
393
+ # Manually constructed results
394
+ assert result == expected
395
+
396
+
397
+# yapf: disable
398
+@pytest.mark.parametrize(
399
+ "repl_by_key",
400
+ [
401
+ {
402
+ "pattern_1": ([32000, 32000], 1),
403
+ "pattern_2": ([], 1),
404
+ "pattern_3": ([1550], 2),
405
+ },
406
+ ],
407
+)
408
+@pytest.mark.parametrize(
409
+ ("prompt", "expected"),
410
+ [
411
+ (
412
+ [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
413
+ [
414
+ _PlaceholderInfo(
415
+ modality="pattern_1",
416
+ start_idx=6,
417
+ unit=[32000, 32000],
418
+ unit_count=1,
419
+ ),
420
+ ],
421
+ ),
422
+ (
423
+ [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550],
424
+ [
425
+ _PlaceholderInfo(
426
+ modality="pattern_1",
427
+ start_idx=1,
428
+ unit=[32000, 32000],
429
+ unit_count=1,
430
+ ),
431
+ _PlaceholderInfo(
432
+ modality="pattern_1",
433
+ start_idx=5,
434
+ unit=[32000, 32000],
435
+ unit_count=1,
436
+ ),
437
+ _PlaceholderInfo(
438
+ modality="pattern_3",
439
+ start_idx=7,
440
+ unit=[1550],
441
+ unit_count=2,
442
+ ),
443
+ ],
444
+ ),
445
+ (
446
+ [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550],
447
+ [
448
+ _PlaceholderInfo(
449
+ modality="pattern_1",
450
+ start_idx=1,
451
+ unit=[32000, 32000],
452
+ unit_count=2,
453
+ ),
454
+ _PlaceholderInfo(
455
+ modality="pattern_3",
456
+ start_idx=6,
457
+ unit=[1550],
458
+ unit_count=2,
459
+ ),
460
+ ],
461
+ ),
462
+ ]
463
+)
464
+def test_iter_placeholders(
465
+ repl_by_key,
466
+ prompt,
467
+ expected,
468
+):
469
+ # Should not be used since there is nothing to convert to tokens
470
+ mock_tokenizer = cast(AnyTokenizer, object())
471
+
472
+ prompt_repls = [
473
+ PromptReplacement([], *repl).bind(key, mock_tokenizer)
474
+ for key, repl in repl_by_key.items()
475
+ ]
476
+
477
+ result = list(iter_placeholders(prompt_repls, prompt))
478
+
479
+ # Only displayed on error
480
+ print("result:", result)
481
482
# Manually constructed results
370
- assert result_by_mm_count == expected_by_mm_count
483
+ assert result == expected
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+5
-7
@@ -2,19 +2,17 @@ from typing import Optional
2
3
import torch
4
5
-from vllm.inputs import INPUT_REGISTRY
5
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
7
- dummy_data_for_llava,
8
- get_max_llava_image_tokens,
9
- input_processor_for_llava)
6
+ create_metadata_for_llava,
7
+ dummy_mm_kwargs_for_llava,
8
+ get_max_llava_image_tokens)
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.multimodal import MULTIMODAL_REGISTRY
11
12
14
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
13
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
16
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
17
-@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
14
+@MULTIMODAL_REGISTRY.register_processor_by_metadata(create_metadata_for_llava,
15
+ dummy_mm_kwargs_for_llava)
16
class MyLlava(LlavaForConditionalGeneration):
17
18
def compute_logits(
vllm/inputs/registry.py
+30
-12
@@ -232,19 +232,35 @@ class InputRegistry:
232
"""
233
# Avoid circular import
234
from vllm.model_executor.model_loader import get_model_architecture
235
-
236
- model_cls, _ = get_model_architecture(model_config)
237
- if is_encoder_data:
238
- dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
235
+ from vllm.multimodal import MultiModalKwargs
236
+ from vllm.multimodal.utils import cached_get_tokenizer
237
+
238
+ if mm_registry.has_processor(model_config):
239
+ tokenizer = cached_get_tokenizer(
240
+ model_config.tokenizer,
241
+ trust_remote_code=model_config.trust_remote_code,
242
+ )
243
+ processor = mm_registry.create_processor(model_config, tokenizer)
244
+
245
+ mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
246
+ mm_max_tokens = mm_registry.get_max_tokens_by_modality(
247
+ model_config)
248
+
249
+ dummy_data = processor.get_dummy_data(seq_len, mm_counts,
250
+ mm_max_tokens)
251
else:
240
- dummy_factory = self._get_dummy_data_factory(model_cls)
241
- mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
242
- mm_processor_kwargs = get_allowed_kwarg_only_overrides(
243
- dummy_factory, overrides=model_config.mm_processor_kwargs)
252
+ model_cls, _ = get_model_architecture(model_config)
253
+ if is_encoder_data:
254
+ dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
255
+ else:
256
+ dummy_factory = self._get_dummy_data_factory(model_cls)
257
+ mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
258
+ mm_processor_kwargs = get_allowed_kwarg_only_overrides(
259
+ dummy_factory, overrides=model_config.mm_processor_kwargs)
260
245
- dummy_data = dummy_factory(InputContext(model_config), seq_len,
246
- _MultiModalCounts(mm_counts),
247
- **mm_processor_kwargs)
261
+ dummy_data = dummy_factory(InputContext(model_config), seq_len,
262
+ _MultiModalCounts(mm_counts),
263
+ **mm_processor_kwargs)
264
265
# Having more tokens is over-conservative but otherwise fine
266
num_tokens = dummy_data.seq_data.prompt_token_ids
@@ -257,7 +273,9 @@ class InputRegistry:
273
raise AssertionError(
274
f"Expected at least {seq_len} dummy tokens for profiling, "
275
f"but found {len(num_tokens)} tokens instead.")
260
- if dummy_data.multi_modal_data is not None:
276
+
277
+ if (dummy_data.multi_modal_data is not None and
278
+ not isinstance(dummy_data.multi_modal_data, MultiModalKwargs)):
279
for k, v in dummy_data.multi_modal_data.items():
280
num_items = len(v) if isinstance(v, list) else 1
281
num_expected = mm_counts[k]
vllm/model_executor/models/llava.py
+88
-131
@@ -1,17 +1,19 @@
1
from functools import cached_property
2
+from types import MethodType
3
from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
4
Tuple, TypedDict, Union)
5
6
import torch
7
import torch.nn as nn
7
-from PIL import Image
8
-from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9
- PretrainedConfig, SiglipVisionConfig)
8
+from PIL.Image import Image
9
+from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
10
+ PixtralVisionConfig, PretrainedConfig,
11
+ ProcessorMixin, SiglipVisionConfig)
12
+from transformers.models.pixtral import PixtralProcessor
13
14
from vllm.attention import AttentionMetadata
15
from vllm.config import VllmConfig
13
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14
- InputContext)
16
+from vllm.inputs import InputContext
17
from vllm.model_executor.layers.activation import get_act_fn
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
RowParallelLinear)
@@ -19,21 +21,20 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
22
-from vllm.multimodal.inputs import NestedTensors
24
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
25
+from vllm.multimodal.processing import (InputProcessingContext,
26
+ ModalityProcessingMetadata,
27
+ MultiModalProcessingMetadata,
28
+ MultiModalProcessor, PromptReplacement)
29
from vllm.sequence import IntermediateTensors
24
-from vllm.utils import is_list_of
30
31
from .clip import (CLIPVisionModel, dummy_image_for_clip,
27
- dummy_seq_data_for_clip, get_max_clip_image_tokens,
28
- input_processor_for_clip)
32
+ get_max_clip_image_tokens)
33
from .interfaces import SupportsMultiModal, SupportsPP
34
from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
31
- dummy_seq_data_for_pixtral_hf,
32
- get_max_pixtral_hf_image_tokens,
33
- input_processor_for_pixtral_hf)
35
+ get_max_pixtral_hf_image_tokens)
36
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
35
- dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
36
- input_processor_for_siglip)
37
+ get_max_siglip_image_tokens)
38
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39
maybe_prefix, merge_multimodal_embeddings)
40
@@ -113,102 +114,86 @@ def get_max_llava_image_tokens(ctx: InputContext):
114
raise ValueError(f"Unexpected select feature strategy: {strategy}")
115
116
116
-def dummy_data_for_llava(ctx: InputContext, seq_len: int,
117
- mm_counts: Mapping[str, int]):
117
+def dummy_mm_kwargs_for_llava(ctx: InputProcessingContext,
118
+ mm_counts: Mapping[str, int]):
119
hf_config = ctx.get_hf_config(LlavaConfig)
120
vision_config = hf_config.vision_config
121
num_images = mm_counts["image"]
122
122
- image_feature_size = get_max_llava_image_tokens(ctx)
123
-
123
if isinstance(vision_config, CLIPVisionConfig):
125
- seq_data, ranges = dummy_seq_data_for_clip(
126
- vision_config,
127
- seq_len,
128
- num_images,
129
- image_token_id=hf_config.image_token_index,
130
- image_feature_size_override=image_feature_size,
131
- )
132
-
133
- mm_data = dummy_image_for_clip(vision_config, num_images)
134
- return DummyData(seq_data, mm_data, ranges)
124
+ data = dummy_image_for_clip(vision_config, num_images)
125
elif isinstance(vision_config, SiglipVisionConfig):
136
- seq_data, ranges = dummy_seq_data_for_siglip(
137
- vision_config,
138
- seq_len,
139
- num_images,
140
- image_token_id=hf_config.image_token_index,
141
- image_feature_size_override=image_feature_size,
142
- )
143
-
144
- mm_data = dummy_image_for_siglip(vision_config, num_images)
145
- return DummyData(seq_data, mm_data, ranges)
126
+ data = dummy_image_for_siglip(vision_config, num_images)
127
elif isinstance(vision_config, PixtralVisionConfig):
147
- seq_data, ranges = dummy_seq_data_for_pixtral_hf(
148
- vision_config,
149
- seq_len,
150
- num_images,
151
- image_token_id=hf_config.image_token_index,
152
- image_feature_size_override=image_feature_size,
153
- )
154
-
155
- mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
156
- return DummyData(seq_data, mm_data, ranges)
128
+ data = dummy_image_for_pixtral_hf(vision_config, num_images)
129
+ else:
130
+ msg = f"Unsupported vision config: {type(vision_config)}"
131
+ raise NotImplementedError(msg)
132
158
- msg = f"Unsupported vision config: {type(vision_config)}"
159
- raise NotImplementedError(msg)
133
+ hf_processor = ctx.get_hf_processor()
134
+ image_processor = hf_processor.image_processor # type: ignore
135
+ hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
136
+ is_pixtral = isinstance(hf_processor, PixtralProcessor)
137
138
+ return MultiModalKwargs(
139
+ **hf_inputs,
140
+ is_pixtral=torch.tensor(is_pixtral),
141
+ )
142
162
-def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
163
- multi_modal_data = inputs.get("multi_modal_data")
164
- if multi_modal_data is None or "image" not in multi_modal_data:
165
- return inputs
143
167
- model_config = ctx.model_config
144
+def create_metadata_for_llava(
145
+ ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
146
hf_config = ctx.get_hf_config(LlavaConfig)
169
- vision_config = hf_config.vision_config
147
+ image_token_id = hf_config.image_token_index
148
+
149
+ def get_repl_count(
150
+ mm_items: list[Image],
151
+ hf_inputs: BatchFeature,
152
+ item_idx: int,
153
+ ) -> int:
154
+ return get_max_llava_image_tokens(ctx)
155
+
156
+ return {
157
+ "image":
158
+ ModalityProcessingMetadata(prompt_repls=[
159
+ PromptReplacement(target=[image_token_id],
160
+ repl_unit=[image_token_id],
161
+ repl_count=get_repl_count),
162
+ ]),
163
+ }
164
171
- image_data = multi_modal_data["image"]
172
- if isinstance(image_data, Image.Image):
173
- image_feature_size = get_max_llava_image_tokens(ctx)
174
- elif is_list_of(image_data, Image.Image):
175
- image_feature_size = [get_max_llava_image_tokens(ctx)
176
- ] * len(image_data)
177
- elif isinstance(image_data, torch.Tensor):
178
- num_images, image_feature_size, hidden_size = image_data.shape
179
- elif is_list_of(image_data, torch.Tensor):
180
- image_feature_size = [item.shape[1] for item in image_data]
181
- else:
182
- raise TypeError(f"Invalid image type: {type(image_data)}")
165
184
- if isinstance(vision_config, CLIPVisionConfig):
185
- return input_processor_for_clip(
186
- model_config,
187
- vision_config,
188
- inputs,
189
- image_token_id=hf_config.image_token_index,
190
- image_feature_size_override=image_feature_size,
191
- )
192
- elif isinstance(vision_config, SiglipVisionConfig):
193
- return input_processor_for_siglip(
194
- model_config,
195
- vision_config,
196
- inputs,
197
- image_token_id=hf_config.image_token_index,
198
- image_feature_size_override=image_feature_size,
199
- )
200
- elif isinstance(vision_config, PixtralVisionConfig):
201
- # We ignore image_feature_size_override since we have non-uniform
202
- # image sizes for Pixtral
203
- return input_processor_for_pixtral_hf(
204
- model_config,
205
- vision_config,
206
- inputs,
207
- image_token_id=hf_config.image_token_index,
208
- )
166
+class LlavaProcessor(MultiModalProcessor):
167
210
- msg = f"Unsupported vision config: {type(vision_config)}"
211
- raise NotImplementedError(msg)
168
+ def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
169
+ if getattr(hf_processor, "__is_patched__", False):
170
+ return # Already patched
171
+
172
+ image_processor = hf_processor.image_processor # type: ignore
173
+ orig_preprocess = image_processor.preprocess
174
+
175
+ def preprocess(__self, *args, **kwargs):
176
+ hf_inputs = orig_preprocess(*args, **kwargs)
177
+ hf_inputs["is_pixtral"] = torch.tensor(True)
178
+ return hf_inputs
179
+
180
+ image_processor.preprocess = MethodType(preprocess, image_processor)
181
+
182
+ hf_processor.__is_patched__ = True # type: ignore
183
+
184
+ def _get_hf_processor(self) -> ProcessorMixin:
185
+ hf_processor = self.ctx.get_hf_processor()
186
+
187
+ if isinstance(hf_processor, PixtralProcessor):
188
+ self._patch_pixtral_processor(hf_processor)
189
+
190
+ return hf_processor
191
+
192
+ def _get_dummy_mm_kwargs(
193
+ self,
194
+ mm_counts: Mapping[str, int],
195
+ ) -> MultiModalKwargs:
196
+ return dummy_mm_kwargs_for_llava(self.ctx, mm_counts)
197
198
199
class LlavaLikeConfig(Protocol):
@@ -291,10 +276,11 @@ def init_vision_tower_for_llava(
276
raise NotImplementedError(msg)
277
278
294
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
279
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
296
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
297
-@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
280
+@MULTIMODAL_REGISTRY.register_processor(lambda ctx: LlavaProcessor(
281
+ ctx=ctx,
282
+ metadata=create_metadata_for_llava(ctx),
283
+))
284
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
285
# BitandBytes specific attributes
286
bitsandbytes_stacked_params_mapping = {
@@ -367,38 +353,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
353
354
return data
355
370
- def _validate_image_sizes(self, images: List[torch.Tensor],
371
- sizes: List[torch.Tensor]) -> List[torch.Tensor]:
372
- if not isinstance(sizes, list):
373
- sizes = [sizes]
374
-
375
- total_images = sum(size.numel() // 2 for size in sizes)
376
- if total_images != len(images):
377
- raise ValueError("Mismatch in number of images. "
378
- f"Expected {total_images}, got {len(images)}")
379
- img_idx = 0
380
- for size in sizes:
381
- # Flatten the size tensor to a list of (height, width) pairs
382
- size = size.view(-1, 2).tolist()
383
- for expected_h, expected_w in size:
384
- if img_idx >= len(images):
385
- raise ValueError("Ran out of images before sizes. "
386
- f"{img_idx} >= {len(images)}")
387
- img = images[img_idx]
388
- if img.shape[-2:] != (expected_h, expected_w):
389
- raise ValueError(
390
- "Image size mismatch. Expected "
391
- f"{(expected_h, expected_w)}, got {img.shape[-2:]}")
392
- if img.shape[-3] != 3:
393
- raise ValueError("Image channel mismatch. Expected 3, "
394
- f"got {img.shape[-3]}")
395
- img_idx += 1
396
- return images
397
-
356
def _parse_and_validate_image_input(
357
self, **kwargs: object) -> Optional[LlavaImageInputs]:
358
pixel_values = kwargs.pop("pixel_values", None)
401
- image_sizes = kwargs.pop("image_sizes", None)
359
+ is_pixtral = kwargs.pop("is_pixtral", torch.tensor([False]))
360
image_embeds = kwargs.pop("image_embeds", None)
361
362
if pixel_values is None and image_embeds is None:
@@ -409,9 +367,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
367
raise ValueError("Incorrect type of pixel values. "
368
f"Got type: {type(pixel_values)}")
369
412
- # Case for models like PixtralHF that have dynamic image sizes
413
- # so we need to produce a list of tensors
414
- if image_sizes is not None:
370
+ assert isinstance(is_pixtral, torch.Tensor)
371
+ if is_pixtral.any():
372
images = pixel_values
373
374
def flatten_to_3d_tensors(item):
@@ -434,7 +391,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
391
392
return LlavaImagePixelInputs(
393
type="pixel_values",
437
- data=self._validate_image_sizes(images, image_sizes),
394
+ data=images,
395
)
396
397
return LlavaImagePixelInputs(
vllm/multimodal/base.py
+36
-15
@@ -226,16 +226,16 @@ class MultiModalPlugin(ABC):
226
"""
227
# Avoid circular import
228
from vllm.model_executor.model_loader import get_model_architecture
229
+ from vllm.model_executor.models import supports_multimodal
230
231
model_cls, _ = get_model_architecture(model_config)
232
232
- if model_cls not in self._input_mappers:
233
+ if not supports_multimodal(model_cls):
234
return 0
235
236
max_mm_tokens = self._max_mm_tokens.get(model_cls)
237
if max_mm_tokens is None:
237
- raise KeyError(f"No maximum number of multi-modal tokens is given "
238
- f"for model class {model_cls.__name__} in {self}.")
238
+ return 0
239
240
if callable(max_mm_tokens):
241
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
@@ -326,26 +326,47 @@ class MultiModalPlaceholderMap:
326
src_ranges = []
327
dest_ranges = []
328
"""
329
- if (not seq_group.multi_modal_data
330
- or not seq_group.multi_modal_placeholders):
331
- return seq_group.multi_modal_data, {}
329
+ seq_mm_data = seq_group.multi_modal_data
330
+ seq_mm_placeholders = seq_group.multi_modal_placeholders
331
+
332
+ if not seq_mm_data or not seq_mm_placeholders:
333
+ return seq_mm_data, {}
334
+
335
+ # For merged processor, we directly use mm_kwargs as mm_data
336
+ if isinstance(seq_mm_data, MultiModalKwargs):
337
+ placeholder_maps = dict[str, MultiModalPlaceholderMap]()
338
+
339
+ for modality, placeholders in seq_mm_placeholders.items():
340
+ placeholder_map = MultiModalPlaceholderMap()
341
+
342
+ if positions:
343
+ placeholder_map.append_items_from_seq_group(
344
+ positions,
345
+ # Dummy, since we don't care about intersecting items
346
+ [None] * len(placeholders),
347
+ placeholders,
348
+ )
349
+
350
+ placeholder_maps[modality] = placeholder_map
351
+
352
+ return seq_mm_data, placeholder_maps
353
333
- mm_data = {**seq_group.multi_modal_data}
334
- placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
354
+ mm_data = {**seq_mm_data}
355
+ placeholder_maps = defaultdict[str, MultiModalPlaceholderMap](
356
MultiModalPlaceholderMap)
357
337
- for (
338
- modality,
339
- placeholders,
340
- ) in seq_group.multi_modal_placeholders.items():
358
+ for modality, placeholders in seq_mm_placeholders.items():
359
mm_items = mm_data.pop(modality)
360
if not isinstance(mm_items, list):
361
mm_items = [mm_items]
362
363
if positions:
346
- intersecting_items = placeholder_maps[
347
- modality].append_items_from_seq_group(
348
- positions, mm_items, placeholders)
364
+ intersecting_items = placeholder_maps[modality] \
365
+ .append_items_from_seq_group(
366
+ positions,
367
+ mm_items,
368
+ placeholders,
369
+ )
370
371
if intersecting_items:
372
mm_data[modality] = intersecting_items
vllm/multimodal/processing.py
+197
-116
@@ -3,14 +3,13 @@ from abc import ABC, abstractmethod
3
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4
from dataclasses import dataclass
5
from functools import lru_cache
6
-from itertools import groupby
6
from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
7
9
-import numpy as np
10
-from transformers import BatchFeature
8
+import torch
9
+from transformers import BatchFeature, ProcessorMixin
10
from typing_extensions import TypeAlias, TypedDict
11
13
-from vllm.inputs import InputProcessingContext
12
+from vllm.inputs import DummyData, InputProcessingContext
13
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
14
from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
15
@@ -256,63 +255,6 @@ def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
255
return multi_data
256
257
259
-class _TokenRun(NamedTuple):
260
- token_id: int
261
-
262
- start_idx: int
263
- length: int
264
-
265
-
266
-def iter_token_runs(token_ids: list[int]) -> Iterable[_TokenRun]:
267
- """
268
- Yield the starting index and length of each run of tokens that are the same.
269
- """
270
- start_idx = 0
271
-
272
- for token_id, it in groupby(token_ids):
273
- length = sum(1 for _ in it)
274
- yield _TokenRun(token_id=token_id, start_idx=start_idx, length=length)
275
-
276
- start_idx += length
277
-
278
-
279
-class _PlaceholderInfo(NamedTuple):
280
- modality: str
281
- offset: int
282
- length: int
283
-
284
- def to_range(self) -> PlaceholderRange:
285
- return PlaceholderRange(offset=self.offset, length=self.length)
286
-
287
-
288
-def iter_placeholders(
289
- prompt_repls: Sequence[_BoundPromptReplacement[Any]],
290
- token_ids: list[int],
291
- *,
292
- min_placeholder_count: int,
293
-) -> Iterable[_PlaceholderInfo]:
294
- """Yield each set of placeholder tokens found in :code:`token_ids`."""
295
- placeholder_ids_by_modality = {
296
- modality: {
297
- token_id
298
- for prompt_repl in repls
299
- for token_id in prompt_repl.repl_unit.token_ids
300
- }
301
- for modality, repls in full_groupby_modality(prompt_repls)
302
- }
303
-
304
- for run_info in iter_token_runs(token_ids):
305
- if run_info.length > min_placeholder_count:
306
- for (modality,
307
- placeholder_ids) in placeholder_ids_by_modality.items():
308
- if run_info.token_id in placeholder_ids:
309
- yield _PlaceholderInfo(
310
- modality=modality,
311
- offset=run_info.start_idx,
312
- length=run_info.length,
313
- )
314
-
315
-
258
class _TokenMatch(NamedTuple):
259
start_idx: int
260
end_idx: int
@@ -353,13 +295,9 @@ class _PromptReplacementMatch(ABC, Generic[_T, _S]):
295
def end_idx(self) -> int:
296
raise NotImplementedError
297
298
+ @property
299
@abstractmethod
357
- def get_repl(
358
- self,
359
- mm_items: list[_T],
360
- hf_inputs: BatchFeature,
361
- item_idx: int,
362
- ) -> _S:
300
+ def repl_unit(self) -> _S:
301
raise NotImplementedError
302
303
def __repr__(self) -> str:
@@ -380,15 +318,9 @@ class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
318
def end_idx(self) -> int:
319
return self.match.end_idx
320
383
- def get_repl(
384
- self,
385
- mm_items: list[_T],
386
- hf_inputs: BatchFeature,
387
- item_idx: int,
388
- ) -> list[int]:
389
- prompt_repl = self.prompt_repl
390
- count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
391
- return prompt_repl.repl_unit.token_ids * count
321
+ @property
322
+ def repl_unit(self) -> list[int]:
323
+ return self.prompt_repl.repl_unit.token_ids
324
325
326
@dataclass(repr=False)
@@ -404,15 +336,26 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
336
def end_idx(self) -> int:
337
return self.match.end()
338
407
- def get_repl(
408
- self,
409
- mm_items: list[_T],
410
- hf_inputs: BatchFeature,
411
- item_idx: int,
412
- ) -> str:
413
- prompt_repl = self.prompt_repl
414
- count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
415
- return prompt_repl.repl_unit.text * count
339
+ @property
340
+ def repl_unit(self) -> str:
341
+ return self.prompt_repl.repl_unit.text
342
+
343
+
344
+class _PlaceholderInfo(NamedTuple):
345
+ modality: str
346
+ start_idx: int
347
+ unit: list[int]
348
+ unit_count: int
349
+
350
+ @property
351
+ def length(self) -> int:
352
+ return len(self.unit) * self.unit_count
353
+
354
+ def to_range(self) -> PlaceholderRange:
355
+ return PlaceholderRange(
356
+ offset=self.start_idx,
357
+ length=self.length,
358
+ )
359
360
361
def find_token_matches(
@@ -447,15 +390,17 @@ def _resolve_matches(
390
Resolve :code:`matches` to ensure that there are no overlapping matches,
391
and sort them such that earlier matches take priority over later ones.
392
"""
450
- num_matches_by_idx = np.zeros(len(prompt), dtype=int)
393
+ seen_matches: list[Optional[_PromptReplacementMatch[_T, _S]]] \
394
+ = [None] * len(prompt)
395
+
396
for match in matches:
452
- num_matches_by_idx[match.start_idx:match.end_idx] += 1
397
+ for idx in range(match.start_idx, match.end_idx):
398
+ if seen_matches[idx] is not None:
399
+ raise ValueError("Found overlapping matches "
400
+ f"({seen_matches[idx]} and {match}) "
401
+ f"at index={idx} of prompt={prompt}")
402
454
- duplicate_matches_idxs, = np.nonzero(num_matches_by_idx > 1)
455
- if len(duplicate_matches_idxs) > 0:
456
- raise ValueError("Unable to find a unique replacement "
457
- f"at indices={duplicate_matches_idxs} "
458
- f"of prompt={prompt}")
403
+ seen_matches[idx] = match
404
405
return sorted(matches, key=lambda x: x.start_idx)
406
@@ -480,9 +425,12 @@ def _replace_matches(
425
426
start_idx = match.start_idx
427
end_idx = match.end_idx
483
- repl_ids = match.get_repl(mm_items, hf_inputs, item_idx)
428
+ repl_unit = match.repl_unit
429
+ repl_info = match.prompt_repl
430
+ repl_count = repl_info.get_count(mm_items, hf_inputs, item_idx)
431
485
- out_seqs.append(prompt[prev_end_idx:start_idx] + repl_ids)
432
+ out_seqs.append(prompt[prev_end_idx:start_idx] +
433
+ repl_unit * repl_count)
434
prev_end_idx = end_idx
435
next_idx_by_modality[modality] += 1
436
@@ -531,7 +479,57 @@ def replace_text_matches(
479
return "".join(texts)
480
481
534
-class MultiModalProcessor:
482
+def _merge_placeholder_matches(
483
+ matches: Iterable[_PromptReplacementTokenMatch],
484
+) -> Iterable[_PromptReplacementTokenMatch]:
485
+ current_match = None
486
+
487
+ for match in sorted(matches, key=lambda x: x.start_idx):
488
+ if current_match is None:
489
+ current_match = match
490
+ elif (current_match.prompt_repl == match.prompt_repl
491
+ and current_match.end_idx == match.start_idx):
492
+ current_match = _PromptReplacementTokenMatch(
493
+ current_match.prompt_repl,
494
+ match=_TokenMatch(current_match.start_idx, match.end_idx),
495
+ )
496
+ else:
497
+ yield current_match
498
+ current_match = match
499
+
500
+ if current_match is not None:
501
+ yield current_match
502
+
503
+
504
+def iter_placeholders(
505
+ prompt_repls: Sequence[_BoundPromptReplacement[Any]],
506
+ prompt: list[int],
507
+ *,
508
+ min_unit_count: int = 1,
509
+) -> Iterable[_PlaceholderInfo]:
510
+ """Yield each set of placeholder tokens found in :code:`token_ids`."""
511
+ if min_unit_count <= 0:
512
+ raise ValueError("`min_unit_count` must be a positive integer")
513
+
514
+ matches = (_PromptReplacementTokenMatch(prompt_repl, match)
515
+ for prompt_repl in prompt_repls
516
+ if len(repl_unit := prompt_repl.repl_unit.token_ids) > 0
517
+ for match in iter_token_matches(prompt, repl_unit))
518
+
519
+ for match in _merge_placeholder_matches(matches):
520
+ unit = match.repl_unit
521
+ placeholder = _PlaceholderInfo(
522
+ modality=match.modality,
523
+ start_idx=match.start_idx,
524
+ unit=unit,
525
+ unit_count=(match.end_idx - match.start_idx) // len(unit),
526
+ )
527
+
528
+ if placeholder.unit_count >= min_unit_count:
529
+ yield placeholder
530
+
531
+
532
+class MultiModalProcessor(ABC):
533
"""
534
Helper class to process multi-modal inputs to be used in vLLM.
535
"""
@@ -546,6 +544,12 @@ class MultiModalProcessor:
544
self.ctx = ctx
545
self.metadata = metadata
546
547
+ def _get_hf_processor(self) -> ProcessorMixin:
548
+ return self.ctx.get_hf_processor()
549
+
550
+ def _get_tokenizer(self) -> AnyTokenizer:
551
+ return self.ctx.tokenizer
552
+
553
def __call__(
554
self,
555
prompt: str,
@@ -562,13 +566,13 @@ class MultiModalProcessor:
566
# To avoid false positives from multi-input when detecting
567
# whether placeholder tokens have been inserted, in case
568
# the target sequence is a subset of the replacement tokens
565
- min_placeholder_count: int = 16,
569
+ min_unit_count: int = 16,
570
) -> list[_PlaceholderInfo]:
571
return list(
572
iter_placeholders(
573
all_prompt_repls,
574
new_token_ids,
571
- min_placeholder_count=min_placeholder_count,
575
+ min_unit_count=min_unit_count,
576
))
577
578
def _apply_hf_processor(
@@ -577,19 +581,49 @@ class MultiModalProcessor:
581
mm_data: MultiModalDataDict,
582
mm_processor_kwargs: Mapping[str, object],
583
) -> BatchFeature:
580
- hf_processor = self.ctx.get_hf_processor()
584
+ hf_processor = self._get_hf_processor()
585
+
586
+ processor_data = dict[str, Any]()
587
+ passthrough_data = dict[str, Any]()
588
+ for k, v in mm_data.items():
589
+ # TODO: Make a separate modality for embedding inputs
590
+ # to avoid confusion
591
+ if k in ("image", "video", "audio"):
592
+ if isinstance(v, torch.Tensor) and v.ndim == 3:
593
+ # Pass through embedding inputs (single)
594
+ passthrough_data[f"{k}_embeds"] = [v]
595
+ elif is_list_of(v, torch.Tensor) and v[0].ndim == 2:
596
+ # Pass through embedding inputs (multi)
597
+ passthrough_data[f"{k}_embeds"] = v
598
+ else:
599
+ # Map keys to plural form, e.g.: image -> images
600
+ processor_data[f"{k}s"] = v
601
+ else:
602
+ processor_data[k] = v
603
+
604
+ try:
605
+ hf_inputs = hf_processor(
606
+ text=prompt, # type: ignore
607
+ **processor_data,
608
+ **mm_processor_kwargs,
609
+ return_tensors="pt",
610
+ )
611
+ except Exception as exc:
612
+ data = dict(text=prompt, **processor_data)
613
582
- return hf_processor(
583
- text=prompt, # type: ignore
584
- **mm_data,
585
- **mm_processor_kwargs,
586
- )
614
+ raise RuntimeError(
615
+ f"Failed to apply {type(hf_processor).__name__} "
616
+ f"on data={data} with kwargs={mm_processor_kwargs}") from exc
617
+
618
+ hf_inputs.update(passthrough_data)
619
+
620
+ return hf_inputs
621
622
def _bind_prompt_replacements(
623
self,
624
mm_data: MultiModalDataDict,
625
) -> list[_BoundPromptReplacement[Any]]:
592
- tokenizer = self.ctx.tokenizer
626
+ tokenizer = self._get_tokenizer()
627
628
return [
629
prompt_repl.bind(modality, tokenizer)
@@ -604,7 +638,7 @@ class MultiModalProcessor:
638
token_ids: list[int],
639
prompt_repls: Sequence[_BoundPromptReplacement[Any]],
640
) -> tuple[list[int], str, list[_PlaceholderInfo]]:
607
- tokenizer = self.ctx.tokenizer
641
+ tokenizer = self._get_tokenizer()
642
643
mm_items = to_multi_format(mm_data)
644
token_matches = find_token_matches(token_ids, prompt_repls)
@@ -620,7 +654,7 @@ class MultiModalProcessor:
654
# of the search text in the prompt, we instead perform string
655
# replacement on the decoded token IDs, then encode them back.
656
if all(
623
- len(matches) >= len(mm_data[modality])
657
+ len(matches) >= len(mm_items[modality])
658
for modality, matches in full_groupby_modality(token_matches)
659
): # yapf: disable
660
token_ids = replace_token_matches(
@@ -648,15 +682,6 @@ class MultiModalProcessor:
682
683
placeholders = self._find_placeholders(matched_repls, token_ids)
684
651
- # Sanity check
652
- assert len(placeholders) == len(matched_repls), dict(
653
- # Log this information for easier debugging
654
- text=text,
655
- token_ids=token_ids,
656
- placeholders=placeholders,
657
- matched_repls=matched_repls,
658
- )
659
-
685
return token_ids, text, placeholders
686
687
def apply(
@@ -678,7 +703,7 @@ class MultiModalProcessor:
703
3. Extract information about the placeholder tokens from the
704
processed token IDs.
705
"""
681
- tokenizer = self.ctx.tokenizer
706
+ tokenizer = self._get_tokenizer()
707
708
hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
709
mm_processor_kwargs)
@@ -717,3 +742,59 @@ class MultiModalProcessor:
742
mm_kwargs=mm_kwargs,
743
mm_placeholders=mm_placeholders,
744
)
745
+
746
+ @abstractmethod
747
+ def _get_dummy_mm_kwargs(
748
+ self,
749
+ mm_counts: Mapping[str, int],
750
+ ) -> MultiModalKwargs:
751
+ """
752
+ Build the input that corresponds to `mm_max_tokens` in
753
+ :meth:`get_dummy_data`.
754
+ """
755
+ raise NotImplementedError
756
+
757
+ def get_dummy_data(
758
+ self,
759
+ seq_len: int,
760
+ mm_counts: Mapping[str, int],
761
+ mm_max_tokens: Mapping[str, int],
762
+ ) -> DummyData:
763
+ # Avoid circular import
764
+ from vllm.sequence import SequenceData
765
+
766
+ tokenizer = self._get_tokenizer()
767
+
768
+ mm_placeholders = dict[str, _PlaceholderInfo]()
769
+ offset = 0
770
+
771
+ for modality, max_tokens in mm_max_tokens.items():
772
+ if max_tokens == 0:
773
+ continue
774
+
775
+ metadata = self.metadata[modality]
776
+ repl = metadata.prompt_repls[0].bind(modality, tokenizer)
777
+ repl_token_ids = repl.repl_unit.token_ids
778
+
779
+ placeholders = _PlaceholderInfo(
780
+ modality=modality,
781
+ start_idx=offset,
782
+ unit=repl_token_ids,
783
+ unit_count=max_tokens // len(repl_token_ids),
784
+ )
785
+
786
+ mm_placeholders[modality] = placeholders
787
+ offset += placeholders.length
788
+
789
+ prompt_token_ids = flatten_2d_lists(
790
+ [p.unit * p.unit_count for p in mm_placeholders.values()])
791
+ prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
792
+
793
+ return DummyData(
794
+ seq_data=SequenceData.from_seqs(prompt_token_ids),
795
+ multi_modal_data=self._get_dummy_mm_kwargs(mm_counts),
796
+ multi_modal_placeholders={
797
+ modality: [p.to_range()]
798
+ for modality, p in mm_placeholders.items()
799
+ },
800
+ )
vllm/multimodal/registry.py
+60
-7
@@ -15,7 +15,7 @@ from .audio import AudioPlugin
15
from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16
from .image import ImagePlugin
17
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18
-from .processing import MultiModalProcessor
18
+from .processing import MultiModalProcessingMetadata, MultiModalProcessor
19
from .video import VideoPlugin
20
21
if TYPE_CHECKING:
@@ -200,9 +200,12 @@ class MultiModalRegistry:
200
"""
201
return self.register_max_multimodal_tokens("image", max_mm_tokens)
202
203
- def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
203
+ def get_max_tokens_by_modality(
204
+ self,
205
+ model_config: "ModelConfig",
206
+ ) -> Mapping[str, int]:
207
"""
205
- Get the maximum number of multi-modal tokens
208
+ Get the maximum number of tokens from each modality
209
for profiling the memory usage of a model.
210
211
See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
@@ -212,9 +215,23 @@ class MultiModalRegistry:
215
"""
216
limits_per_plugin = self._limits_by_model[model_config]
217
215
- return sum((limits_per_plugin[key] *
216
- plugin.get_max_multimodal_tokens(model_config))
217
- for key, plugin in self._plugins.items())
218
+ return {
219
+ key: (limits_per_plugin[key] *
220
+ plugin.get_max_multimodal_tokens(model_config))
221
+ for key, plugin in self._plugins.items()
222
+ }
223
+
224
+ def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
225
+ """
226
+ Get the maximum number of multi-modal tokens
227
+ for profiling the memory usage of a model.
228
+
229
+ See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
230
+
231
+ Note:
232
+ This should be called after :meth:`init_mm_limits_per_prompt`.
233
+ """
234
+ return sum(self.get_max_tokens_by_modality(model_config).values())
235
236
def init_mm_limits_per_prompt(
237
self,
@@ -270,7 +287,8 @@ class MultiModalRegistry:
287
factory: MultiModalProcessorFactory,
288
):
289
"""
273
- Register a multi-modal processor to a model class.
290
+ Register a multi-modal processor to a model class. The processor
291
+ is constructed lazily, hence a factory method should be passed.
292
293
When the model receives multi-modal data, the provided function is
294
invoked to transform the data into a dictionary of model inputs.
@@ -293,6 +311,41 @@ class MultiModalRegistry:
311
312
return wrapper
313
314
+ def register_processor_by_metadata(
315
+ self,
316
+ metadata_factory: Callable[[InputProcessingContext],
317
+ MultiModalProcessingMetadata],
318
+ get_dummy_mm_kwargs: Callable[
319
+ [InputProcessingContext, Mapping[str, int]], MultiModalKwargs],
320
+ ):
321
+ """
322
+ Convenience method to register a multi-modal processor to a model class
323
+ according to a function that constructs its metadata.
324
+
325
+ When the model receives multi-modal data, the provided function is
326
+ invoked to transform the data into a dictionary of model inputs.
327
+
328
+ See also:
329
+ - :ref:`input_processing_pipeline`
330
+ - :ref:`enabling_multimodal_inputs`
331
+ """
332
+
333
+ class ConcreteMultiModalProcessor(MultiModalProcessor):
334
+
335
+ def _get_dummy_mm_kwargs(
336
+ self,
337
+ mm_counts: Mapping[str, int],
338
+ ) -> MultiModalKwargs:
339
+ return get_dummy_mm_kwargs(self.ctx, mm_counts)
340
+
341
+ def factory(ctx: InputProcessingContext):
342
+ return ConcreteMultiModalProcessor(
343
+ ctx=ctx,
344
+ metadata=metadata_factory(ctx),
345
+ )
346
+
347
+ return self.register_processor(factory)
348
+
349
def has_processor(self, model_config: "ModelConfig") -> bool:
350
"""
351
Test whether a multi-modal processor is defined for a specific model.
vllm/v1/engine/mm_input_mapper.py
+1
@@ -12,6 +12,7 @@ class MMInputMapper:
12
model_config: ModelConfig,
13
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
14
):
15
+ self.model_config = model_config
16
self.mm_registry = mm_registry
17
self.multi_modal_input_mapper = mm_registry.create_input_mapper(
18
model_config)
vllm/v1/engine/processor.py
+11
-5
@@ -7,7 +7,8 @@ from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
7
from vllm.inputs.parse import is_encoder_decoder_inputs
8
from vllm.inputs.preprocess import InputPreprocessor
9
from vllm.lora.request import LoRARequest
10
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
11
+ MultiModalRegistry)
12
from vllm.pooling_params import PoolingParams
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sampling_params import SamplingParams
@@ -101,10 +102,15 @@ class Processor:
102
self.generation_config_fields, eos_token_id)
103
104
# Preprocess multi-modal data
104
- mm_inputs = self.mm_input_mapper.process_inputs(
105
- decoder_inputs.multi_modal_data,
106
- decoder_inputs.mm_processor_kwargs) if len(
107
- decoder_inputs.multi_modal_data) > 0 else None
105
+ if len(decoder_inputs.multi_modal_data) == 0:
106
+ mm_inputs = None
107
+ elif isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
108
+ mm_inputs = [decoder_inputs.multi_modal_data]
109
+ else:
110
+ mm_inputs = self.mm_input_mapper.process_inputs(
111
+ decoder_inputs.multi_modal_data,
112
+ decoder_inputs.mm_processor_kwargs,
113
+ )
114
115
# Make Request for Detokenizer.
116
detokenizer_request = DetokenizerRequest(