A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[VLM] Fully dynamic prompt replacement in merged input processor (#11199)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 15, 2024 at 01:52 UTC
93abf23a648051fe6dc053ba0b74499d119920bf
12 files changed
+569
-510
examples/offline_inference_vision_language.py
+1
-4
@@ -97,9 +97,6 @@ def run_phi3v(question: str, modality: str):
97
# max_model_len (128k) for this model may cause OOM.
98
# You may lower either to run this example on lower-end GPUs.
99
100
- # In this example, we override max_num_seqs to 5 while
101
- # keeping the original context length of 128k.
102
-
100
# num_crops is an override kwarg to the multimodal image processor;
101
# For some models, e.g., Phi-3.5-vision-instruct, it is recommended
102
# to use 16 for single frame scenarios, and 4 for multi-frame.
@@ -113,7 +110,7 @@ def run_phi3v(question: str, modality: str):
110
# https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
111
# https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
112
llm = LLM(
116
- model="microsoft/Phi-3-vision-128k-instruct",
113
+ model="microsoft/Phi-3.5-vision-instruct",
114
trust_remote_code=True,
115
max_model_len=4096,
116
max_num_seqs=2,
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2
-2
@@ -16,8 +16,8 @@ models = ["microsoft/Phi-3.5-vision-instruct"]
16
# Wrap lazy imports to avoid initializing CUDA during test collection
17
@pytest.fixture()
18
def processor_for_phi3v():
19
- from vllm.model_executor.models.phi3v import Phi3VProcessor
20
- return Phi3VProcessor
19
+ from vllm.model_executor.models.phi3v import Phi3VMultiModalProcessor
20
+ return Phi3VMultiModalProcessor
21
22
23
@pytest.fixture()
tests/multimodal/test_processing.py
+54
-51
@@ -1,11 +1,11 @@
1
from typing import cast
2
3
import pytest
4
-from transformers import BatchFeature
4
6
-from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
7
- find_text_matches, find_token_matches,
8
- iter_placeholders, iter_token_matches,
5
+from vllm.multimodal.processing import (MultiModalDataItems, PromptReplacement,
6
+ _PlaceholderInfo, find_text_matches,
7
+ find_token_matches, iter_placeholders,
8
+ iter_token_matches,
9
replace_text_matches,
10
replace_token_matches)
11
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -16,7 +16,7 @@ from vllm.utils import full_groupby
16
@pytest.mark.parametrize(
17
("token_ids", "match_ids", "expected"),
18
[
19
- ([], [], [{ "start_idx": 0, "end_idx": 0 }]),
19
+ ([], [], []),
20
([], [32000], []),
21
(
22
[32000, 32000, 32000],
@@ -83,7 +83,7 @@ def test_iter_token_matches(token_ids, match_ids, expected):
83
"pattern_2": [32000],
84
},
85
{
86
- "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
86
+ "pattern_1": [],
87
"pattern_2": [],
88
}
89
),
@@ -136,7 +136,7 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
136
mock_tokenizer = cast(AnyTokenizer, object())
137
138
prompt_repls = [
139
- PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
139
+ PromptReplacement(key, target, []).bind(mock_tokenizer)
140
for key, target in target_by_key.items()
141
]
142
result = find_token_matches(prompt, prompt_repls)
@@ -243,7 +243,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
243
mock_tokenizer = cast(AnyTokenizer, object())
244
245
prompt_repls = [
246
- PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
246
+ PromptReplacement(key, target, []).bind(mock_tokenizer)
247
for key, target in target_by_key.items()
248
]
249
result = find_text_matches(prompt, prompt_repls)
@@ -276,12 +276,12 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
276
"pattern_3": "!",
277
},
278
{
279
- # Test whether target is confused with repl_unit
280
- "pattern_1": ("<image><image>", 1),
281
- # Test empty repl_unit
282
- "pattern_2": ("", 1),
283
- # Test multiple repl_count
284
- "pattern_3": ("?", 2),
279
+ # Test whether target is confused with replacement
280
+ "pattern_1": "<image><image>",
281
+ # Test empty replacement
282
+ "pattern_2": "",
283
+ # Test dynamic replacement (beyond the form of `unit * count`)
284
+ "pattern_3": "?!?",
285
},
286
),
287
]
@@ -290,8 +290,8 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
290
("mm_count", "expected"),
291
[
292
(0, "Image:<image>Image:<image><image>!"),
293
- (1, "<image><image>Image:<image><image>??"),
294
- (2, "<image><image><image><image><image>??"),
293
+ (1, "<image><image>Image:<image><image>?!?"),
294
+ (2, "<image><image><image><image><image>?!?"),
295
]
296
)
297
# yapf: enable
@@ -306,7 +306,7 @@ def test_find_replace_text(
306
mock_tokenizer = cast(AnyTokenizer, object())
307
308
prompt_repls = [
309
- PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
309
+ PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
310
for key, target in target_by_key.items()
311
]
312
matches = find_text_matches(prompt, prompt_repls)
@@ -314,9 +314,8 @@ def test_find_replace_text(
314
result = replace_text_matches(
315
prompt,
316
matches,
317
- {key: list(range(mm_count))
318
- for key in repl_by_key},
319
- BatchFeature(),
317
+ MultiModalDataItems({key: [None] * mm_count
318
+ for key in repl_by_key}),
319
)
320
321
# Only displayed on error
@@ -343,12 +342,12 @@ def test_find_replace_text(
342
"pattern_3": [918],
343
},
344
{
346
- # Test whether target is confused with repl_unit
347
- "pattern_1": ([32000, 32000], 1),
348
- # Test empty repl_unit
349
- "pattern_2": ([], 1),
350
- # Test multiple repl_count
351
- "pattern_3": ([1550], 2),
345
+ # Test whether target is confused with replacement
346
+ "pattern_1": [32000, 32000],
347
+ # Test empty replacement
348
+ "pattern_2": [],
349
+ # Test dynamic replacement (beyond the form of `unit * count`)
350
+ "pattern_3": [1550, 918, 1550],
351
},
352
),
353
]
@@ -357,8 +356,8 @@ def test_find_replace_text(
356
("mm_count", "expected"),
357
[
358
(0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
360
- (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550]),
361
- (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550]),
359
+ (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550]),
360
+ (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550]),
361
]
362
)
363
# yapf: enable
@@ -373,7 +372,7 @@ def test_find_replace_tokens(
372
mock_tokenizer = cast(AnyTokenizer, object())
373
374
prompt_repls = [
376
- PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
375
+ PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
376
for key, target in target_by_key.items()
377
]
378
matches = find_token_matches(prompt, prompt_repls)
@@ -381,9 +380,8 @@ def test_find_replace_tokens(
380
result = replace_token_matches(
381
prompt,
382
matches,
384
- {key: list(range(mm_count))
385
- for key in repl_by_key},
386
- BatchFeature(),
383
+ MultiModalDataItems({key: [None] * mm_count
384
+ for key in repl_by_key}),
385
)
386
387
# Only displayed on error
@@ -399,9 +397,9 @@ def test_find_replace_tokens(
397
"repl_by_key",
398
[
399
{
402
- "pattern_1": ([32000, 32000], 1),
403
- "pattern_2": ([], 1),
404
- "pattern_3": ([1550], 2),
400
+ "pattern_1": [32000, 32000],
401
+ "pattern_2": [],
402
+ "pattern_3": [1550, 918, 1550],
403
},
404
],
405
)
@@ -414,48 +412,47 @@ def test_find_replace_tokens(
412
_PlaceholderInfo(
413
modality="pattern_1",
414
start_idx=6,
417
- unit=[32000, 32000],
418
- unit_count=1,
415
+ replacement=[32000, 32000],
416
),
417
],
418
),
419
(
423
- [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550],
420
+ [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
421
[
422
_PlaceholderInfo(
423
modality="pattern_1",
424
start_idx=1,
428
- unit=[32000, 32000],
429
- unit_count=1,
425
+ replacement=[32000, 32000],
426
),
427
_PlaceholderInfo(
428
modality="pattern_1",
429
start_idx=5,
434
- unit=[32000, 32000],
435
- unit_count=1,
430
+ replacement=[32000, 32000],
431
),
432
_PlaceholderInfo(
433
modality="pattern_3",
434
start_idx=7,
440
- unit=[1550],
441
- unit_count=2,
435
+ replacement=[1550, 918, 1550],
436
),
437
],
438
),
439
(
446
- [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550],
440
+ [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
441
[
442
_PlaceholderInfo(
443
modality="pattern_1",
444
start_idx=1,
451
- unit=[32000, 32000],
452
- unit_count=2,
445
+ replacement=[32000, 32000],
446
+ ),
447
+ _PlaceholderInfo(
448
+ modality="pattern_1",
449
+ start_idx=3,
450
+ replacement=[32000, 32000],
451
),
452
_PlaceholderInfo(
453
modality="pattern_3",
454
start_idx=6,
457
- unit=[1550],
458
- unit_count=2,
455
+ replacement=[1550, 918, 1550],
456
),
457
],
458
),
@@ -470,11 +467,17 @@ def test_iter_placeholders(
467
mock_tokenizer = cast(AnyTokenizer, object())
468
469
prompt_repls = [
473
- PromptReplacement([], *repl).bind(key, mock_tokenizer)
470
+ PromptReplacement(key, [], repl).bind(mock_tokenizer)
471
for key, repl in repl_by_key.items()
472
]
473
477
- result = list(iter_placeholders(prompt_repls, prompt))
474
+ result = list(
475
+ iter_placeholders(
476
+ prompt_repls,
477
+ prompt,
478
+ # Effectively match all occurrences in the prompt
479
+ MultiModalDataItems({key: [None] * 3 for key in repl_by_key}),
480
+ ))
481
482
# Only displayed on error
483
print("result:", result)
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+2
-2
@@ -3,14 +3,14 @@ from typing import Optional
3
import torch
4
5
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6
- LlavaProcessor,
6
+ LlavaMultiModalProcessor,
7
get_max_llava_image_tokens)
8
from vllm.model_executor.sampling_metadata import SamplingMetadata
9
from vllm.multimodal import MULTIMODAL_REGISTRY
10
11
12
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
13
-@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
13
+@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
14
class MyLlava(LlavaForConditionalGeneration):
15
16
def compute_logits(
vllm/inputs/registry.py
+61
-10
@@ -2,7 +2,7 @@ import functools
2
from collections import UserDict
3
from dataclasses import dataclass
4
from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5
- Optional, Protocol, Type, cast)
5
+ Optional, Protocol, Type)
6
7
from torch import nn
8
from transformers import PretrainedConfig, ProcessorMixin
@@ -47,7 +47,6 @@ class InputContext:
47
Raises:
48
TypeError: If the model is not of the specified type.
49
"""
50
-
50
hf_config = self.model_config.hf_config
51
if not isinstance(hf_config, hf_config_type):
52
raise TypeError("Invalid type of HuggingFace config. "
@@ -60,21 +59,70 @@ class InputContext:
59
"""
60
Get the HuggingFace image processor configuration of the model.
61
"""
63
-
62
return self.model_config.hf_image_processor_config
63
64
+ def get_mm_config(self):
65
+ """
66
+ Get the multimodal config of the model.
67
+
68
+ Raises:
69
+ RuntimeError: If the model is not a multimodal model.
70
+ """
71
+ mm_config = self.model_config.multimodal_config
72
+ if mm_config is None:
73
+ raise RuntimeError("Not a multimodal model")
74
+
75
+ return mm_config
76
+
77
+ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
78
+ base_kwargs = self.model_config.mm_processor_kwargs
79
+ if base_kwargs is None:
80
+ base_kwargs = {}
81
+
82
+ merged_kwargs = {**base_kwargs, **kwargs}
83
+
84
+ return cached_get_processor(
85
+ self.model_config.model,
86
+ trust_remote_code=self.model_config.trust_remote_code,
87
+ **merged_kwargs,
88
+ )
89
+
90
91
@dataclass(frozen=True)
92
class InputProcessingContext(InputContext):
93
tokenizer: AnyTokenizer
94
"""The tokenizer used to tokenize the inputs."""
95
72
- def get_hf_processor(self, **kwargs) -> ProcessorMixin:
96
+ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
97
+ base_kwargs = self.model_config.mm_processor_kwargs
98
+ if base_kwargs is None:
99
+ base_kwargs = {}
100
+
101
+ merged_kwargs = {**base_kwargs, **kwargs}
102
+
103
return cached_get_processor(
74
- self.model_config.tokenizer,
104
+ self.model_config.model,
105
tokenizer=self.tokenizer, # Override the tokenizer with ours
106
trust_remote_code=self.model_config.trust_remote_code,
77
- **kwargs)
107
+ **merged_kwargs,
108
+ )
109
+
110
+ def resolve_hf_processor_call_kwargs(
111
+ self,
112
+ hf_processor: ProcessorMixin,
113
+ inference_kwargs: Mapping[str, object],
114
+ ) -> Mapping[str, object]:
115
+ assert callable(hf_processor)
116
+
117
+ base_kwargs = self.model_config.mm_processor_kwargs
118
+ if base_kwargs is None:
119
+ base_kwargs = {}
120
+
121
+ return resolve_mm_processor_kwargs(
122
+ base_kwargs,
123
+ inference_kwargs,
124
+ hf_processor,
125
+ )
126
127
128
N = TypeVar("N", bound=Type[nn.Module])
@@ -171,7 +219,8 @@ class InputRegistry:
219
"""
220
221
def wrapper(model_cls: N) -> N:
174
- if model_cls in self._dummy_factories_by_model_type:
222
+ if self._dummy_factories_by_model_type.contains(model_cls,
223
+ strict=True):
224
logger.warning(
225
"Model class %s already has dummy data "
226
"registered to %s. It is overwritten by the new one.",
@@ -195,7 +244,8 @@ class InputRegistry:
244
"""
245
246
def wrapper(model_cls: N) -> N:
198
- if model_cls in self._dummy_encoder_factories_by_model_type:
247
+ if self._dummy_encoder_factories_by_model_type.contains(
248
+ model_cls, strict=True):
249
logger.warning(
250
"Model class %s already has dummy encoder data "
251
"registered to %s. It is overwritten by the new one.",
@@ -305,7 +355,8 @@ class InputRegistry:
355
"""
356
357
def wrapper(model_cls: N) -> N:
308
- if model_cls in self._input_processors_by_model_type:
358
+ if self._input_processors_by_model_type.contains(model_cls,
359
+ strict=True):
360
logger.warning(
361
"Model class %s already has input processor "
362
"registered to %s. It is overwritten by the new one.",
@@ -357,7 +408,7 @@ class InputRegistry:
408
# If it's empty, it'll fall back to the default kwarg values
409
mm_processor_kwargs = resolve_mm_processor_kwargs(
410
model_config.mm_processor_kwargs,
360
- cast(Dict[str, Any], inputs.get("mm_processor_kwargs")),
411
+ inputs.get("mm_processor_kwargs", {}), # type: ignore
412
processor,
413
)
414
vllm/model_executor/models/llava.py
+72
-72
@@ -5,10 +5,10 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
5
6
import torch
7
import torch.nn as nn
8
-from PIL.Image import Image
8
from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
9
PixtralVisionConfig, PretrainedConfig,
10
ProcessorMixin, SiglipVisionConfig)
11
+from transformers.models.llava import LlavaProcessor
12
from transformers.models.pixtral import PixtralProcessor
13
14
from vllm.attention import AttentionMetadata
@@ -21,11 +21,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
24
+from vllm.multimodal.inputs import NestedTensors
25
from vllm.multimodal.processing import (BaseMultiModalProcessor,
26
- InputProcessingContext,
27
- ModalityProcessingMetadata,
28
- MultiModalProcessingMetadata,
26
+ MultiModalDataItems, ProcessorInputs,
27
PromptReplacement)
28
from vllm.sequence import IntermediateTensors
29
@@ -33,7 +31,8 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
31
get_max_clip_image_tokens)
32
from .interfaces import SupportsMultiModal, SupportsPP
33
from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
36
- get_max_pixtral_hf_image_tokens)
34
+ get_max_pixtral_hf_image_tokens,
35
+ get_pixtral_hf_image_feature_size)
36
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
37
get_max_siglip_image_tokens)
38
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
@@ -115,62 +114,7 @@ def get_max_llava_image_tokens(ctx: InputContext):
114
raise ValueError(f"Unexpected select feature strategy: {strategy}")
115
116
118
-def dummy_mm_kwargs_for_llava(ctx: InputProcessingContext,
119
- mm_counts: Mapping[str, int]):
120
- hf_config = ctx.get_hf_config(LlavaConfig)
121
- vision_config = hf_config.vision_config
122
- num_images = mm_counts["image"]
123
-
124
- if isinstance(vision_config, CLIPVisionConfig):
125
- data = dummy_image_for_clip(vision_config, num_images)
126
- elif isinstance(vision_config, SiglipVisionConfig):
127
- data = dummy_image_for_siglip(vision_config, num_images)
128
- elif isinstance(vision_config, PixtralVisionConfig):
129
- data = dummy_image_for_pixtral_hf(vision_config, num_images)
130
- else:
131
- msg = f"Unsupported vision config: {type(vision_config)}"
132
- raise NotImplementedError(msg)
133
-
134
- hf_processor = ctx.get_hf_processor()
135
- image_processor = hf_processor.image_processor # type: ignore
136
- hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
137
- is_pixtral = isinstance(hf_processor, PixtralProcessor)
138
-
139
- return MultiModalKwargs(
140
- **hf_inputs,
141
- is_pixtral=torch.tensor(is_pixtral),
142
- )
143
-
144
-
145
-def create_metadata_for_llava(
146
- ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
147
- hf_config = ctx.get_hf_config(LlavaConfig)
148
- image_token_id = hf_config.image_token_index
149
-
150
- def get_repl_count(
151
- mm_items: list[Image],
152
- hf_inputs: BatchFeature,
153
- item_idx: int,
154
- ) -> int:
155
- return get_max_llava_image_tokens(ctx)
156
-
157
- return {
158
- "image":
159
- ModalityProcessingMetadata(prompt_repls=[
160
- PromptReplacement(target=[image_token_id],
161
- repl_unit=[image_token_id],
162
- repl_count=get_repl_count),
163
- ]),
164
- }
165
-
166
-
167
-class LlavaProcessor(BaseMultiModalProcessor):
168
-
169
- def __init__(self, ctx: InputProcessingContext) -> None:
170
- super().__init__(
171
- ctx=ctx,
172
- metadata=create_metadata_for_llava(ctx),
173
- )
117
+class LlavaMultiModalProcessor(BaseMultiModalProcessor):
118
119
def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
120
if getattr(hf_processor, "__is_patched__", False):
@@ -188,18 +132,72 @@ class LlavaProcessor(BaseMultiModalProcessor):
132
133
hf_processor.__is_patched__ = True # type: ignore
134
191
- def _get_hf_processor(self) -> ProcessorMixin:
135
+ def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136
hf_processor = self.ctx.get_hf_processor()
137
+ assert isinstance(hf_processor, (LlavaProcessor, PixtralProcessor))
138
139
if isinstance(hf_processor, PixtralProcessor):
140
self._patch_pixtral_processor(hf_processor)
141
142
return hf_processor
143
199
- def _get_dummy_mm_kwargs(
144
+ def _get_prompt_replacements(
145
+ self,
146
+ mm_items: MultiModalDataItems,
147
+ hf_inputs: BatchFeature,
148
+ mm_processor_kwargs: Mapping[str, object],
149
+ ) -> list[PromptReplacement]:
150
+ hf_config = self.ctx.get_hf_config(LlavaConfig)
151
+ image_token_id = hf_config.image_token_index
152
+
153
+ processor = self._get_hf_processor()
154
+ if isinstance(processor, PixtralProcessor):
155
+ image_token = processor.image_token
156
+ image_break_token = processor.image_break_token
157
+ image_end_token = processor.image_end_token
158
+
159
+ vision_config = hf_config.vision_config
160
+ assert isinstance(vision_config, PixtralVisionConfig)
161
+
162
+ def get_replacement_pixtral(item_idx: int):
163
+ image_size = mm_items.get_image_size(item_idx)
164
+ (
165
+ num_width_tokens,
166
+ num_height_tokens,
167
+ ) = get_pixtral_hf_image_feature_size(
168
+ vision_config,
169
+ image_width=image_size.width,
170
+ image_height=image_size.height,
171
+ )
172
+
173
+ tokens = ([image_token] * num_width_tokens +
174
+ [image_break_token]) * num_height_tokens
175
+ tokens[-1] = image_end_token
176
+
177
+ return "".join(tokens)
178
+
179
+ return [
180
+ PromptReplacement(
181
+ modality="image",
182
+ target=[image_token_id],
183
+ replacement=get_replacement_pixtral,
184
+ ),
185
+ ]
186
+
187
+ max_image_tokens = get_max_llava_image_tokens(self.ctx)
188
+
189
+ return [
190
+ PromptReplacement(
191
+ modality="image",
192
+ target=[image_token_id],
193
+ replacement=[image_token_id] * max_image_tokens,
194
+ )
195
+ ]
196
+
197
+ def _get_dummy_mm_inputs(
198
self,
199
mm_counts: Mapping[str, int],
202
- ) -> MultiModalKwargs:
200
+ ) -> ProcessorInputs:
201
hf_config = self.ctx.get_hf_config(LlavaConfig)
202
vision_config = hf_config.vision_config
203
num_images = mm_counts["image"]
@@ -215,11 +213,13 @@ class LlavaProcessor(BaseMultiModalProcessor):
213
raise NotImplementedError(msg)
214
215
hf_processor = self._get_hf_processor()
218
- image_processor = hf_processor.image_processor # type: ignore
219
- hf_inputs = image_processor.preprocess(data['image'],
220
- return_tensors="pt")
216
+ image_token = hf_processor.image_token
217
222
- return MultiModalKwargs(**hf_inputs)
218
+ return ProcessorInputs(
219
+ prompt_text=image_token * num_images,
220
+ mm_data=data,
221
+ mm_processor_kwargs={},
222
+ )
223
224
225
class LlavaLikeConfig(Protocol):
@@ -303,7 +303,7 @@ def init_vision_tower_for_llava(
303
304
305
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
306
-@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
306
+@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
307
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
308
# BitandBytes specific attributes
309
bitsandbytes_stacked_params_mapping = {
@@ -584,7 +584,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
584
return loader.load_weights(weights)
585
586
587
-class MantisProcessor(LlavaProcessor):
587
+class MantisMultiModalProcessor(LlavaMultiModalProcessor):
588
589
def _get_hf_processor(self) -> ProcessorMixin:
590
try:
@@ -604,6 +604,6 @@ class MantisProcessor(LlavaProcessor):
604
# To use this model, please use
605
# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
606
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
607
-@MULTIMODAL_REGISTRY.register_processor(MantisProcessor)
607
+@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
608
class MantisForConditionalGeneration(LlavaForConditionalGeneration):
609
pass
vllm/model_executor/models/phi3v.py
+57
-61
@@ -32,13 +32,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.models.clip import CLIPVisionModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
-from vllm.multimodal.image import cached_get_image_processor
36
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
35
+from vllm.multimodal.inputs import NestedTensors
36
from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
- InputProcessingContext,
39
- ModalityProcessingMetadata,
37
MultiModalDataDict,
41
- MultiModalProcessingMetadata,
38
+ MultiModalDataItems, ProcessorInputs,
39
PromptReplacement)
40
from vllm.sequence import IntermediateTensors
41
from vllm.utils import is_list_of
@@ -305,64 +302,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
302
return image_features_hd_newline
303
304
308
-def get_max_phi3v_image_tokens(ctx: InputContext,
309
- *,
310
- num_crops: Optional[int] = None):
311
- mm_processor_kwargs = {}
312
- if num_crops is not None:
313
- mm_processor_kwargs["num_crops"] = num_crops
305
+def get_max_phi3v_image_tokens(ctx: InputContext) -> int:
306
+ processor = ctx.get_hf_processor()
307
+ image_processor = processor.image_processor # type: ignore
308
315
- model_config = ctx.model_config
316
- image_processor = cached_get_image_processor(
317
- model_config.model,
318
- trust_remote_code=model_config.trust_remote_code,
319
- **mm_processor_kwargs,
320
- )
321
-
322
- num_tokens = image_processor.calc_num_image_tokens_from_image_size(
309
+ return image_processor.calc_num_image_tokens_from_image_size(
310
width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
311
height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
312
)
326
- return num_tokens
327
-
328
-
329
-def dummy_mm_kwargs_for_phi3v(ctx: InputProcessingContext,
330
- mm_counts: Mapping[str, int]):
331
- num_images = mm_counts["image"]
332
-
333
- data = dummy_image_for_clip(
334
- CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335
- num_images,
336
- image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
337
- image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
338
- )
339
-
340
- hf_processor = ctx.get_hf_processor()
341
- image_processor = hf_processor.image_processor # type: ignore
342
- hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
343
-
344
- return MultiModalKwargs(**hf_inputs)
345
-
346
-
347
-def create_metadata_for_phi3v(
348
- ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
349
- return {
350
- "image":
351
- ModalityProcessingMetadata(prompt_repls=[
352
- PromptReplacement(target=[_IMAGE_TOKEN_ID],
353
- repl_unit=[_IMAGE_TOKEN_ID],
354
- repl_count=get_max_phi3v_image_tokens(ctx)),
355
- ]),
356
- }
357
-
313
359
-class Phi3VProcessor(BaseMultiModalProcessor):
314
361
- def __init__(self, ctx: InputProcessingContext) -> None:
362
- super().__init__(
363
- ctx=ctx,
364
- metadata=create_metadata_for_phi3v(ctx),
365
- )
315
+class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
316
317
def _get_hf_processor(
318
self,
@@ -389,15 +339,61 @@ class Phi3VProcessor(BaseMultiModalProcessor):
339
processed_outputs['input_ids'] = token_ids
340
return processed_outputs
341
392
- def _get_dummy_mm_kwargs(
342
+ def _get_prompt_replacements(
343
+ self,
344
+ mm_items: MultiModalDataItems,
345
+ hf_inputs: BatchFeature,
346
+ mm_processor_kwargs: Mapping[str, object],
347
+ ) -> list[PromptReplacement]:
348
+ hf_processor = self._get_hf_processor()
349
+ image_tokens: list[str] = hf_processor.img_tokens # type: ignore
350
+ image_processor = hf_processor.image_processor # type: ignore
351
+
352
+ mm_config = self.ctx.get_mm_config()
353
+ max_images = mm_config.limit_per_prompt.get("image", 1)
354
+
355
+ def get_replacement_phi3v(item_idx: int):
356
+ image_size = mm_items.get_image_size(item_idx)
357
+ num_tokens = image_processor.calc_num_image_tokens_from_image_size(
358
+ width=image_size.width,
359
+ height=image_size.height,
360
+ )
361
+
362
+ return [_IMAGE_TOKEN_ID] * num_tokens
363
+
364
+ return [
365
+ PromptReplacement(
366
+ modality="image",
367
+ target=image_token,
368
+ replacement=get_replacement_phi3v,
369
+ ) for image_token in image_tokens[:max_images]
370
+ ]
371
+
372
+ def _get_dummy_mm_inputs(
373
self,
374
mm_counts: Mapping[str, int],
395
- ) -> MultiModalKwargs:
396
- return dummy_mm_kwargs_for_phi3v(self.ctx, mm_counts)
375
+ ) -> ProcessorInputs:
376
+ num_images = mm_counts["image"]
377
+
378
+ data = dummy_image_for_clip(
379
+ CLIP_VIT_LARGE_PATCH14_336_CONFIG,
380
+ num_images,
381
+ image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
382
+ image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
383
+ )
384
+
385
+ hf_processor = self._get_hf_processor()
386
+ image_tokens: list[str] = hf_processor.img_tokens # type: ignore
387
+
388
+ return ProcessorInputs(
389
+ prompt_text="".join(image_tokens[:num_images]),
390
+ mm_data=data,
391
+ mm_processor_kwargs={},
392
+ )
393
394
395
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
400
-@MULTIMODAL_REGISTRY.register_processor(Phi3VProcessor)
396
+@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
397
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
398
399
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/pixtral.py
+1
-1
@@ -72,7 +72,7 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
72
mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
73
image_token_id = mm_encoder.special_ids.img
74
75
- mm_config = ctx.model_config.multimodal_config
75
+ mm_config = ctx.get_mm_config()
76
num_images = mm_config.limit_per_prompt.get("image", 1)
77
78
# dummy size
vllm/multimodal/base.py
+2
-2
@@ -99,7 +99,7 @@ class MultiModalPlugin(ABC):
99
"""
100
101
def wrapper(model_cls: N) -> N:
102
- if model_cls in self._input_mappers:
102
+ if self._input_mappers.contains(model_cls, strict=True):
103
logger.warning(
104
"Model class %s already has an input mapper "
105
"registered to %s. It is overwritten by the new one.",
@@ -194,7 +194,7 @@ class MultiModalPlugin(ABC):
194
"""
195
196
def wrapper(model_cls: N) -> N:
197
- if model_cls in self._max_mm_tokens:
197
+ if self._max_mm_tokens.contains(model_cls, strict=True):
198
logger.warning(
199
"Model class %s already calculates maximum number of "
200
"tokens in %s. It is overwritten by the new one.",
vllm/multimodal/processing.py
+306
-300
@@ -1,116 +1,59 @@
1
import re
2
from abc import ABC, abstractmethod
3
+from collections import UserDict
4
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4
-from dataclasses import dataclass
5
+from dataclasses import dataclass, field
6
from functools import lru_cache
6
-from typing import (Any, Dict, Generic, NamedTuple, Optional, Protocol,
7
- TypeVar, Union, cast)
7
+from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
9
+import numpy as np
10
import torch
11
+from PIL.Image import Image
12
from transformers import BatchFeature, ProcessorMixin
11
-from typing_extensions import TypeAlias, TypedDict
13
+from typing_extensions import assert_never
14
15
from vllm.inputs import DummyData, InputProcessingContext
16
+from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
15
-from vllm.utils import (flatten_2d_lists, full_groupby, is_list_of,
16
- resolve_mm_processor_kwargs)
18
+from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
19
20
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
21
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
22
VideoItem)
23
24
+logger = init_logger(__name__)
25
23
-def bind_prompt_sequence(
24
- seq: Union[str, list[int]],
25
- tokenizer: AnyTokenizer,
26
-) -> "_BoundPromptSequence":
27
- """
28
- Bind a text or token sequence to a tokenizer so that it can be
29
- lazily converted into the other format on demand.
30
- """
31
- return _BoundPromptSequence(
32
- tokenizer=tokenizer,
33
- _text=seq if isinstance(seq, str) else None,
34
- _token_ids=seq if isinstance(seq, list) else None,
35
- )
36
-
37
-
38
-_T = TypeVar("_T")
26
_S = TypeVar("_S", str, list[int])
27
+_PromptSeq = Union[str, list[int]]
28
29
30
@dataclass
43
-class PromptReplacement(Generic[_S, _T]):
44
- target: _S
45
- """The text or token sequence to find and replace."""
31
+class PromptReplacement:
32
+ modality: str
33
+ """The modality for which the replacement is made"""
34
47
- repl_unit: _S
48
- """
49
- The unit making up the replacement text or token sequence.
50
-
51
- See :code:`repl_count` for more details.
52
- """
35
+ target: _PromptSeq
36
+ """The text or token sequence to find and replace."""
37
54
- repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
38
+ replacement: Union[Callable[[int], _PromptSeq],
39
+ _PromptSeq] = field(repr=False)
40
"""
56
- Given the original multi-modal items for this modality, HF-processed data,
57
- and index of the processed item, output the number of repetitions of
58
- :code:`repl_unit` to build up the replacement text or token sequence.
41
+ Given the index of the processed item within :attr:`modality`, output the
42
+ replacement text or token sequence.
43
60
- For convenience, you can pass in an integer if the number of repetitions is
61
- a constant.
44
+ For convenience, you can pass in the replacement instead of a function
45
+ if it does not depend on the input.
46
"""
47
64
- def __repr__(self) -> str:
65
- return (f"{type(self).__name__}(target={self.target!r}, "
66
- f"repl_unit={self.repl_unit!r})")
67
-
68
- def bind(
69
- self,
70
- modality: str,
71
- tokenizer: AnyTokenizer,
72
- ) -> "_BoundPromptReplacement[_T]":
48
+ def bind(self, tokenizer: AnyTokenizer) -> "_BoundPromptReplacement":
49
return _BoundPromptReplacement(
74
- modality=modality,
75
- target=bind_prompt_sequence(self.target, tokenizer),
76
- repl_unit=bind_prompt_sequence(self.repl_unit, tokenizer),
77
- repl_count=self.repl_count,
50
+ tokenizer=tokenizer,
51
+ modality=self.modality,
52
+ _target=self.target,
53
+ _replacement=self.replacement,
54
)
55
56
81
-@dataclass
82
-class ModalityProcessingMetadata(Generic[_T]):
83
- prompt_repls: Sequence[Union[PromptReplacement[str, _T],
84
- PromptReplacement[list[int], _T]]]
85
- """
86
- Defines each text or token sequence to replace in the HF-processed prompt.
87
-
88
- This is skipped if the HF-processed prompt is found to already contain
89
- the replacement prompts.
90
- """
91
-
92
-
93
-class MultiModalProcessingMetadataBuiltins(TypedDict, total=False):
94
- """Type annotations for modality types predefined by vLLM."""
95
-
96
- image: ModalityProcessingMetadata[ImageItem]
97
- video: ModalityProcessingMetadata[VideoItem]
98
- audio: ModalityProcessingMetadata[AudioItem]
99
-
100
-
101
-MultiModalProcessingMetadata: TypeAlias = \
102
- Mapping[str, ModalityProcessingMetadata[Any]]
103
-"""
104
-A dictionary containing an entry for each modality type to process.
105
-
106
-Note:
107
- This dictionary also accepts modality keys defined outside
108
- :class:`MultiModalProcessingMetadataBuiltins` as long as a customized plugin
109
- is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
110
- Read more on that :ref:`here <adding_multimodal_plugin>`.
111
-"""
112
-
113
-
57
def _encode(
58
tokenizer: AnyTokenizer,
59
text: str,
@@ -185,7 +128,8 @@ def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
128
129
@dataclass
130
class _BoundPromptSequence:
188
- tokenizer: AnyTokenizer
131
+ tokenizer: AnyTokenizer = field(repr=False)
132
+
133
_text: Optional[str]
134
_token_ids: Optional[list[int]]
135
@@ -210,38 +154,92 @@ class _BoundPromptSequence:
154
155
return self._token_ids
156
213
- def __repr__(self) -> str:
214
- return (f"{type(self).__name__}(_text={self._text!r}, "
215
- f"_token_ids={self._token_ids!r})")
216
-
157
158
@dataclass
219
-class _BoundPromptReplacement(Generic[_T]):
159
+class _BoundPromptReplacement:
160
+ tokenizer: AnyTokenizer = field(repr=False)
161
modality: str
221
- target: _BoundPromptSequence
222
- repl_unit: _BoundPromptSequence
223
- repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
162
225
- def get_count(
226
- self,
227
- mm_items: list[_T],
228
- hf_inputs: BatchFeature,
229
- item_idx: int,
230
- ) -> int:
231
- repl_count = self.repl_count
232
- if isinstance(repl_count, int):
233
- return repl_count
163
+ _target: _PromptSeq
164
+ _replacement: Union[Callable[[int], _PromptSeq],
165
+ _PromptSeq] = field(repr=False)
166
235
- return repl_count(mm_items, hf_inputs, item_idx)
167
+ def __post_init__(self) -> None:
168
+ self._replacement_cache = dict[int, _BoundPromptSequence]()
169
+
170
+ @property
171
+ def target(self) -> _BoundPromptSequence:
172
+ target = self._target
173
174
+ return _BoundPromptSequence(
175
+ tokenizer=self.tokenizer,
176
+ _text=target if isinstance(target, str) else None,
177
+ _token_ids=target if isinstance(target, list) else None,
178
+ )
179
238
-def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
180
+ def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
181
+ replacement = self._replacement
182
+ if callable(replacement):
183
+ cache_key = item_idx
184
+ if cache_key in self._replacement_cache:
185
+ return self._replacement_cache[cache_key]
186
+
187
+ replacement = replacement(item_idx)
188
+ else:
189
+ cache_key = None
190
+
191
+ bound_replacement = _BoundPromptSequence(
192
+ tokenizer=self.tokenizer,
193
+ _text=replacement if isinstance(replacement, str) else None,
194
+ _token_ids=replacement if isinstance(replacement, list) else None,
195
+ )
196
+
197
+ if cache_key is not None:
198
+ self._replacement_cache[cache_key] = bound_replacement
199
+
200
+ return bound_replacement
201
+
202
+
203
+class ImageSize(NamedTuple):
204
+ width: int
205
+ height: int
206
+
207
+
208
+class MultiModalDataItems(UserDict[str, list[Any]]):
209
"""
240
- Convert a :class:`MultiModalDataDict` containing single data items
241
- to a :class:`MultiModalMultiDataDict` containing multiple data items
242
- per entry.
210
+ As :class:`MultiModalDataDict`, but normalized such that each entry
211
+ corresponds to a list.
212
"""
244
- multi_data = dict[str, list[Any]]()
213
+
214
+ @property
215
+ def image(self) -> list[ImageItem]:
216
+ return self["image"]
217
+
218
+ @property
219
+ def video(self) -> list[VideoItem]:
220
+ return self["video"]
221
+
222
+ @property
223
+ def audio(self) -> list[AudioItem]:
224
+ return self["audio"]
225
+
226
+ def get_image_size(self, item_idx: int) -> ImageSize:
227
+ image = self.image[item_idx]
228
+
229
+ if isinstance(image, Image):
230
+ return ImageSize(*image.size)
231
+ if isinstance(image, (np.ndarray, torch.Tensor)):
232
+ _, h, w = image.shape
233
+ return ImageSize(w, h)
234
+
235
+ assert_never(image)
236
+
237
+
238
+def to_multi_format(data: MultiModalDataDict) -> MultiModalDataItems:
239
+ """
240
+ Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
241
+ """
242
+ multi_data = MultiModalDataItems()
243
244
for k, v in data.items():
245
# yapf: disable
@@ -266,22 +264,33 @@ def iter_token_matches(
264
token_ids: list[int],
265
match_ids: list[int],
266
) -> Iterable[_TokenMatch]:
269
- """Yield each occurrence of :code:`match_ids` in :code:`token_ids`."""
267
+ """
268
+ Yield each occurrence of :code:`match_ids` in :code:`token_ids`.
269
+
270
+ Note that empty matches are ignored.
271
+ """
272
+ prompt_len = len(token_ids)
273
match_len = len(match_ids)
274
272
- last_end_idx = 0
273
- for start_idx in range(len(token_ids) - match_len + 1):
274
- if start_idx < last_end_idx:
275
- continue # Exclude overlapping matches
275
+ if match_len == 0:
276
+ return
277
278
+ start_idx = 0
279
+ while start_idx < prompt_len - match_len + 1:
280
end_idx = start_idx + match_len
281
+
282
if token_ids[start_idx:end_idx] == match_ids:
283
yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
280
- last_end_idx = end_idx
284
+
285
+ # Exclude overlapping matches
286
+ start_idx = end_idx
287
+ else:
288
+ start_idx += 1
289
290
283
-class _PromptReplacementMatch(ABC, Generic[_T, _S]):
284
- prompt_repl: _BoundPromptReplacement[_T]
291
+@dataclass(repr=False)
292
+class _PromptReplacementMatch(ABC):
293
+ prompt_repl: _BoundPromptReplacement
294
295
@property
296
def modality(self) -> str:
@@ -297,19 +306,13 @@ class _PromptReplacementMatch(ABC, Generic[_T, _S]):
306
def end_idx(self) -> int:
307
raise NotImplementedError
308
300
- @property
301
- @abstractmethod
302
- def repl_unit(self) -> _S:
303
- raise NotImplementedError
304
-
309
def __repr__(self) -> str:
310
return (f"{type(self).__name__}(modality={self.modality!r}, "
311
f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
312
313
314
@dataclass(repr=False)
311
-class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
312
- prompt_repl: _BoundPromptReplacement[_T]
315
+class _PromptReplacementTokenMatch(_PromptReplacementMatch):
316
match: _TokenMatch
317
318
@property
@@ -320,14 +323,9 @@ class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
323
def end_idx(self) -> int:
324
return self.match.end_idx
325
323
- @property
324
- def repl_unit(self) -> list[int]:
325
- return self.prompt_repl.repl_unit.token_ids
326
-
326
327
@dataclass(repr=False)
329
-class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
330
- prompt_repl: _BoundPromptReplacement[_T]
328
+class _PromptReplacementTextMatch(_PromptReplacementMatch):
329
match: re.Match[str]
330
331
@property
@@ -338,20 +336,15 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
336
def end_idx(self) -> int:
337
return self.match.end()
338
341
- @property
342
- def repl_unit(self) -> str:
343
- return self.prompt_repl.repl_unit.text
344
-
339
340
class _PlaceholderInfo(NamedTuple):
341
modality: str
342
start_idx: int
349
- unit: list[int]
350
- unit_count: int
343
+ replacement: list[int]
344
345
@property
346
def length(self) -> int:
354
- return len(self.unit) * self.unit_count
347
+ return len(self.replacement)
348
349
def to_range(self) -> PlaceholderRange:
350
return PlaceholderRange(
@@ -362,8 +355,8 @@ class _PlaceholderInfo(NamedTuple):
355
356
def find_token_matches(
357
prompt: list[int],
365
- prompt_repls: Sequence[_BoundPromptReplacement[_T]],
366
-) -> list[_PromptReplacementTokenMatch[_T]]:
358
+ prompt_repls: Sequence[_BoundPromptReplacement],
359
+) -> list[_PromptReplacementTokenMatch]:
360
"""Return each target of :code:`prompt_repls` found in :code:`prompt`."""
361
return [
362
_PromptReplacementTokenMatch(prompt_repl, match)
@@ -374,8 +367,8 @@ def find_token_matches(
367
368
def find_text_matches(
369
prompt: str,
377
- prompt_repls: Sequence[_BoundPromptReplacement[_T]],
378
-) -> list[_PromptReplacementTextMatch[_T]]:
370
+ prompt_repls: Sequence[_BoundPromptReplacement],
371
+) -> list[_PromptReplacementTextMatch]:
372
"""Return each target of :code:`prompt_repls` found in :code:`prompt`."""
373
return [
374
_PromptReplacementTextMatch(prompt_repl, match)
@@ -385,15 +378,15 @@ def find_text_matches(
378
379
380
def _resolve_matches(
388
- prompt: _S,
389
- matches: Sequence[_PromptReplacementMatch[_T, _S]],
390
-) -> list[_PromptReplacementMatch[_T, _S]]:
381
+ prompt: _PromptSeq,
382
+ matches: Sequence[_PromptReplacementMatch],
383
+) -> list[_PromptReplacementMatch]:
384
"""
385
Resolve :code:`matches` to ensure that there are no overlapping matches,
386
and sort them such that earlier matches take priority over later ones.
387
"""
395
- seen_matches: list[Optional[_PromptReplacementMatch[_T, _S]]] \
396
- = [None] * len(prompt)
388
+ seen_matches: list[Optional[_PromptReplacementMatch]] = [None
389
+ ] * len(prompt)
390
391
for match in matches:
392
for idx in range(match.start_idx, match.end_idx):
@@ -409,30 +402,34 @@ def _resolve_matches(
402
403
def _replace_matches(
404
prompt: _S,
412
- matches: Sequence[_PromptReplacementMatch[_T, _S]],
413
- mm_items_by_modality: Mapping[str, list[_T]],
414
- hf_inputs: BatchFeature,
405
+ matches: Sequence[_PromptReplacementMatch],
406
+ mm_items: MultiModalDataItems,
407
) -> list[_S]:
408
out_seqs = list[_S]()
409
prev_end_idx = 0
418
- next_idx_by_modality = {modality: 0 for modality in mm_items_by_modality}
410
+ next_idx_by_modality = {modality: 0 for modality in mm_items}
411
412
for match in _resolve_matches(prompt, matches):
413
modality = match.modality
422
- mm_items = mm_items_by_modality[modality]
414
+ modal_items = mm_items[modality]
415
416
item_idx = next_idx_by_modality[modality]
425
- if item_idx >= len(mm_items):
417
+ if item_idx >= len(modal_items):
418
continue
419
420
start_idx = match.start_idx
421
end_idx = match.end_idx
430
- repl_unit = match.repl_unit
422
+
423
repl_info = match.prompt_repl
432
- repl_count = repl_info.get_count(mm_items, hf_inputs, item_idx)
424
+ replacement = repl_info.get_replacement(item_idx)
425
+
426
+ if isinstance(prompt, str):
427
+ repl_seq = replacement.text
428
+ out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
429
+ else:
430
+ repl_seq = replacement.token_ids
431
+ out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
432
434
- out_seqs.append(prompt[prev_end_idx:start_idx] +
435
- repl_unit * repl_count)
433
prev_end_idx = end_idx
434
next_idx_by_modality[modality] += 1
435
@@ -443,92 +440,104 @@ def _replace_matches(
440
441
def replace_token_matches(
442
prompt: list[int],
446
- matches: Sequence[_PromptReplacementMatch[_T, list[int]]],
447
- mm_items_by_modality: Mapping[str, list[_T]],
448
- hf_inputs: BatchFeature,
443
+ matches: Sequence[_PromptReplacementTokenMatch],
444
+ mm_items: MultiModalDataItems,
445
) -> list[int]:
446
"""Apply :code:`prompt_repls` to :code:`prompt`."""
447
if not matches:
448
return prompt
449
454
- token_id_seqs = _replace_matches(
455
- prompt,
456
- matches,
457
- mm_items_by_modality,
458
- hf_inputs,
459
- )
450
+ token_id_seqs = _replace_matches(prompt, matches, mm_items)
451
452
return flatten_2d_lists(token_id_seqs)
453
454
455
def replace_text_matches(
456
prompt: str,
466
- matches: Sequence[_PromptReplacementMatch[_T, str]],
467
- mm_items_by_modality: Mapping[str, list[_T]],
468
- hf_inputs: BatchFeature,
457
+ matches: Sequence[_PromptReplacementTextMatch],
458
+ mm_items: MultiModalDataItems,
459
) -> str:
460
"""Apply :code:`prompt_repls` to :code:`prompt`."""
461
if not matches:
462
return prompt
463
474
- texts = _replace_matches(
475
- prompt,
476
- matches,
477
- mm_items_by_modality,
478
- hf_inputs,
479
- )
464
+ texts = _replace_matches(prompt, matches, mm_items)
465
466
return "".join(texts)
467
468
484
-def _merge_placeholder_matches(
485
- matches: Iterable[_PromptReplacementTokenMatch],
486
-) -> Iterable[_PromptReplacementTokenMatch]:
487
- current_match = None
469
+def _iter_modality_placeholders(
470
+ prompt: list[int],
471
+ modality: str,
472
+ modality_repls: Sequence[_BoundPromptReplacement],
473
+ modal_items: list[Any],
474
+) -> Iterable[_PlaceholderInfo]:
475
+ if len(modal_items) == 0:
476
+ return
477
489
- for match in sorted(matches, key=lambda x: x.start_idx):
490
- if current_match is None:
491
- current_match = match
492
- elif (current_match.prompt_repl == match.prompt_repl
493
- and current_match.end_idx == match.start_idx):
494
- current_match = _PromptReplacementTokenMatch(
495
- current_match.prompt_repl,
496
- match=_TokenMatch(current_match.start_idx, match.end_idx),
497
- )
498
- else:
499
- yield current_match
500
- current_match = match
478
+ prompt_len = len(prompt)
479
+ item_index = 0
480
+
481
+ start_idx = 0
482
+ while start_idx < prompt_len:
483
+ found = False
484
+
485
+ for repl_info in modality_repls:
486
+ replacement = repl_info.get_replacement(item_index)
487
+ repl_tokens = replacement.token_ids
488
+ repl_len = len(repl_tokens)
489
+ end_idx = start_idx + repl_len
490
+
491
+ if repl_len == 0 or end_idx > prompt_len:
492
+ continue
493
502
- if current_match is not None:
503
- yield current_match
494
+ if prompt[start_idx:end_idx] == repl_tokens:
495
+ yield _PlaceholderInfo(
496
+ modality=modality,
497
+ start_idx=start_idx,
498
+ replacement=repl_tokens,
499
+ )
500
+
501
+ item_index += 1
502
+ if item_index >= len(modal_items):
503
+ return
504
+
505
+ # Exclude overlapping matches
506
+ start_idx = end_idx
507
+ found = True
508
+ break
509
+
510
+ if not found:
511
+ start_idx += 1
512
513
514
def iter_placeholders(
507
- prompt_repls: Sequence[_BoundPromptReplacement[Any]],
515
+ prompt_repls: Sequence[_BoundPromptReplacement],
516
prompt: list[int],
509
- *,
510
- min_unit_count: int = 1,
517
+ mm_items: MultiModalDataItems,
518
) -> Iterable[_PlaceholderInfo]:
512
- """Yield each set of placeholder tokens found in :code:`token_ids`."""
513
- if min_unit_count <= 0:
514
- raise ValueError("`min_unit_count` must be a positive integer")
515
-
516
- matches = (_PromptReplacementTokenMatch(prompt_repl, match)
517
- for prompt_repl in prompt_repls
518
- if len(repl_unit := prompt_repl.repl_unit.token_ids) > 0
519
- for match in iter_token_matches(prompt, repl_unit))
520
-
521
- for match in _merge_placeholder_matches(matches):
522
- unit = match.repl_unit
523
- placeholder = _PlaceholderInfo(
524
- modality=match.modality,
525
- start_idx=match.start_idx,
526
- unit=unit,
527
- unit_count=(match.end_idx - match.start_idx) // len(unit),
528
- )
519
+ """
520
+ Yield each set of placeholder tokens found in :code:`prompt`.
521
+
522
+ Note that empty matches are ignored.
523
+ """
524
+ repls_by_modality = dict(full_groupby_modality(prompt_repls))
525
+
526
+ for modality, modal_items in mm_items.items():
527
+ if modality in repls_by_modality:
528
+ yield from _iter_modality_placeholders(
529
+ prompt,
530
+ modality,
531
+ repls_by_modality[modality],
532
+ modal_items,
533
+ )
534
+
535
530
- if placeholder.unit_count >= min_unit_count:
531
- yield placeholder
536
+class ProcessorInputs(NamedTuple):
537
+ """Keyword arguments to :meth:`BaseMultiModalProcessor`"""
538
+ prompt_text: str
539
+ mm_data: MultiModalDataDict
540
+ mm_processor_kwargs: Mapping[str, object]
541
542
543
class BaseMultiModalProcessor(ABC):
@@ -536,52 +545,55 @@ class BaseMultiModalProcessor(ABC):
545
Abstract base class to process multi-modal inputs to be used in vLLM.
546
"""
547
539
- def __init__(
540
- self,
541
- ctx: InputProcessingContext,
542
- metadata: MultiModalProcessingMetadata,
543
- ) -> None:
548
+ def __init__(self, ctx: InputProcessingContext) -> None:
549
super().__init__()
550
551
self.ctx = ctx
547
- self.metadata = metadata
548
- self.init_mm_processor_kwargs = (ctx.model_config.mm_processor_kwargs
549
- or {})
552
551
- def _get_hf_processor(
553
+ def __call__(
554
self,
553
- **mm_processor_kwargs: Mapping[str, object],
554
- ) -> ProcessorMixin:
555
- # by default, we won't pass any kwargs to the processor initialization
555
+ prompt: str,
556
+ mm_data: MultiModalDataDict,
557
+ mm_processor_kwargs: Mapping[str, object],
558
+ ) -> MultiModalInputsV2:
559
+ return self.apply(prompt, mm_data, mm_processor_kwargs)
560
+
561
+ def _get_hf_processor(self) -> ProcessorMixin:
562
+ """
563
+ Subclasses can add keyword arguments to this method to accept
564
+ additional kwargs from model config or user inputs.
565
+ """
566
return self.ctx.get_hf_processor()
567
568
def _get_tokenizer(self) -> AnyTokenizer:
569
return self.ctx.tokenizer
570
561
- def __call__(
571
+ @abstractmethod
572
+ def _get_prompt_replacements(
573
self,
563
- prompt: str,
564
- mm_data: MultiModalDataDict,
574
+ mm_items: MultiModalDataItems,
575
+ hf_inputs: BatchFeature,
576
mm_processor_kwargs: Mapping[str, object],
566
- ) -> MultiModalInputsV2:
567
- return self.apply(prompt, mm_data, mm_processor_kwargs)
577
+ ) -> list[PromptReplacement]:
578
+ """
579
+ Given the original multi-modal items for this modality
580
+ and HF-processed data, output the replacements to perform.
581
+
582
+ Note:
583
+ Even when the HF processor already performs replacement for us,
584
+ we still use this replacement information to determine
585
+ the placeholder token positions for each multi-modal item.
586
+ """
587
+ raise NotImplementedError
588
589
def _find_placeholders(
590
self,
571
- all_prompt_repls: Sequence[_BoundPromptReplacement[Any]],
591
+ all_prompt_repls: Sequence[_BoundPromptReplacement],
592
new_token_ids: list[int],
573
- *,
574
- # To avoid false positives from multi-input when detecting
575
- # whether placeholder tokens have been inserted, in case
576
- # the target sequence is a subset of the replacement tokens
577
- min_unit_count: int = 16,
593
+ mm_items: MultiModalDataItems,
594
) -> list[_PlaceholderInfo]:
595
return list(
580
- iter_placeholders(
581
- all_prompt_repls,
582
- new_token_ids,
583
- min_unit_count=min_unit_count,
584
- ))
596
+ iter_placeholders(all_prompt_repls, new_token_ids, mm_items))
597
598
def _apply_hf_processor(
599
self,
@@ -589,13 +601,7 @@ class BaseMultiModalProcessor(ABC):
601
mm_data: MultiModalDataDict,
602
mm_processor_kwargs: Mapping[str, object],
603
) -> BatchFeature:
592
- # some mm_processor_kwargs may be used in processor initialization
593
- # instead of processor call
594
- processor_init_kwargs = {
595
- **self.init_mm_processor_kwargs,
596
- **mm_processor_kwargs,
597
- }
598
- hf_processor = self._get_hf_processor(**processor_init_kwargs)
604
+ hf_processor = self._get_hf_processor(**mm_processor_kwargs)
605
606
processor_data = dict[str, Any]()
607
passthrough_data = dict[str, Any]()
@@ -615,11 +621,10 @@ class BaseMultiModalProcessor(ABC):
621
else:
622
processor_data[k] = v
623
618
- # filter mm_processor_kwargs used in processor call
619
- mm_processor_kwargs = resolve_mm_processor_kwargs(
620
- self.init_mm_processor_kwargs,
621
- cast(Dict[str, Any], mm_processor_kwargs),
624
+ assert callable(hf_processor)
625
+ mm_processor_kwargs = self.ctx.resolve_hf_processor_call_kwargs(
626
hf_processor,
627
+ mm_processor_kwargs,
628
)
629
630
try:
@@ -642,26 +647,21 @@ class BaseMultiModalProcessor(ABC):
647
648
def _bind_prompt_replacements(
649
self,
645
- mm_data: MultiModalDataDict,
646
- ) -> list[_BoundPromptReplacement[Any]]:
650
+ prompt_repls: list[PromptReplacement],
651
+ ) -> list[_BoundPromptReplacement]:
652
tokenizer = self._get_tokenizer()
653
649
- return [
650
- prompt_repl.bind(modality, tokenizer)
651
- for modality, metadata in self.metadata.items()
652
- if modality in mm_data for prompt_repl in metadata.prompt_repls
653
- ]
654
+ return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
655
656
def _apply_prompt_replacements(
657
self,
657
- mm_data: MultiModalDataDict,
658
+ mm_items: MultiModalDataItems,
659
hf_inputs: BatchFeature,
660
token_ids: list[int],
660
- prompt_repls: Sequence[_BoundPromptReplacement[Any]],
661
+ prompt_repls: Sequence[_BoundPromptReplacement],
662
) -> tuple[list[int], str, list[_PlaceholderInfo]]:
663
tokenizer = self._get_tokenizer()
664
664
- mm_items = to_multi_format(mm_data)
665
token_matches = find_token_matches(token_ids, prompt_repls)
666
667
# If the search text does not represent a special token,
@@ -682,7 +682,6 @@ class BaseMultiModalProcessor(ABC):
682
token_ids,
683
token_matches,
684
mm_items,
685
- hf_inputs,
685
)
686
687
text = _decode(tokenizer, token_ids)
@@ -695,13 +694,13 @@ class BaseMultiModalProcessor(ABC):
694
text,
695
text_matches,
696
mm_items,
698
- hf_inputs,
697
)
698
699
token_ids = _encode(tokenizer, text)
700
matched_repls = [match.prompt_repl for match in text_matches]
701
704
- placeholders = self._find_placeholders(matched_repls, token_ids)
702
+ placeholders = self._find_placeholders(matched_repls, token_ids,
703
+ mm_items)
704
705
return token_ids, text, placeholders
706
@@ -731,12 +730,16 @@ class BaseMultiModalProcessor(ABC):
730
prompt_ids, = hf_inputs.pop("input_ids").tolist()
731
mm_kwargs = MultiModalKwargs(hf_inputs)
732
734
- all_prompt_repls = self._bind_prompt_replacements(mm_data)
733
+ mm_items = to_multi_format(mm_data)
734
+ prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
735
+ mm_processor_kwargs)
736
+ all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
737
738
# If HF processor already inserts placeholder tokens,
739
# there is no need for us to insert them
740
all_placeholders = self._find_placeholders(all_prompt_repls,
739
- prompt_ids)
741
+ prompt_ids, mm_items)
742
+
743
if all_placeholders:
744
prompt_text = _decode(tokenizer, prompt_ids)
745
else:
@@ -745,7 +748,7 @@ class BaseMultiModalProcessor(ABC):
748
prompt_text,
749
all_placeholders,
750
) = self._apply_prompt_replacements(
748
- mm_data,
751
+ mm_items,
752
hf_inputs,
753
prompt_ids,
754
all_prompt_repls,
@@ -765,13 +768,13 @@ class BaseMultiModalProcessor(ABC):
768
)
769
770
@abstractmethod
768
- def _get_dummy_mm_kwargs(
771
+ def _get_dummy_mm_inputs(
772
self,
773
mm_counts: Mapping[str, int],
771
- ) -> MultiModalKwargs:
774
+ ) -> ProcessorInputs:
775
"""
773
- Build the input that corresponds to `mm_max_tokens` in
774
- :meth:`get_dummy_data`.
776
+ Build the multi-modal portion of the input which, after processing,
777
+ results in `mm_max_tokens` in :meth:`get_dummy_data`.
778
"""
779
raise NotImplementedError
780
@@ -784,38 +787,41 @@ class BaseMultiModalProcessor(ABC):
787
# Avoid circular import
788
from vllm.sequence import SequenceData
789
787
- tokenizer = self._get_tokenizer()
788
-
789
- mm_placeholders = dict[str, _PlaceholderInfo]()
790
- offset = 0
791
-
792
- for modality, max_tokens in mm_max_tokens.items():
793
- if max_tokens == 0:
794
- continue
795
-
796
- metadata = self.metadata[modality]
797
- repl = metadata.prompt_repls[0].bind(modality, tokenizer)
798
- repl_token_ids = repl.repl_unit.token_ids
799
-
800
- placeholders = _PlaceholderInfo(
801
- modality=modality,
802
- start_idx=offset,
803
- unit=repl_token_ids,
804
- unit_count=max_tokens // len(repl_token_ids),
805
- )
806
-
807
- mm_placeholders[modality] = placeholders
808
- offset += placeholders.length
790
+ processor_inputs = self._get_dummy_mm_inputs(mm_counts)
791
+ mm_inputs = self.apply(*processor_inputs)
792
+
793
+ prompt_token_ids = mm_inputs["prompt_token_ids"]
794
+ placeholders_by_modality = mm_inputs["mm_placeholders"]
795
+
796
+ total_placeholders_by_modality = dict[str, int]()
797
+ for modality, placeholders in placeholders_by_modality.items():
798
+ num_placeholders = sum(item["length"] for item in placeholders)
799
+ max_tokens = mm_max_tokens[modality]
800
+
801
+ if num_placeholders != max_tokens:
802
+ logger.warning(
803
+ "The processed dummy data has a total of %d placeholder "
804
+ "tokens for the '%s' modality, which is not the expected "
805
+ "%d tokens.", num_placeholders, modality, max_tokens)
806
+
807
+ total_placeholders_by_modality[modality] = num_placeholders
808
+
809
+ total_len = len(prompt_token_ids)
810
+ if total_len > seq_len:
811
+ logger.warning(
812
+ "The context length (%d) of the model is too short "
813
+ "to hold the multi-modal embeddings in the worst case "
814
+ "(%d tokens in total, out of which %s are reserved for "
815
+ "multi-modal embeddings). This may cause certain multi-modal "
816
+ "inputs to fail during inference, even when the input text is "
817
+ "short. To avoid this, you should increase `max_model_len`, "
818
+ "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
819
+ total_len, total_placeholders_by_modality)
820
810
- prompt_token_ids = flatten_2d_lists(
811
- [p.unit * p.unit_count for p in mm_placeholders.values()])
821
prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
822
823
return DummyData(
824
seq_data=SequenceData.from_seqs(prompt_token_ids),
816
- multi_modal_data=self._get_dummy_mm_kwargs(mm_counts),
817
- multi_modal_placeholders={
818
- modality: [p.to_range()]
819
- for modality, p in mm_placeholders.items()
820
- },
825
+ multi_modal_data=mm_inputs["mm_kwargs"],
826
+ multi_modal_placeholders=placeholders_by_modality,
827
)
vllm/multimodal/registry.py
+2
-2
@@ -299,9 +299,9 @@ class MultiModalRegistry:
299
"""
300
301
def wrapper(model_cls: N) -> N:
302
- if model_cls in self._processor_factories:
302
+ if self._processor_factories.contains(model_cls, strict=True):
303
logger.warning(
304
- "Model class %s already has an input mapper "
304
+ "Model class %s already has a multi-modal processor "
305
"registered to %s. It is overwritten by the new one.",
306
model_cls, self)
307
vllm/utils.py
+9
-3
@@ -1370,8 +1370,8 @@ def supports_kw(
1370
1371
1372
def resolve_mm_processor_kwargs(
1373
- init_kwargs: Optional[Dict[str, Any]],
1374
- inference_kwargs: Optional[Dict[str, Any]],
1373
+ init_kwargs: Optional[Mapping[str, object]],
1374
+ inference_kwargs: Optional[Mapping[str, object]],
1375
callable: Callable[..., object],
1376
allow_var_kwargs: bool = False,
1377
) -> Dict[str, Any]:
@@ -1405,7 +1405,7 @@ def resolve_mm_processor_kwargs(
1405
1406
def get_allowed_kwarg_only_overrides(
1407
callable: Callable[..., object],
1408
- overrides: Optional[Dict[str, Any]],
1408
+ overrides: Optional[Mapping[str, object]],
1409
allow_var_kwargs: bool = False,
1410
) -> Dict[str, Any]:
1411
"""
@@ -1524,9 +1524,15 @@ class ClassRegistry(UserDict[Type[T], _V]):
1524
raise KeyError(key)
1525
1526
def __contains__(self, key: object) -> bool:
1527
+ return self.contains(key)
1528
+
1529
+ def contains(self, key: object, *, strict: bool = False) -> bool:
1530
if not isinstance(key, type):
1531
return False
1532
1533
+ if strict:
1534
+ return key in self.data
1535
+
1536
return any(cls in self.data for cls in key.mro())
1537
1538