@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93abf23a64
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[VLM] Fully dynamic prompt replacement in merged input processor (#11199)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 15, 2024 at 01:52 UTC 93abf23a648051fe6dc053ba0b74499d119920bf
12 files changed +569 -510
examples/offline_inference_vision_language.py
+1 -4
@@ -97,9 +97,6 @@ def run_phi3v(question: str, modality: str):
97 # max_model_len (128k) for this model may cause OOM.
98 # You may lower either to run this example on lower-end GPUs.
99
100 - # In this example, we override max_num_seqs to 5 while
101 - # keeping the original context length of 128k.
102 -
100 # num_crops is an override kwarg to the multimodal image processor;
101 # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
102 # to use 16 for single frame scenarios, and 4 for multi-frame.
@@ -113,7 +110,7 @@ def run_phi3v(question: str, modality: str):
110 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
111 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
112 llm = LLM(
116 - model="microsoft/Phi-3-vision-128k-instruct",
113 + model="microsoft/Phi-3.5-vision-instruct",
114 trust_remote_code=True,
115 max_model_len=4096,
116 max_num_seqs=2,
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2 -2
@@ -16,8 +16,8 @@ models = ["microsoft/Phi-3.5-vision-instruct"]
16 # Wrap lazy imports to avoid initializing CUDA during test collection
17 @pytest.fixture()
18 def processor_for_phi3v():
19 - from vllm.model_executor.models.phi3v import Phi3VProcessor
20 - return Phi3VProcessor
19 + from vllm.model_executor.models.phi3v import Phi3VMultiModalProcessor
20 + return Phi3VMultiModalProcessor
21
22
23 @pytest.fixture()
tests/multimodal/test_processing.py
+54 -51
@@ -1,11 +1,11 @@
1 from typing import cast
2
3 import pytest
4 -from transformers import BatchFeature
4
6 -from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
7 - find_text_matches, find_token_matches,
8 - iter_placeholders, iter_token_matches,
5 +from vllm.multimodal.processing import (MultiModalDataItems, PromptReplacement,
6 + _PlaceholderInfo, find_text_matches,
7 + find_token_matches, iter_placeholders,
8 + iter_token_matches,
9 replace_text_matches,
10 replace_token_matches)
11 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -16,7 +16,7 @@ from vllm.utils import full_groupby
16 @pytest.mark.parametrize(
17 ("token_ids", "match_ids", "expected"),
18 [
19 - ([], [], [{ "start_idx": 0, "end_idx": 0 }]),
19 + ([], [], []),
20 ([], [32000], []),
21 (
22 [32000, 32000, 32000],
@@ -83,7 +83,7 @@ def test_iter_token_matches(token_ids, match_ids, expected):
83 "pattern_2": [32000],
84 },
85 {
86 - "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
86 + "pattern_1": [],
87 "pattern_2": [],
88 }
89 ),
@@ -136,7 +136,7 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
136 mock_tokenizer = cast(AnyTokenizer, object())
137
138 prompt_repls = [
139 - PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
139 + PromptReplacement(key, target, []).bind(mock_tokenizer)
140 for key, target in target_by_key.items()
141 ]
142 result = find_token_matches(prompt, prompt_repls)
@@ -243,7 +243,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
243 mock_tokenizer = cast(AnyTokenizer, object())
244
245 prompt_repls = [
246 - PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
246 + PromptReplacement(key, target, []).bind(mock_tokenizer)
247 for key, target in target_by_key.items()
248 ]
249 result = find_text_matches(prompt, prompt_repls)
@@ -276,12 +276,12 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
276 "pattern_3": "!",
277 },
278 {
279 - # Test whether target is confused with repl_unit
280 - "pattern_1": ("<image><image>", 1),
281 - # Test empty repl_unit
282 - "pattern_2": ("", 1),
283 - # Test multiple repl_count
284 - "pattern_3": ("?", 2),
279 + # Test whether target is confused with replacement
280 + "pattern_1": "<image><image>",
281 + # Test empty replacement
282 + "pattern_2": "",
283 + # Test dynamic replacement (beyond the form of `unit * count`)
284 + "pattern_3": "?!?",
285 },
286 ),
287 ]
@@ -290,8 +290,8 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
290 ("mm_count", "expected"),
291 [
292 (0, "Image:<image>Image:<image><image>!"),
293 - (1, "<image><image>Image:<image><image>??"),
294 - (2, "<image><image><image><image><image>??"),
293 + (1, "<image><image>Image:<image><image>?!?"),
294 + (2, "<image><image><image><image><image>?!?"),
295 ]
296 )
297 # yapf: enable
@@ -306,7 +306,7 @@ def test_find_replace_text(
306 mock_tokenizer = cast(AnyTokenizer, object())
307
308 prompt_repls = [
309 - PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
309 + PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
310 for key, target in target_by_key.items()
311 ]
312 matches = find_text_matches(prompt, prompt_repls)
@@ -314,9 +314,8 @@ def test_find_replace_text(
314 result = replace_text_matches(
315 prompt,
316 matches,
317 - {key: list(range(mm_count))
318 - for key in repl_by_key},
319 - BatchFeature(),
317 + MultiModalDataItems({key: [None] * mm_count
318 + for key in repl_by_key}),
319 )
320
321 # Only displayed on error
@@ -343,12 +342,12 @@ def test_find_replace_text(
342 "pattern_3": [918],
343 },
344 {
346 - # Test whether target is confused with repl_unit
347 - "pattern_1": ([32000, 32000], 1),
348 - # Test empty repl_unit
349 - "pattern_2": ([], 1),
350 - # Test multiple repl_count
351 - "pattern_3": ([1550], 2),
345 + # Test whether target is confused with replacement
346 + "pattern_1": [32000, 32000],
347 + # Test empty replacement
348 + "pattern_2": [],
349 + # Test dynamic replacement (beyond the form of `unit * count`)
350 + "pattern_3": [1550, 918, 1550],
351 },
352 ),
353 ]
@@ -357,8 +356,8 @@ def test_find_replace_text(
356 ("mm_count", "expected"),
357 [
358 (0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
360 - (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550]),
361 - (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550]),
359 + (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550]),
360 + (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550]),
361 ]
362 )
363 # yapf: enable
@@ -373,7 +372,7 @@ def test_find_replace_tokens(
372 mock_tokenizer = cast(AnyTokenizer, object())
373
374 prompt_repls = [
376 - PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
375 + PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
376 for key, target in target_by_key.items()
377 ]
378 matches = find_token_matches(prompt, prompt_repls)
@@ -381,9 +380,8 @@ def test_find_replace_tokens(
380 result = replace_token_matches(
381 prompt,
382 matches,
384 - {key: list(range(mm_count))
385 - for key in repl_by_key},
386 - BatchFeature(),
383 + MultiModalDataItems({key: [None] * mm_count
384 + for key in repl_by_key}),
385 )
386
387 # Only displayed on error
@@ -399,9 +397,9 @@ def test_find_replace_tokens(
397 "repl_by_key",
398 [
399 {
402 - "pattern_1": ([32000, 32000], 1),
403 - "pattern_2": ([], 1),
404 - "pattern_3": ([1550], 2),
400 + "pattern_1": [32000, 32000],
401 + "pattern_2": [],
402 + "pattern_3": [1550, 918, 1550],
403 },
404 ],
405 )
@@ -414,48 +412,47 @@ def test_find_replace_tokens(
412 _PlaceholderInfo(
413 modality="pattern_1",
414 start_idx=6,
417 - unit=[32000, 32000],
418 - unit_count=1,
415 + replacement=[32000, 32000],
416 ),
417 ],
418 ),
419 (
423 - [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550],
420 + [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
421 [
422 _PlaceholderInfo(
423 modality="pattern_1",
424 start_idx=1,
428 - unit=[32000, 32000],
429 - unit_count=1,
425 + replacement=[32000, 32000],
426 ),
427 _PlaceholderInfo(
428 modality="pattern_1",
429 start_idx=5,
434 - unit=[32000, 32000],
435 - unit_count=1,
430 + replacement=[32000, 32000],
431 ),
432 _PlaceholderInfo(
433 modality="pattern_3",
434 start_idx=7,
440 - unit=[1550],
441 - unit_count=2,
435 + replacement=[1550, 918, 1550],
436 ),
437 ],
438 ),
439 (
446 - [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550],
440 + [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
441 [
442 _PlaceholderInfo(
443 modality="pattern_1",
444 start_idx=1,
451 - unit=[32000, 32000],
452 - unit_count=2,
445 + replacement=[32000, 32000],
446 + ),
447 + _PlaceholderInfo(
448 + modality="pattern_1",
449 + start_idx=3,
450 + replacement=[32000, 32000],
451 ),
452 _PlaceholderInfo(
453 modality="pattern_3",
454 start_idx=6,
457 - unit=[1550],
458 - unit_count=2,
455 + replacement=[1550, 918, 1550],
456 ),
457 ],
458 ),
@@ -470,11 +467,17 @@ def test_iter_placeholders(
467 mock_tokenizer = cast(AnyTokenizer, object())
468
469 prompt_repls = [
473 - PromptReplacement([], *repl).bind(key, mock_tokenizer)
470 + PromptReplacement(key, [], repl).bind(mock_tokenizer)
471 for key, repl in repl_by_key.items()
472 ]
473
477 - result = list(iter_placeholders(prompt_repls, prompt))
474 + result = list(
475 + iter_placeholders(
476 + prompt_repls,
477 + prompt,
478 + # Effectively match all occurrences in the prompt
479 + MultiModalDataItems({key: [None] * 3 for key in repl_by_key}),
480 + ))
481
482 # Only displayed on error
483 print("result:", result)
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+2 -2
@@ -3,14 +3,14 @@ from typing import Optional
3 import torch
4
5 from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6 - LlavaProcessor,
6 + LlavaMultiModalProcessor,
7 get_max_llava_image_tokens)
8 from vllm.model_executor.sampling_metadata import SamplingMetadata
9 from vllm.multimodal import MULTIMODAL_REGISTRY
10
11
12 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
13 -@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
13 +@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
14 class MyLlava(LlavaForConditionalGeneration):
15
16 def compute_logits(
vllm/inputs/registry.py
+61 -10
@@ -2,7 +2,7 @@ import functools
2 from collections import UserDict
3 from dataclasses import dataclass
4 from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5 - Optional, Protocol, Type, cast)
5 + Optional, Protocol, Type)
6
7 from torch import nn
8 from transformers import PretrainedConfig, ProcessorMixin
@@ -47,7 +47,6 @@ class InputContext:
47 Raises:
48 TypeError: If the model is not of the specified type.
49 """
50 -
50 hf_config = self.model_config.hf_config
51 if not isinstance(hf_config, hf_config_type):
52 raise TypeError("Invalid type of HuggingFace config. "
@@ -60,21 +59,70 @@ class InputContext:
59 """
60 Get the HuggingFace image processor configuration of the model.
61 """
63 -
62 return self.model_config.hf_image_processor_config
63
64 + def get_mm_config(self):
65 + """
66 + Get the multimodal config of the model.
67 +
68 + Raises:
69 + RuntimeError: If the model is not a multimodal model.
70 + """
71 + mm_config = self.model_config.multimodal_config
72 + if mm_config is None:
73 + raise RuntimeError("Not a multimodal model")
74 +
75 + return mm_config
76 +
77 + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
78 + base_kwargs = self.model_config.mm_processor_kwargs
79 + if base_kwargs is None:
80 + base_kwargs = {}
81 +
82 + merged_kwargs = {**base_kwargs, **kwargs}
83 +
84 + return cached_get_processor(
85 + self.model_config.model,
86 + trust_remote_code=self.model_config.trust_remote_code,
87 + **merged_kwargs,
88 + )
89 +
90
91 @dataclass(frozen=True)
92 class InputProcessingContext(InputContext):
93 tokenizer: AnyTokenizer
94 """The tokenizer used to tokenize the inputs."""
95
72 - def get_hf_processor(self, **kwargs) -> ProcessorMixin:
96 + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
97 + base_kwargs = self.model_config.mm_processor_kwargs
98 + if base_kwargs is None:
99 + base_kwargs = {}
100 +
101 + merged_kwargs = {**base_kwargs, **kwargs}
102 +
103 return cached_get_processor(
74 - self.model_config.tokenizer,
104 + self.model_config.model,
105 tokenizer=self.tokenizer, # Override the tokenizer with ours
106 trust_remote_code=self.model_config.trust_remote_code,
77 - **kwargs)
107 + **merged_kwargs,
108 + )
109 +
110 + def resolve_hf_processor_call_kwargs(
111 + self,
112 + hf_processor: ProcessorMixin,
113 + inference_kwargs: Mapping[str, object],
114 + ) -> Mapping[str, object]:
115 + assert callable(hf_processor)
116 +
117 + base_kwargs = self.model_config.mm_processor_kwargs
118 + if base_kwargs is None:
119 + base_kwargs = {}
120 +
121 + return resolve_mm_processor_kwargs(
122 + base_kwargs,
123 + inference_kwargs,
124 + hf_processor,
125 + )
126
127
128 N = TypeVar("N", bound=Type[nn.Module])
@@ -171,7 +219,8 @@ class InputRegistry:
219 """
220
221 def wrapper(model_cls: N) -> N:
174 - if model_cls in self._dummy_factories_by_model_type:
222 + if self._dummy_factories_by_model_type.contains(model_cls,
223 + strict=True):
224 logger.warning(
225 "Model class %s already has dummy data "
226 "registered to %s. It is overwritten by the new one.",
@@ -195,7 +244,8 @@ class InputRegistry:
244 """
245
246 def wrapper(model_cls: N) -> N:
198 - if model_cls in self._dummy_encoder_factories_by_model_type:
247 + if self._dummy_encoder_factories_by_model_type.contains(
248 + model_cls, strict=True):
249 logger.warning(
250 "Model class %s already has dummy encoder data "
251 "registered to %s. It is overwritten by the new one.",
@@ -305,7 +355,8 @@ class InputRegistry:
355 """
356
357 def wrapper(model_cls: N) -> N:
308 - if model_cls in self._input_processors_by_model_type:
358 + if self._input_processors_by_model_type.contains(model_cls,
359 + strict=True):
360 logger.warning(
361 "Model class %s already has input processor "
362 "registered to %s. It is overwritten by the new one.",
@@ -357,7 +408,7 @@ class InputRegistry:
408 # If it's empty, it'll fall back to the default kwarg values
409 mm_processor_kwargs = resolve_mm_processor_kwargs(
410 model_config.mm_processor_kwargs,
360 - cast(Dict[str, Any], inputs.get("mm_processor_kwargs")),
411 + inputs.get("mm_processor_kwargs", {}), # type: ignore
412 processor,
413 )
414
vllm/model_executor/models/llava.py
+72 -72
@@ -5,10 +5,10 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
5
6 import torch
7 import torch.nn as nn
8 -from PIL.Image import Image
8 from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
9 PixtralVisionConfig, PretrainedConfig,
10 ProcessorMixin, SiglipVisionConfig)
11 +from transformers.models.llava import LlavaProcessor
12 from transformers.models.pixtral import PixtralProcessor
13
14 from vllm.attention import AttentionMetadata
@@ -21,11 +21,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
21 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
24 +from vllm.multimodal.inputs import NestedTensors
25 from vllm.multimodal.processing import (BaseMultiModalProcessor,
26 - InputProcessingContext,
27 - ModalityProcessingMetadata,
28 - MultiModalProcessingMetadata,
26 + MultiModalDataItems, ProcessorInputs,
27 PromptReplacement)
28 from vllm.sequence import IntermediateTensors
29
@@ -33,7 +31,8 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
31 get_max_clip_image_tokens)
32 from .interfaces import SupportsMultiModal, SupportsPP
33 from .pixtral import (PixtralHFVisionModel, dummy_image_for_pixtral_hf,
36 - get_max_pixtral_hf_image_tokens)
34 + get_max_pixtral_hf_image_tokens,
35 + get_pixtral_hf_image_feature_size)
36 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
37 get_max_siglip_image_tokens)
38 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
@@ -115,62 +114,7 @@ def get_max_llava_image_tokens(ctx: InputContext):
114 raise ValueError(f"Unexpected select feature strategy: {strategy}")
115
116
118 -def dummy_mm_kwargs_for_llava(ctx: InputProcessingContext,
119 - mm_counts: Mapping[str, int]):
120 - hf_config = ctx.get_hf_config(LlavaConfig)
121 - vision_config = hf_config.vision_config
122 - num_images = mm_counts["image"]
123 -
124 - if isinstance(vision_config, CLIPVisionConfig):
125 - data = dummy_image_for_clip(vision_config, num_images)
126 - elif isinstance(vision_config, SiglipVisionConfig):
127 - data = dummy_image_for_siglip(vision_config, num_images)
128 - elif isinstance(vision_config, PixtralVisionConfig):
129 - data = dummy_image_for_pixtral_hf(vision_config, num_images)
130 - else:
131 - msg = f"Unsupported vision config: {type(vision_config)}"
132 - raise NotImplementedError(msg)
133 -
134 - hf_processor = ctx.get_hf_processor()
135 - image_processor = hf_processor.image_processor # type: ignore
136 - hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
137 - is_pixtral = isinstance(hf_processor, PixtralProcessor)
138 -
139 - return MultiModalKwargs(
140 - **hf_inputs,
141 - is_pixtral=torch.tensor(is_pixtral),
142 - )
143 -
144 -
145 -def create_metadata_for_llava(
146 - ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
147 - hf_config = ctx.get_hf_config(LlavaConfig)
148 - image_token_id = hf_config.image_token_index
149 -
150 - def get_repl_count(
151 - mm_items: list[Image],
152 - hf_inputs: BatchFeature,
153 - item_idx: int,
154 - ) -> int:
155 - return get_max_llava_image_tokens(ctx)
156 -
157 - return {
158 - "image":
159 - ModalityProcessingMetadata(prompt_repls=[
160 - PromptReplacement(target=[image_token_id],
161 - repl_unit=[image_token_id],
162 - repl_count=get_repl_count),
163 - ]),
164 - }
165 -
166 -
167 -class LlavaProcessor(BaseMultiModalProcessor):
168 -
169 - def __init__(self, ctx: InputProcessingContext) -> None:
170 - super().__init__(
171 - ctx=ctx,
172 - metadata=create_metadata_for_llava(ctx),
173 - )
117 +class LlavaMultiModalProcessor(BaseMultiModalProcessor):
118
119 def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
120 if getattr(hf_processor, "__is_patched__", False):
@@ -188,18 +132,72 @@ class LlavaProcessor(BaseMultiModalProcessor):
132
133 hf_processor.__is_patched__ = True # type: ignore
134
191 - def _get_hf_processor(self) -> ProcessorMixin:
135 + def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136 hf_processor = self.ctx.get_hf_processor()
137 + assert isinstance(hf_processor, (LlavaProcessor, PixtralProcessor))
138
139 if isinstance(hf_processor, PixtralProcessor):
140 self._patch_pixtral_processor(hf_processor)
141
142 return hf_processor
143
199 - def _get_dummy_mm_kwargs(
144 + def _get_prompt_replacements(
145 + self,
146 + mm_items: MultiModalDataItems,
147 + hf_inputs: BatchFeature,
148 + mm_processor_kwargs: Mapping[str, object],
149 + ) -> list[PromptReplacement]:
150 + hf_config = self.ctx.get_hf_config(LlavaConfig)
151 + image_token_id = hf_config.image_token_index
152 +
153 + processor = self._get_hf_processor()
154 + if isinstance(processor, PixtralProcessor):
155 + image_token = processor.image_token
156 + image_break_token = processor.image_break_token
157 + image_end_token = processor.image_end_token
158 +
159 + vision_config = hf_config.vision_config
160 + assert isinstance(vision_config, PixtralVisionConfig)
161 +
162 + def get_replacement_pixtral(item_idx: int):
163 + image_size = mm_items.get_image_size(item_idx)
164 + (
165 + num_width_tokens,
166 + num_height_tokens,
167 + ) = get_pixtral_hf_image_feature_size(
168 + vision_config,
169 + image_width=image_size.width,
170 + image_height=image_size.height,
171 + )
172 +
173 + tokens = ([image_token] * num_width_tokens +
174 + [image_break_token]) * num_height_tokens
175 + tokens[-1] = image_end_token
176 +
177 + return "".join(tokens)
178 +
179 + return [
180 + PromptReplacement(
181 + modality="image",
182 + target=[image_token_id],
183 + replacement=get_replacement_pixtral,
184 + ),
185 + ]
186 +
187 + max_image_tokens = get_max_llava_image_tokens(self.ctx)
188 +
189 + return [
190 + PromptReplacement(
191 + modality="image",
192 + target=[image_token_id],
193 + replacement=[image_token_id] * max_image_tokens,
194 + )
195 + ]
196 +
197 + def _get_dummy_mm_inputs(
198 self,
199 mm_counts: Mapping[str, int],
202 - ) -> MultiModalKwargs:
200 + ) -> ProcessorInputs:
201 hf_config = self.ctx.get_hf_config(LlavaConfig)
202 vision_config = hf_config.vision_config
203 num_images = mm_counts["image"]
@@ -215,11 +213,13 @@ class LlavaProcessor(BaseMultiModalProcessor):
213 raise NotImplementedError(msg)
214
215 hf_processor = self._get_hf_processor()
218 - image_processor = hf_processor.image_processor # type: ignore
219 - hf_inputs = image_processor.preprocess(data['image'],
220 - return_tensors="pt")
216 + image_token = hf_processor.image_token
217
222 - return MultiModalKwargs(**hf_inputs)
218 + return ProcessorInputs(
219 + prompt_text=image_token * num_images,
220 + mm_data=data,
221 + mm_processor_kwargs={},
222 + )
223
224
225 class LlavaLikeConfig(Protocol):
@@ -303,7 +303,7 @@ def init_vision_tower_for_llava(
303
304
305 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
306 -@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
306 +@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
307 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
308 # BitandBytes specific attributes
309 bitsandbytes_stacked_params_mapping = {
@@ -584,7 +584,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
584 return loader.load_weights(weights)
585
586
587 -class MantisProcessor(LlavaProcessor):
587 +class MantisMultiModalProcessor(LlavaMultiModalProcessor):
588
589 def _get_hf_processor(self) -> ProcessorMixin:
590 try:
@@ -604,6 +604,6 @@ class MantisProcessor(LlavaProcessor):
604 # To use this model, please use
605 # `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
606 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
607 -@MULTIMODAL_REGISTRY.register_processor(MantisProcessor)
607 +@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
608 class MantisForConditionalGeneration(LlavaForConditionalGeneration):
609 pass
vllm/model_executor/models/phi3v.py
+57 -61
@@ -32,13 +32,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 from vllm.model_executor.models.clip import CLIPVisionModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.image import cached_get_image_processor
36 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
35 +from vllm.multimodal.inputs import NestedTensors
36 from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 - InputProcessingContext,
39 - ModalityProcessingMetadata,
37 MultiModalDataDict,
41 - MultiModalProcessingMetadata,
38 + MultiModalDataItems, ProcessorInputs,
39 PromptReplacement)
40 from vllm.sequence import IntermediateTensors
41 from vllm.utils import is_list_of
@@ -305,64 +302,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
302 return image_features_hd_newline
303
304
308 -def get_max_phi3v_image_tokens(ctx: InputContext,
309 - *,
310 - num_crops: Optional[int] = None):
311 - mm_processor_kwargs = {}
312 - if num_crops is not None:
313 - mm_processor_kwargs["num_crops"] = num_crops
305 +def get_max_phi3v_image_tokens(ctx: InputContext) -> int:
306 + processor = ctx.get_hf_processor()
307 + image_processor = processor.image_processor # type: ignore
308
315 - model_config = ctx.model_config
316 - image_processor = cached_get_image_processor(
317 - model_config.model,
318 - trust_remote_code=model_config.trust_remote_code,
319 - **mm_processor_kwargs,
320 - )
321 -
322 - num_tokens = image_processor.calc_num_image_tokens_from_image_size(
309 + return image_processor.calc_num_image_tokens_from_image_size(
310 width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
311 height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
312 )
326 - return num_tokens
327 -
328 -
329 -def dummy_mm_kwargs_for_phi3v(ctx: InputProcessingContext,
330 - mm_counts: Mapping[str, int]):
331 - num_images = mm_counts["image"]
332 -
333 - data = dummy_image_for_clip(
334 - CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335 - num_images,
336 - image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
337 - image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
338 - )
339 -
340 - hf_processor = ctx.get_hf_processor()
341 - image_processor = hf_processor.image_processor # type: ignore
342 - hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
343 -
344 - return MultiModalKwargs(**hf_inputs)
345 -
346 -
347 -def create_metadata_for_phi3v(
348 - ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
349 - return {
350 - "image":
351 - ModalityProcessingMetadata(prompt_repls=[
352 - PromptReplacement(target=[_IMAGE_TOKEN_ID],
353 - repl_unit=[_IMAGE_TOKEN_ID],
354 - repl_count=get_max_phi3v_image_tokens(ctx)),
355 - ]),
356 - }
357 -
313
359 -class Phi3VProcessor(BaseMultiModalProcessor):
314
361 - def __init__(self, ctx: InputProcessingContext) -> None:
362 - super().__init__(
363 - ctx=ctx,
364 - metadata=create_metadata_for_phi3v(ctx),
365 - )
315 +class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
316
317 def _get_hf_processor(
318 self,
@@ -389,15 +339,61 @@ class Phi3VProcessor(BaseMultiModalProcessor):
339 processed_outputs['input_ids'] = token_ids
340 return processed_outputs
341
392 - def _get_dummy_mm_kwargs(
342 + def _get_prompt_replacements(
343 + self,
344 + mm_items: MultiModalDataItems,
345 + hf_inputs: BatchFeature,
346 + mm_processor_kwargs: Mapping[str, object],
347 + ) -> list[PromptReplacement]:
348 + hf_processor = self._get_hf_processor()
349 + image_tokens: list[str] = hf_processor.img_tokens # type: ignore
350 + image_processor = hf_processor.image_processor # type: ignore
351 +
352 + mm_config = self.ctx.get_mm_config()
353 + max_images = mm_config.limit_per_prompt.get("image", 1)
354 +
355 + def get_replacement_phi3v(item_idx: int):
356 + image_size = mm_items.get_image_size(item_idx)
357 + num_tokens = image_processor.calc_num_image_tokens_from_image_size(
358 + width=image_size.width,
359 + height=image_size.height,
360 + )
361 +
362 + return [_IMAGE_TOKEN_ID] * num_tokens
363 +
364 + return [
365 + PromptReplacement(
366 + modality="image",
367 + target=image_token,
368 + replacement=get_replacement_phi3v,
369 + ) for image_token in image_tokens[:max_images]
370 + ]
371 +
372 + def _get_dummy_mm_inputs(
373 self,
374 mm_counts: Mapping[str, int],
395 - ) -> MultiModalKwargs:
396 - return dummy_mm_kwargs_for_phi3v(self.ctx, mm_counts)
375 + ) -> ProcessorInputs:
376 + num_images = mm_counts["image"]
377 +
378 + data = dummy_image_for_clip(
379 + CLIP_VIT_LARGE_PATCH14_336_CONFIG,
380 + num_images,
381 + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
382 + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
383 + )
384 +
385 + hf_processor = self._get_hf_processor()
386 + image_tokens: list[str] = hf_processor.img_tokens # type: ignore
387 +
388 + return ProcessorInputs(
389 + prompt_text="".join(image_tokens[:num_images]),
390 + mm_data=data,
391 + mm_processor_kwargs={},
392 + )
393
394
395 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
400 -@MULTIMODAL_REGISTRY.register_processor(Phi3VProcessor)
396 +@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
397 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
398
399 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/pixtral.py
+1 -1
@@ -72,7 +72,7 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
72 mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
73 image_token_id = mm_encoder.special_ids.img
74
75 - mm_config = ctx.model_config.multimodal_config
75 + mm_config = ctx.get_mm_config()
76 num_images = mm_config.limit_per_prompt.get("image", 1)
77
78 # dummy size
vllm/multimodal/base.py
+2 -2
@@ -99,7 +99,7 @@ class MultiModalPlugin(ABC):
99 """
100
101 def wrapper(model_cls: N) -> N:
102 - if model_cls in self._input_mappers:
102 + if self._input_mappers.contains(model_cls, strict=True):
103 logger.warning(
104 "Model class %s already has an input mapper "
105 "registered to %s. It is overwritten by the new one.",
@@ -194,7 +194,7 @@ class MultiModalPlugin(ABC):
194 """
195
196 def wrapper(model_cls: N) -> N:
197 - if model_cls in self._max_mm_tokens:
197 + if self._max_mm_tokens.contains(model_cls, strict=True):
198 logger.warning(
199 "Model class %s already calculates maximum number of "
200 "tokens in %s. It is overwritten by the new one.",
vllm/multimodal/processing.py
+306 -300
@@ -1,116 +1,59 @@
1 import re
2 from abc import ABC, abstractmethod
3 +from collections import UserDict
4 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4 -from dataclasses import dataclass
5 +from dataclasses import dataclass, field
6 from functools import lru_cache
6 -from typing import (Any, Dict, Generic, NamedTuple, Optional, Protocol,
7 - TypeVar, Union, cast)
7 +from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
9 +import numpy as np
10 import torch
11 +from PIL.Image import Image
12 from transformers import BatchFeature, ProcessorMixin
11 -from typing_extensions import TypeAlias, TypedDict
13 +from typing_extensions import assert_never
14
15 from vllm.inputs import DummyData, InputProcessingContext
16 +from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
15 -from vllm.utils import (flatten_2d_lists, full_groupby, is_list_of,
16 - resolve_mm_processor_kwargs)
18 +from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
19
20 from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
21 MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
22 VideoItem)
23
24 +logger = init_logger(__name__)
25
23 -def bind_prompt_sequence(
24 - seq: Union[str, list[int]],
25 - tokenizer: AnyTokenizer,
26 -) -> "_BoundPromptSequence":
27 - """
28 - Bind a text or token sequence to a tokenizer so that it can be
29 - lazily converted into the other format on demand.
30 - """
31 - return _BoundPromptSequence(
32 - tokenizer=tokenizer,
33 - _text=seq if isinstance(seq, str) else None,
34 - _token_ids=seq if isinstance(seq, list) else None,
35 - )
36 -
37 -
38 -_T = TypeVar("_T")
26 _S = TypeVar("_S", str, list[int])
27 +_PromptSeq = Union[str, list[int]]
28
29
30 @dataclass
43 -class PromptReplacement(Generic[_S, _T]):
44 - target: _S
45 - """The text or token sequence to find and replace."""
31 +class PromptReplacement:
32 + modality: str
33 + """The modality for which the replacement is made"""
34
47 - repl_unit: _S
48 - """
49 - The unit making up the replacement text or token sequence.
50 -
51 - See :code:`repl_count` for more details.
52 - """
35 + target: _PromptSeq
36 + """The text or token sequence to find and replace."""
37
54 - repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
38 + replacement: Union[Callable[[int], _PromptSeq],
39 + _PromptSeq] = field(repr=False)
40 """
56 - Given the original multi-modal items for this modality, HF-processed data,
57 - and index of the processed item, output the number of repetitions of
58 - :code:`repl_unit` to build up the replacement text or token sequence.
41 + Given the index of the processed item within :attr:`modality`, output the
42 + replacement text or token sequence.
43
60 - For convenience, you can pass in an integer if the number of repetitions is
61 - a constant.
44 + For convenience, you can pass in the replacement instead of a function
45 + if it does not depend on the input.
46 """
47
64 - def __repr__(self) -> str:
65 - return (f"{type(self).__name__}(target={self.target!r}, "
66 - f"repl_unit={self.repl_unit!r})")
67 -
68 - def bind(
69 - self,
70 - modality: str,
71 - tokenizer: AnyTokenizer,
72 - ) -> "_BoundPromptReplacement[_T]":
48 + def bind(self, tokenizer: AnyTokenizer) -> "_BoundPromptReplacement":
49 return _BoundPromptReplacement(
74 - modality=modality,
75 - target=bind_prompt_sequence(self.target, tokenizer),
76 - repl_unit=bind_prompt_sequence(self.repl_unit, tokenizer),
77 - repl_count=self.repl_count,
50 + tokenizer=tokenizer,
51 + modality=self.modality,
52 + _target=self.target,
53 + _replacement=self.replacement,
54 )
55
56
81 -@dataclass
82 -class ModalityProcessingMetadata(Generic[_T]):
83 - prompt_repls: Sequence[Union[PromptReplacement[str, _T],
84 - PromptReplacement[list[int], _T]]]
85 - """
86 - Defines each text or token sequence to replace in the HF-processed prompt.
87 -
88 - This is skipped if the HF-processed prompt is found to already contain
89 - the replacement prompts.
90 - """
91 -
92 -
93 -class MultiModalProcessingMetadataBuiltins(TypedDict, total=False):
94 - """Type annotations for modality types predefined by vLLM."""
95 -
96 - image: ModalityProcessingMetadata[ImageItem]
97 - video: ModalityProcessingMetadata[VideoItem]
98 - audio: ModalityProcessingMetadata[AudioItem]
99 -
100 -
101 -MultiModalProcessingMetadata: TypeAlias = \
102 - Mapping[str, ModalityProcessingMetadata[Any]]
103 -"""
104 -A dictionary containing an entry for each modality type to process.
105 -
106 -Note:
107 - This dictionary also accepts modality keys defined outside
108 - :class:`MultiModalProcessingMetadataBuiltins` as long as a customized plugin
109 - is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
110 - Read more on that :ref:`here <adding_multimodal_plugin>`.
111 -"""
112 -
113 -
57 def _encode(
58 tokenizer: AnyTokenizer,
59 text: str,
@@ -185,7 +128,8 @@ def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
128
129 @dataclass
130 class _BoundPromptSequence:
188 - tokenizer: AnyTokenizer
131 + tokenizer: AnyTokenizer = field(repr=False)
132 +
133 _text: Optional[str]
134 _token_ids: Optional[list[int]]
135
@@ -210,38 +154,92 @@ class _BoundPromptSequence:
154
155 return self._token_ids
156
213 - def __repr__(self) -> str:
214 - return (f"{type(self).__name__}(_text={self._text!r}, "
215 - f"_token_ids={self._token_ids!r})")
216 -
157
158 @dataclass
219 -class _BoundPromptReplacement(Generic[_T]):
159 +class _BoundPromptReplacement:
160 + tokenizer: AnyTokenizer = field(repr=False)
161 modality: str
221 - target: _BoundPromptSequence
222 - repl_unit: _BoundPromptSequence
223 - repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
162
225 - def get_count(
226 - self,
227 - mm_items: list[_T],
228 - hf_inputs: BatchFeature,
229 - item_idx: int,
230 - ) -> int:
231 - repl_count = self.repl_count
232 - if isinstance(repl_count, int):
233 - return repl_count
163 + _target: _PromptSeq
164 + _replacement: Union[Callable[[int], _PromptSeq],
165 + _PromptSeq] = field(repr=False)
166
235 - return repl_count(mm_items, hf_inputs, item_idx)
167 + def __post_init__(self) -> None:
168 + self._replacement_cache = dict[int, _BoundPromptSequence]()
169 +
170 + @property
171 + def target(self) -> _BoundPromptSequence:
172 + target = self._target
173
174 + return _BoundPromptSequence(
175 + tokenizer=self.tokenizer,
176 + _text=target if isinstance(target, str) else None,
177 + _token_ids=target if isinstance(target, list) else None,
178 + )
179
238 -def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
180 + def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
181 + replacement = self._replacement
182 + if callable(replacement):
183 + cache_key = item_idx
184 + if cache_key in self._replacement_cache:
185 + return self._replacement_cache[cache_key]
186 +
187 + replacement = replacement(item_idx)
188 + else:
189 + cache_key = None
190 +
191 + bound_replacement = _BoundPromptSequence(
192 + tokenizer=self.tokenizer,
193 + _text=replacement if isinstance(replacement, str) else None,
194 + _token_ids=replacement if isinstance(replacement, list) else None,
195 + )
196 +
197 + if cache_key is not None:
198 + self._replacement_cache[cache_key] = bound_replacement
199 +
200 + return bound_replacement
201 +
202 +
203 +class ImageSize(NamedTuple):
204 + width: int
205 + height: int
206 +
207 +
208 +class MultiModalDataItems(UserDict[str, list[Any]]):
209 """
240 - Convert a :class:`MultiModalDataDict` containing single data items
241 - to a :class:`MultiModalMultiDataDict` containing multiple data items
242 - per entry.
210 + As :class:`MultiModalDataDict`, but normalized such that each entry
211 + corresponds to a list.
212 """
244 - multi_data = dict[str, list[Any]]()
213 +
214 + @property
215 + def image(self) -> list[ImageItem]:
216 + return self["image"]
217 +
218 + @property
219 + def video(self) -> list[VideoItem]:
220 + return self["video"]
221 +
222 + @property
223 + def audio(self) -> list[AudioItem]:
224 + return self["audio"]
225 +
226 + def get_image_size(self, item_idx: int) -> ImageSize:
227 + image = self.image[item_idx]
228 +
229 + if isinstance(image, Image):
230 + return ImageSize(*image.size)
231 + if isinstance(image, (np.ndarray, torch.Tensor)):
232 + _, h, w = image.shape
233 + return ImageSize(w, h)
234 +
235 + assert_never(image)
236 +
237 +
238 +def to_multi_format(data: MultiModalDataDict) -> MultiModalDataItems:
239 + """
240 + Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
241 + """
242 + multi_data = MultiModalDataItems()
243
244 for k, v in data.items():
245 # yapf: disable
@@ -266,22 +264,33 @@ def iter_token_matches(
264 token_ids: list[int],
265 match_ids: list[int],
266 ) -> Iterable[_TokenMatch]:
269 - """Yield each occurrence of :code:`match_ids` in :code:`token_ids`."""
267 + """
268 + Yield each occurrence of :code:`match_ids` in :code:`token_ids`.
269 +
270 + Note that empty matches are ignored.
271 + """
272 + prompt_len = len(token_ids)
273 match_len = len(match_ids)
274
272 - last_end_idx = 0
273 - for start_idx in range(len(token_ids) - match_len + 1):
274 - if start_idx < last_end_idx:
275 - continue # Exclude overlapping matches
275 + if match_len == 0:
276 + return
277
278 + start_idx = 0
279 + while start_idx < prompt_len - match_len + 1:
280 end_idx = start_idx + match_len
281 +
282 if token_ids[start_idx:end_idx] == match_ids:
283 yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
280 - last_end_idx = end_idx
284 +
285 + # Exclude overlapping matches
286 + start_idx = end_idx
287 + else:
288 + start_idx += 1
289
290
283 -class _PromptReplacementMatch(ABC, Generic[_T, _S]):
284 - prompt_repl: _BoundPromptReplacement[_T]
291 +@dataclass(repr=False)
292 +class _PromptReplacementMatch(ABC):
293 + prompt_repl: _BoundPromptReplacement
294
295 @property
296 def modality(self) -> str:
@@ -297,19 +306,13 @@ class _PromptReplacementMatch(ABC, Generic[_T, _S]):
306 def end_idx(self) -> int:
307 raise NotImplementedError
308
300 - @property
301 - @abstractmethod
302 - def repl_unit(self) -> _S:
303 - raise NotImplementedError
304 -
309 def __repr__(self) -> str:
310 return (f"{type(self).__name__}(modality={self.modality!r}, "
311 f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
312
313
314 @dataclass(repr=False)
311 -class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
312 - prompt_repl: _BoundPromptReplacement[_T]
315 +class _PromptReplacementTokenMatch(_PromptReplacementMatch):
316 match: _TokenMatch
317
318 @property
@@ -320,14 +323,9 @@ class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
323 def end_idx(self) -> int:
324 return self.match.end_idx
325
323 - @property
324 - def repl_unit(self) -> list[int]:
325 - return self.prompt_repl.repl_unit.token_ids
326 -
326
327 @dataclass(repr=False)
329 -class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
330 - prompt_repl: _BoundPromptReplacement[_T]
328 +class _PromptReplacementTextMatch(_PromptReplacementMatch):
329 match: re.Match[str]
330
331 @property
@@ -338,20 +336,15 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
336 def end_idx(self) -> int:
337 return self.match.end()
338
341 - @property
342 - def repl_unit(self) -> str:
343 - return self.prompt_repl.repl_unit.text
344 -
339
340 class _PlaceholderInfo(NamedTuple):
341 modality: str
342 start_idx: int
349 - unit: list[int]
350 - unit_count: int
343 + replacement: list[int]
344
345 @property
346 def length(self) -> int:
354 - return len(self.unit) * self.unit_count
347 + return len(self.replacement)
348
349 def to_range(self) -> PlaceholderRange:
350 return PlaceholderRange(
@@ -362,8 +355,8 @@ class _PlaceholderInfo(NamedTuple):
355
356 def find_token_matches(
357 prompt: list[int],
365 - prompt_repls: Sequence[_BoundPromptReplacement[_T]],
366 -) -> list[_PromptReplacementTokenMatch[_T]]:
358 + prompt_repls: Sequence[_BoundPromptReplacement],
359 +) -> list[_PromptReplacementTokenMatch]:
360 """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
361 return [
362 _PromptReplacementTokenMatch(prompt_repl, match)
@@ -374,8 +367,8 @@ def find_token_matches(
367
368 def find_text_matches(
369 prompt: str,
377 - prompt_repls: Sequence[_BoundPromptReplacement[_T]],
378 -) -> list[_PromptReplacementTextMatch[_T]]:
370 + prompt_repls: Sequence[_BoundPromptReplacement],
371 +) -> list[_PromptReplacementTextMatch]:
372 """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
373 return [
374 _PromptReplacementTextMatch(prompt_repl, match)
@@ -385,15 +378,15 @@ def find_text_matches(
378
379
380 def _resolve_matches(
388 - prompt: _S,
389 - matches: Sequence[_PromptReplacementMatch[_T, _S]],
390 -) -> list[_PromptReplacementMatch[_T, _S]]:
381 + prompt: _PromptSeq,
382 + matches: Sequence[_PromptReplacementMatch],
383 +) -> list[_PromptReplacementMatch]:
384 """
385 Resolve :code:`matches` to ensure that there are no overlapping matches,
386 and sort them such that earlier matches take priority over later ones.
387 """
395 - seen_matches: list[Optional[_PromptReplacementMatch[_T, _S]]] \
396 - = [None] * len(prompt)
388 + seen_matches: list[Optional[_PromptReplacementMatch]] = [None
389 + ] * len(prompt)
390
391 for match in matches:
392 for idx in range(match.start_idx, match.end_idx):
@@ -409,30 +402,34 @@ def _resolve_matches(
402
403 def _replace_matches(
404 prompt: _S,
412 - matches: Sequence[_PromptReplacementMatch[_T, _S]],
413 - mm_items_by_modality: Mapping[str, list[_T]],
414 - hf_inputs: BatchFeature,
405 + matches: Sequence[_PromptReplacementMatch],
406 + mm_items: MultiModalDataItems,
407 ) -> list[_S]:
408 out_seqs = list[_S]()
409 prev_end_idx = 0
418 - next_idx_by_modality = {modality: 0 for modality in mm_items_by_modality}
410 + next_idx_by_modality = {modality: 0 for modality in mm_items}
411
412 for match in _resolve_matches(prompt, matches):
413 modality = match.modality
422 - mm_items = mm_items_by_modality[modality]
414 + modal_items = mm_items[modality]
415
416 item_idx = next_idx_by_modality[modality]
425 - if item_idx >= len(mm_items):
417 + if item_idx >= len(modal_items):
418 continue
419
420 start_idx = match.start_idx
421 end_idx = match.end_idx
430 - repl_unit = match.repl_unit
422 +
423 repl_info = match.prompt_repl
432 - repl_count = repl_info.get_count(mm_items, hf_inputs, item_idx)
424 + replacement = repl_info.get_replacement(item_idx)
425 +
426 + if isinstance(prompt, str):
427 + repl_seq = replacement.text
428 + out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
429 + else:
430 + repl_seq = replacement.token_ids
431 + out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
432
434 - out_seqs.append(prompt[prev_end_idx:start_idx] +
435 - repl_unit * repl_count)
433 prev_end_idx = end_idx
434 next_idx_by_modality[modality] += 1
435
@@ -443,92 +440,104 @@ def _replace_matches(
440
441 def replace_token_matches(
442 prompt: list[int],
446 - matches: Sequence[_PromptReplacementMatch[_T, list[int]]],
447 - mm_items_by_modality: Mapping[str, list[_T]],
448 - hf_inputs: BatchFeature,
443 + matches: Sequence[_PromptReplacementTokenMatch],
444 + mm_items: MultiModalDataItems,
445 ) -> list[int]:
446 """Apply :code:`prompt_repls` to :code:`prompt`."""
447 if not matches:
448 return prompt
449
454 - token_id_seqs = _replace_matches(
455 - prompt,
456 - matches,
457 - mm_items_by_modality,
458 - hf_inputs,
459 - )
450 + token_id_seqs = _replace_matches(prompt, matches, mm_items)
451
452 return flatten_2d_lists(token_id_seqs)
453
454
455 def replace_text_matches(
456 prompt: str,
466 - matches: Sequence[_PromptReplacementMatch[_T, str]],
467 - mm_items_by_modality: Mapping[str, list[_T]],
468 - hf_inputs: BatchFeature,
457 + matches: Sequence[_PromptReplacementTextMatch],
458 + mm_items: MultiModalDataItems,
459 ) -> str:
460 """Apply :code:`prompt_repls` to :code:`prompt`."""
461 if not matches:
462 return prompt
463
474 - texts = _replace_matches(
475 - prompt,
476 - matches,
477 - mm_items_by_modality,
478 - hf_inputs,
479 - )
464 + texts = _replace_matches(prompt, matches, mm_items)
465
466 return "".join(texts)
467
468
484 -def _merge_placeholder_matches(
485 - matches: Iterable[_PromptReplacementTokenMatch],
486 -) -> Iterable[_PromptReplacementTokenMatch]:
487 - current_match = None
469 +def _iter_modality_placeholders(
470 + prompt: list[int],
471 + modality: str,
472 + modality_repls: Sequence[_BoundPromptReplacement],
473 + modal_items: list[Any],
474 +) -> Iterable[_PlaceholderInfo]:
475 + if len(modal_items) == 0:
476 + return
477
489 - for match in sorted(matches, key=lambda x: x.start_idx):
490 - if current_match is None:
491 - current_match = match
492 - elif (current_match.prompt_repl == match.prompt_repl
493 - and current_match.end_idx == match.start_idx):
494 - current_match = _PromptReplacementTokenMatch(
495 - current_match.prompt_repl,
496 - match=_TokenMatch(current_match.start_idx, match.end_idx),
497 - )
498 - else:
499 - yield current_match
500 - current_match = match
478 + prompt_len = len(prompt)
479 + item_index = 0
480 +
481 + start_idx = 0
482 + while start_idx < prompt_len:
483 + found = False
484 +
485 + for repl_info in modality_repls:
486 + replacement = repl_info.get_replacement(item_index)
487 + repl_tokens = replacement.token_ids
488 + repl_len = len(repl_tokens)
489 + end_idx = start_idx + repl_len
490 +
491 + if repl_len == 0 or end_idx > prompt_len:
492 + continue
493
502 - if current_match is not None:
503 - yield current_match
494 + if prompt[start_idx:end_idx] == repl_tokens:
495 + yield _PlaceholderInfo(
496 + modality=modality,
497 + start_idx=start_idx,
498 + replacement=repl_tokens,
499 + )
500 +
501 + item_index += 1
502 + if item_index >= len(modal_items):
503 + return
504 +
505 + # Exclude overlapping matches
506 + start_idx = end_idx
507 + found = True
508 + break
509 +
510 + if not found:
511 + start_idx += 1
512
513
514 def iter_placeholders(
507 - prompt_repls: Sequence[_BoundPromptReplacement[Any]],
515 + prompt_repls: Sequence[_BoundPromptReplacement],
516 prompt: list[int],
509 - *,
510 - min_unit_count: int = 1,
517 + mm_items: MultiModalDataItems,
518 ) -> Iterable[_PlaceholderInfo]:
512 - """Yield each set of placeholder tokens found in :code:`token_ids`."""
513 - if min_unit_count <= 0:
514 - raise ValueError("`min_unit_count` must be a positive integer")
515 -
516 - matches = (_PromptReplacementTokenMatch(prompt_repl, match)
517 - for prompt_repl in prompt_repls
518 - if len(repl_unit := prompt_repl.repl_unit.token_ids) > 0
519 - for match in iter_token_matches(prompt, repl_unit))
520 -
521 - for match in _merge_placeholder_matches(matches):
522 - unit = match.repl_unit
523 - placeholder = _PlaceholderInfo(
524 - modality=match.modality,
525 - start_idx=match.start_idx,
526 - unit=unit,
527 - unit_count=(match.end_idx - match.start_idx) // len(unit),
528 - )
519 + """
520 + Yield each set of placeholder tokens found in :code:`prompt`.
521 +
522 + Note that empty matches are ignored.
523 + """
524 + repls_by_modality = dict(full_groupby_modality(prompt_repls))
525 +
526 + for modality, modal_items in mm_items.items():
527 + if modality in repls_by_modality:
528 + yield from _iter_modality_placeholders(
529 + prompt,
530 + modality,
531 + repls_by_modality[modality],
532 + modal_items,
533 + )
534 +
535
530 - if placeholder.unit_count >= min_unit_count:
531 - yield placeholder
536 +class ProcessorInputs(NamedTuple):
537 + """Keyword arguments to :meth:`BaseMultiModalProcessor`"""
538 + prompt_text: str
539 + mm_data: MultiModalDataDict
540 + mm_processor_kwargs: Mapping[str, object]
541
542
543 class BaseMultiModalProcessor(ABC):
@@ -536,52 +545,55 @@ class BaseMultiModalProcessor(ABC):
545 Abstract base class to process multi-modal inputs to be used in vLLM.
546 """
547
539 - def __init__(
540 - self,
541 - ctx: InputProcessingContext,
542 - metadata: MultiModalProcessingMetadata,
543 - ) -> None:
548 + def __init__(self, ctx: InputProcessingContext) -> None:
549 super().__init__()
550
551 self.ctx = ctx
547 - self.metadata = metadata
548 - self.init_mm_processor_kwargs = (ctx.model_config.mm_processor_kwargs
549 - or {})
552
551 - def _get_hf_processor(
553 + def __call__(
554 self,
553 - **mm_processor_kwargs: Mapping[str, object],
554 - ) -> ProcessorMixin:
555 - # by default, we won't pass any kwargs to the processor initialization
555 + prompt: str,
556 + mm_data: MultiModalDataDict,
557 + mm_processor_kwargs: Mapping[str, object],
558 + ) -> MultiModalInputsV2:
559 + return self.apply(prompt, mm_data, mm_processor_kwargs)
560 +
561 + def _get_hf_processor(self) -> ProcessorMixin:
562 + """
563 + Subclasses can add keyword arguments to this method to accept
564 + additional kwargs from model config or user inputs.
565 + """
566 return self.ctx.get_hf_processor()
567
568 def _get_tokenizer(self) -> AnyTokenizer:
569 return self.ctx.tokenizer
570
561 - def __call__(
571 + @abstractmethod
572 + def _get_prompt_replacements(
573 self,
563 - prompt: str,
564 - mm_data: MultiModalDataDict,
574 + mm_items: MultiModalDataItems,
575 + hf_inputs: BatchFeature,
576 mm_processor_kwargs: Mapping[str, object],
566 - ) -> MultiModalInputsV2:
567 - return self.apply(prompt, mm_data, mm_processor_kwargs)
577 + ) -> list[PromptReplacement]:
578 + """
579 + Given the original multi-modal items for this modality
580 + and HF-processed data, output the replacements to perform.
581 +
582 + Note:
583 + Even when the HF processor already performs replacement for us,
584 + we still use this replacement information to determine
585 + the placeholder token positions for each multi-modal item.
586 + """
587 + raise NotImplementedError
588
589 def _find_placeholders(
590 self,
571 - all_prompt_repls: Sequence[_BoundPromptReplacement[Any]],
591 + all_prompt_repls: Sequence[_BoundPromptReplacement],
592 new_token_ids: list[int],
573 - *,
574 - # To avoid false positives from multi-input when detecting
575 - # whether placeholder tokens have been inserted, in case
576 - # the target sequence is a subset of the replacement tokens
577 - min_unit_count: int = 16,
593 + mm_items: MultiModalDataItems,
594 ) -> list[_PlaceholderInfo]:
595 return list(
580 - iter_placeholders(
581 - all_prompt_repls,
582 - new_token_ids,
583 - min_unit_count=min_unit_count,
584 - ))
596 + iter_placeholders(all_prompt_repls, new_token_ids, mm_items))
597
598 def _apply_hf_processor(
599 self,
@@ -589,13 +601,7 @@ class BaseMultiModalProcessor(ABC):
601 mm_data: MultiModalDataDict,
602 mm_processor_kwargs: Mapping[str, object],
603 ) -> BatchFeature:
592 - # some mm_processor_kwargs may be used in processor initialization
593 - # instead of processor call
594 - processor_init_kwargs = {
595 - **self.init_mm_processor_kwargs,
596 - **mm_processor_kwargs,
597 - }
598 - hf_processor = self._get_hf_processor(**processor_init_kwargs)
604 + hf_processor = self._get_hf_processor(**mm_processor_kwargs)
605
606 processor_data = dict[str, Any]()
607 passthrough_data = dict[str, Any]()
@@ -615,11 +621,10 @@ class BaseMultiModalProcessor(ABC):
621 else:
622 processor_data[k] = v
623
618 - # filter mm_processor_kwargs used in processor call
619 - mm_processor_kwargs = resolve_mm_processor_kwargs(
620 - self.init_mm_processor_kwargs,
621 - cast(Dict[str, Any], mm_processor_kwargs),
624 + assert callable(hf_processor)
625 + mm_processor_kwargs = self.ctx.resolve_hf_processor_call_kwargs(
626 hf_processor,
627 + mm_processor_kwargs,
628 )
629
630 try:
@@ -642,26 +647,21 @@ class BaseMultiModalProcessor(ABC):
647
648 def _bind_prompt_replacements(
649 self,
645 - mm_data: MultiModalDataDict,
646 - ) -> list[_BoundPromptReplacement[Any]]:
650 + prompt_repls: list[PromptReplacement],
651 + ) -> list[_BoundPromptReplacement]:
652 tokenizer = self._get_tokenizer()
653
649 - return [
650 - prompt_repl.bind(modality, tokenizer)
651 - for modality, metadata in self.metadata.items()
652 - if modality in mm_data for prompt_repl in metadata.prompt_repls
653 - ]
654 + return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
655
656 def _apply_prompt_replacements(
657 self,
657 - mm_data: MultiModalDataDict,
658 + mm_items: MultiModalDataItems,
659 hf_inputs: BatchFeature,
660 token_ids: list[int],
660 - prompt_repls: Sequence[_BoundPromptReplacement[Any]],
661 + prompt_repls: Sequence[_BoundPromptReplacement],
662 ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
663 tokenizer = self._get_tokenizer()
664
664 - mm_items = to_multi_format(mm_data)
665 token_matches = find_token_matches(token_ids, prompt_repls)
666
667 # If the search text does not represent a special token,
@@ -682,7 +682,6 @@ class BaseMultiModalProcessor(ABC):
682 token_ids,
683 token_matches,
684 mm_items,
685 - hf_inputs,
685 )
686
687 text = _decode(tokenizer, token_ids)
@@ -695,13 +694,13 @@ class BaseMultiModalProcessor(ABC):
694 text,
695 text_matches,
696 mm_items,
698 - hf_inputs,
697 )
698
699 token_ids = _encode(tokenizer, text)
700 matched_repls = [match.prompt_repl for match in text_matches]
701
704 - placeholders = self._find_placeholders(matched_repls, token_ids)
702 + placeholders = self._find_placeholders(matched_repls, token_ids,
703 + mm_items)
704
705 return token_ids, text, placeholders
706
@@ -731,12 +730,16 @@ class BaseMultiModalProcessor(ABC):
730 prompt_ids, = hf_inputs.pop("input_ids").tolist()
731 mm_kwargs = MultiModalKwargs(hf_inputs)
732
734 - all_prompt_repls = self._bind_prompt_replacements(mm_data)
733 + mm_items = to_multi_format(mm_data)
734 + prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
735 + mm_processor_kwargs)
736 + all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
737
738 # If HF processor already inserts placeholder tokens,
739 # there is no need for us to insert them
740 all_placeholders = self._find_placeholders(all_prompt_repls,
739 - prompt_ids)
741 + prompt_ids, mm_items)
742 +
743 if all_placeholders:
744 prompt_text = _decode(tokenizer, prompt_ids)
745 else:
@@ -745,7 +748,7 @@ class BaseMultiModalProcessor(ABC):
748 prompt_text,
749 all_placeholders,
750 ) = self._apply_prompt_replacements(
748 - mm_data,
751 + mm_items,
752 hf_inputs,
753 prompt_ids,
754 all_prompt_repls,
@@ -765,13 +768,13 @@ class BaseMultiModalProcessor(ABC):
768 )
769
770 @abstractmethod
768 - def _get_dummy_mm_kwargs(
771 + def _get_dummy_mm_inputs(
772 self,
773 mm_counts: Mapping[str, int],
771 - ) -> MultiModalKwargs:
774 + ) -> ProcessorInputs:
775 """
773 - Build the input that corresponds to `mm_max_tokens` in
774 - :meth:`get_dummy_data`.
776 + Build the multi-modal portion of the input which, after processing,
777 + results in `mm_max_tokens` in :meth:`get_dummy_data`.
778 """
779 raise NotImplementedError
780
@@ -784,38 +787,41 @@ class BaseMultiModalProcessor(ABC):
787 # Avoid circular import
788 from vllm.sequence import SequenceData
789
787 - tokenizer = self._get_tokenizer()
788 -
789 - mm_placeholders = dict[str, _PlaceholderInfo]()
790 - offset = 0
791 -
792 - for modality, max_tokens in mm_max_tokens.items():
793 - if max_tokens == 0:
794 - continue
795 -
796 - metadata = self.metadata[modality]
797 - repl = metadata.prompt_repls[0].bind(modality, tokenizer)
798 - repl_token_ids = repl.repl_unit.token_ids
799 -
800 - placeholders = _PlaceholderInfo(
801 - modality=modality,
802 - start_idx=offset,
803 - unit=repl_token_ids,
804 - unit_count=max_tokens // len(repl_token_ids),
805 - )
806 -
807 - mm_placeholders[modality] = placeholders
808 - offset += placeholders.length
790 + processor_inputs = self._get_dummy_mm_inputs(mm_counts)
791 + mm_inputs = self.apply(*processor_inputs)
792 +
793 + prompt_token_ids = mm_inputs["prompt_token_ids"]
794 + placeholders_by_modality = mm_inputs["mm_placeholders"]
795 +
796 + total_placeholders_by_modality = dict[str, int]()
797 + for modality, placeholders in placeholders_by_modality.items():
798 + num_placeholders = sum(item["length"] for item in placeholders)
799 + max_tokens = mm_max_tokens[modality]
800 +
801 + if num_placeholders != max_tokens:
802 + logger.warning(
803 + "The processed dummy data has a total of %d placeholder "
804 + "tokens for the '%s' modality, which is not the expected "
805 + "%d tokens.", num_placeholders, modality, max_tokens)
806 +
807 + total_placeholders_by_modality[modality] = num_placeholders
808 +
809 + total_len = len(prompt_token_ids)
810 + if total_len > seq_len:
811 + logger.warning(
812 + "The context length (%d) of the model is too short "
813 + "to hold the multi-modal embeddings in the worst case "
814 + "(%d tokens in total, out of which %s are reserved for "
815 + "multi-modal embeddings). This may cause certain multi-modal "
816 + "inputs to fail during inference, even when the input text is "
817 + "short. To avoid this, you should increase `max_model_len`, "
818 + "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
819 + total_len, total_placeholders_by_modality)
820
810 - prompt_token_ids = flatten_2d_lists(
811 - [p.unit * p.unit_count for p in mm_placeholders.values()])
821 prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
822
823 return DummyData(
824 seq_data=SequenceData.from_seqs(prompt_token_ids),
816 - multi_modal_data=self._get_dummy_mm_kwargs(mm_counts),
817 - multi_modal_placeholders={
818 - modality: [p.to_range()]
819 - for modality, p in mm_placeholders.items()
820 - },
825 + multi_modal_data=mm_inputs["mm_kwargs"],
826 + multi_modal_placeholders=placeholders_by_modality,
827 )
vllm/multimodal/registry.py
+2 -2
@@ -299,9 +299,9 @@ class MultiModalRegistry:
299 """
300
301 def wrapper(model_cls: N) -> N:
302 - if model_cls in self._processor_factories:
302 + if self._processor_factories.contains(model_cls, strict=True):
303 logger.warning(
304 - "Model class %s already has an input mapper "
304 + "Model class %s already has a multi-modal processor "
305 "registered to %s. It is overwritten by the new one.",
306 model_cls, self)
307
vllm/utils.py
+9 -3
@@ -1370,8 +1370,8 @@ def supports_kw(
1370
1371
1372 def resolve_mm_processor_kwargs(
1373 - init_kwargs: Optional[Dict[str, Any]],
1374 - inference_kwargs: Optional[Dict[str, Any]],
1373 + init_kwargs: Optional[Mapping[str, object]],
1374 + inference_kwargs: Optional[Mapping[str, object]],
1375 callable: Callable[..., object],
1376 allow_var_kwargs: bool = False,
1377 ) -> Dict[str, Any]:
@@ -1405,7 +1405,7 @@ def resolve_mm_processor_kwargs(
1405
1406 def get_allowed_kwarg_only_overrides(
1407 callable: Callable[..., object],
1408 - overrides: Optional[Dict[str, Any]],
1408 + overrides: Optional[Mapping[str, object]],
1409 allow_var_kwargs: bool = False,
1410 ) -> Dict[str, Any]:
1411 """
@@ -1524,9 +1524,15 @@ class ClassRegistry(UserDict[Type[T], _V]):
1524 raise KeyError(key)
1525
1526 def __contains__(self, key: object) -> bool:
1527 + return self.contains(key)
1528 +
1529 + def contains(self, key: object, *, strict: bool = False) -> bool:
1530 if not isinstance(key, type):
1531 return False
1532
1533 + if strict:
1534 + return key in self.data
1535 +
1536 return any(cls in self.data for cls in key.mro())
1537
1538