A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Enable tokenized inputs for merged multi-modal processor (#11900)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 10, 2025 at 11:24 UTC
b844b99ad309b05f37b1acb5360c82be7b16281d
12 files changed
+207
-77
tests/multimodal/test_processing.py
+25
-6
@@ -649,7 +649,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
649
)
650
651
652
-def _test_processing_cache_correctness(
652
+def _test_processing_correctness(
653
model_id: str,
654
modalities: dict[str, bool],
655
hit_rate: float,
@@ -691,6 +691,7 @@ def _test_processing_cache_correctness(
691
baseline_processor = factories.build_processor(ctx, cache=None)
692
cached_processor = factories.build_processor(ctx, cache=cache)
693
dummy_inputs = baseline_processor.dummy_inputs
694
+ tokenizer = baseline_processor.info.get_tokenizer()
695
696
rng = np.random.RandomState(0)
697
@@ -747,7 +748,25 @@ def _test_processing_cache_correctness(
748
)
749
750
assert baseline_result == cached_result, (
750
- f"Failed ({batch_idx=}, {mm_data=})")
751
+ f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
752
+
753
+ baseline_tokenized_result = baseline_processor.apply(
754
+ tokenizer.encode(prompt),
755
+ mm_data=mm_data,
756
+ hf_processor_mm_kwargs={},
757
+ )
758
+
759
+ assert baseline_result == baseline_tokenized_result, (
760
+ f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
761
+
762
+ cached_tokenized_result = cached_processor.apply(
763
+ tokenizer.encode(prompt),
764
+ mm_data=mm_data,
765
+ hf_processor_mm_kwargs={},
766
+ )
767
+
768
+ assert cached_result == cached_tokenized_result, (
769
+ f"Failed ({batch_idx=}, {prompt=}, {mm_data=})")
770
771
772
# yapf: disable
@@ -771,14 +790,14 @@ def _test_processing_cache_correctness(
790
@pytest.mark.parametrize("num_batches", [32])
791
@pytest.mark.parametrize("simplify_rate", [1.0])
792
# yapf: enable
774
-def test_processing_cache_correctness(
793
+def test_processing_correctness(
794
model_id: str,
795
modalities: dict[str, bool],
796
hit_rate: float,
797
num_batches: int,
798
simplify_rate: float,
799
):
781
- _test_processing_cache_correctness(
800
+ _test_processing_correctness(
801
model_id,
802
modalities,
803
hit_rate=hit_rate,
@@ -795,7 +814,7 @@ def test_processing_cache_correctness(
814
@pytest.mark.parametrize("num_batches", [32])
815
@pytest.mark.parametrize("simplify_rate", [1.0])
816
# yapf: enable
798
-def test_processing_cache_correctness_phi3v(
817
+def test_processing_correctness_phi3v(
818
model_id: str,
819
modalities: dict[str, bool],
820
hit_rate: float,
@@ -809,7 +828,7 @@ def test_processing_cache_correctness_phi3v(
828
829
AutoImageProcessor.from_pretrained(model_id, trust_remote_code=True)
830
812
- _test_processing_cache_correctness(
831
+ _test_processing_correctness(
832
model_id,
833
modalities,
834
hit_rate=hit_rate,
vllm/inputs/data.py
+2
-2
@@ -44,13 +44,13 @@ class TokensPrompt(TypedDict):
44
45
multi_modal_data: NotRequired["MultiModalDataDict"]
46
"""
47
- DEPRECATED: Optional multi-modal data to pass to the model,
47
+ Optional multi-modal data to pass to the model,
48
if the model supports it.
49
"""
50
51
mm_processor_kwargs: NotRequired[Dict[str, Any]]
52
"""
53
- DEPRECATED: Optional multi-modal processor kwargs to be forwarded to the
53
+ Optional multi-modal processor kwargs to be forwarded to the
54
multimodal input mapper & processor. Note that if multiple modalities
55
have registered mappers etc for the model being considered, we attempt
56
to pass the mm_processor_kwargs to each of them.
vllm/inputs/preprocess.py
-4
@@ -279,10 +279,6 @@ class InputPreprocessor:
279
280
mm_processor = self.mm_registry.create_processor(
281
self.model_config, tokenizer)
282
- if isinstance(prompt, list):
283
- logger.warning("Passing `multi_modal_data` in TokensPrompt is"
284
- "deprecated and will be removed in a future update")
285
- prompt = tokenizer.decode(prompt)
282
if mm_processor_kwargs is None:
283
mm_processor_kwargs = {}
284
vllm/model_executor/models/blip2.py
+20
-2
@@ -441,6 +441,24 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
441
442
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
443
444
+ def _call_hf_processor(
445
+ self,
446
+ prompt: str,
447
+ mm_data: Mapping[str, object],
448
+ mm_kwargs: Mapping[str, object],
449
+ ) -> BatchFeature:
450
+ if not mm_data:
451
+ # HF processor always adds placeholders even when there's no image
452
+ tokenizer = self.info.get_tokenizer()
453
+ prompt_ids = tokenizer.encode(prompt)
454
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
455
+
456
+ return super()._call_hf_processor(
457
+ prompt=prompt,
458
+ mm_data=mm_data,
459
+ mm_kwargs=mm_kwargs,
460
+ )
461
+
462
def _get_mm_fields_config(
463
self,
464
hf_inputs: BatchFeature,
@@ -469,11 +487,11 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
487
488
def apply(
489
self,
472
- prompt_text: str,
490
+ prompt: Union[str, list[int]],
491
mm_data: MultiModalDataDict,
492
hf_processor_mm_kwargs: Mapping[str, object],
493
) -> MultiModalInputsV2:
476
- result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
494
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
495
496
# Only <image> tokens should be considered as placeholders,
497
# so we ignore the trailing bos_token
vllm/model_executor/models/chameleon.py
+30
-2
@@ -99,6 +99,34 @@ class ChameleonDummyInputsBuilder(
99
class ChameleonMultiModalProcessor(
100
BaseMultiModalProcessor[ChameleonProcessingInfo]):
101
102
+ def _call_hf_processor(
103
+ self,
104
+ prompt: str,
105
+ mm_data: Mapping[str, object],
106
+ mm_kwargs: Mapping[str, object],
107
+ ) -> BatchFeature:
108
+ if not mm_data:
109
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
110
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
111
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
112
+
113
+ return super()._call_hf_processor(
114
+ prompt=prompt,
115
+ mm_data=mm_data,
116
+ mm_kwargs=mm_kwargs,
117
+ )
118
+
119
+ def _apply_hf_processor_tokens_only(
120
+ self,
121
+ prompt_tokens: list[int],
122
+ ) -> list[int]:
123
+ # HF processor adds sep token for chat mode
124
+ tokenizer = self.info.get_tokenizer()
125
+ sep_token_id: int = \
126
+ tokenizer.vocab[tokenizer.sep_token] # type: ignore
127
+
128
+ return prompt_tokens + [sep_token_id]
129
+
130
def _get_mm_fields_config(
131
self,
132
hf_inputs: BatchFeature,
@@ -128,11 +156,11 @@ class ChameleonMultiModalProcessor(
156
157
def apply(
158
self,
131
- prompt_text: str,
159
+ prompt: Union[str, list[int]],
160
mm_data: MultiModalDataDict,
161
hf_processor_mm_kwargs: Mapping[str, object],
162
) -> MultiModalInputsV2:
135
- result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
163
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
164
165
# Only <image> tokens should be considered as placeholders,
166
# so we ignore the image_start_token and image_end_token
vllm/model_executor/models/fuyu.py
+15
-9
@@ -16,7 +16,7 @@
16
""" PyTorch Fuyu model."""
17
import math
18
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19
- TypedDict)
19
+ TypedDict, Union)
20
21
import torch
22
import torch.nn as nn
@@ -149,14 +149,10 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
149
mm_data: Mapping[str, object],
150
mm_kwargs: Mapping[str, object],
151
) -> BatchFeature:
152
-
152
if not mm_data:
153
# Avoid warning from HF logger for text-only input
155
- # Input_ids format: bos_token_id + prompt_token_ids + boa_token_id
156
- # Tokenizer won't add boa_token_id by default, we add it manually.
157
- tokenizer = self.info.get_tokenizer()
158
- boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
159
- prompt_ids = tokenizer.encode(prompt) + [boa_token_id]
154
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
155
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
156
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
157
158
processed_outputs = super()._call_hf_processor(
@@ -181,6 +177,16 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
177
178
return processed_outputs
179
180
+ def _apply_hf_processor_tokens_only(
181
+ self,
182
+ prompt_tokens: list[int],
183
+ ) -> list[int]:
184
+ # HF processor adds boa_token_id
185
+ tokenizer = self.info.get_tokenizer()
186
+ boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
187
+
188
+ return prompt_tokens + [boa_token_id]
189
+
190
def _get_mm_fields_config(
191
self,
192
hf_inputs: BatchFeature,
@@ -223,11 +229,11 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
229
230
def apply(
231
self,
226
- prompt_text: str,
232
+ prompt: Union[str, list[int]],
233
mm_data: MultiModalDataDict,
234
hf_processor_mm_kwargs: Mapping[str, object],
235
) -> MultiModalInputsV2:
230
- result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
236
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
237
238
# Only |SPEAKER| (image) tokens should be considered as placeholders,
239
# so we ignore the trailing bos_token_id
vllm/model_executor/models/interfaces.py
+4
-4
@@ -39,13 +39,13 @@ class SupportsMultiModal(Protocol):
39
40
The output embeddings must be one of the following formats:
41
42
- - A list or tuple of 2D tensors, where each tensor corresponds to
43
- each input multimodal data item (e.g, image).
42
+ - A list or tuple of 2D tensors, where each tensor corresponds to
43
+ each input multimodal data item (e.g, image).
44
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
45
46
Note:
47
- The returned multimodal embeddings must be in the same order as
48
- the appearances of their corresponding multimodal data item in the
47
+ The returned multimodal embeddings must be in the same order as
48
+ the appearances of their corresponding multimodal data item in the
49
input prompt.
50
"""
51
...
vllm/model_executor/models/llava.py
+4
-4
@@ -724,7 +724,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
724
725
def apply(
726
self,
727
- prompt_text: str,
727
+ prompt: Union[str, list[int]],
728
mm_data: MultiModalDataDict,
729
hf_processor_mm_kwargs: Mapping[str, object],
730
) -> MultiModalInputsV2:
@@ -737,7 +737,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
737
image_height=-1,
738
)
739
740
- result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
740
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
741
742
mm_items = self._to_mm_items(mm_data)
743
mm_item_counts = mm_items.get_all_counts()
@@ -760,7 +760,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
760
)
761
])
762
763
- prompt_ids, prompt_text, _ = self._apply_prompt_replacements(
763
+ prompt_ids, prompt, _ = self._apply_prompt_replacements(
764
result["prompt_token_ids"],
765
mantis_mm_repls,
766
mm_item_counts,
@@ -788,7 +788,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
788
789
return MultiModalInputsV2(
790
type="multimodal",
791
- prompt=prompt_text,
791
+ prompt=prompt,
792
prompt_token_ids=prompt_ids,
793
mm_kwargs=mm_kwargs,
794
mm_placeholders=mm_placeholder_ranges,
vllm/model_executor/models/phi3v.py
+2
-2
@@ -481,11 +481,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
481
482
def apply(
483
self,
484
- prompt_text: str,
484
+ prompt: Union[str, list[int]],
485
mm_data: MultiModalDataDict,
486
hf_processor_mm_kwargs: Mapping[str, object],
487
) -> MultiModalInputsV2:
488
- result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
488
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
489
490
# Only <|image|> tokens should be considered as placeholders,
491
# so we ignore the trailing bos_token_id
vllm/model_executor/models/ultravox.py
+12
-6
@@ -138,12 +138,8 @@ class UltravoxMultiModalProcessor(
138
) -> BatchFeature:
139
# Text-only input not supported in composite processor
140
if not mm_data:
141
- tokenizer = self.info.get_tokenizer()
142
-
143
- prompt_ids = tokenizer.encode(
144
- prompt,
145
- add_special_tokens=False, # type: ignore
146
- )
141
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
142
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
143
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
144
145
mm_data = dict(mm_data)
@@ -188,6 +184,16 @@ class UltravoxMultiModalProcessor(
184
)
185
return BatchFeature(combined_outputs)
186
187
+ def _apply_hf_processor_tokens_only(
188
+ self,
189
+ prompt_tokens: list[int],
190
+ ) -> list[int]:
191
+ # HF processor omits bos_token_id by setting add_special_tokens=False
192
+ tokenizer = self.info.get_tokenizer()
193
+ assert prompt_tokens[0] == tokenizer.bos_token_id
194
+
195
+ return prompt_tokens[1:]
196
+
197
def _get_mm_fields_config(
198
self,
199
hf_inputs: BatchFeature,
vllm/multimodal/processing.py
+92
-35
@@ -725,15 +725,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
725
mm_kwargs,
726
)
727
728
- def _apply_hf_processor(
728
+ def _apply_hf_processor_text_mm(
729
self,
730
prompt_text: str,
731
mm_items: MultiModalDataItems,
732
hf_processor_mm_kwargs: Mapping[str, object],
733
) -> tuple[list[int], MultiModalKwargs]:
734
"""
735
- Wrapper of :meth:`_call_hf_processor` that applies
736
- additional pre-processing and post-processing.
735
+ Apply the HF processor on the prompt text and multi-modal data
736
+ together.
737
"""
738
processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
739
@@ -753,40 +753,93 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
753
754
return prompt_ids, mm_kwargs
755
756
- def _apply_hf_processor_missing(
757
- self,
758
- prompt_text: str,
759
- mm_missing_data_items: MultiModalDataItems,
760
- hf_processor_mm_kwargs: Mapping[str, object],
761
- ):
756
+ def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
757
"""
763
- Apply the HF processor on the full prompt text, but only on the
764
- multi-modal data that are missing from the cache.
758
+ Apply the HF processor on the prompt text only.
759
766
- Note:
767
- We pass prompt text and multi-modal data into the HF processor
768
- in separate calls to avoid HF prompt replacement being done for
769
- cached items; instead, we rely on our own prompt replacement logic
770
- (:meth:`_get_prompt_replacements`) for the full text.
760
+ Since HF processor requires that text and multi-modal items
761
+ correspond to each other, we create dummy multi-modal items
762
+ to go along with the text.
763
"""
772
- mm_missing_counts = mm_missing_data_items.get_all_counts()
773
-
774
- prompt_ids, _ = self._apply_hf_processor(
764
+ prompt_ids, _ = self._apply_hf_processor_text_mm(
765
prompt_text=prompt_text,
766
mm_items=MultiModalDataItems({}),
767
hf_processor_mm_kwargs={},
768
)
769
780
- # Some HF processors (e.g. Qwen2-VL) expect corresponding
781
- # multi-modal tokens to be in the prompt text
770
+ return prompt_ids
771
+
772
+ def _apply_hf_processor_tokens_only(
773
+ self,
774
+ prompt_tokens: list[int],
775
+ ) -> list[int]:
776
+ """
777
+ Apply the HF processor on the prompt tokens only.
778
+
779
+ Most HF processors accept prompt text but not prompt tokens.
780
+ If the HF processor adds or removes tokens that are not related to
781
+ multi-modal data, you should override this method so it is consistent
782
+ with the output of :meth:`_apply_hf_processor_text_only` on the
783
+ corresponding text.
784
+ """
785
+ return prompt_tokens
786
+
787
+ def _apply_hf_processor_mm_only(
788
+ self,
789
+ mm_items: MultiModalDataItems,
790
+ hf_processor_mm_kwargs: Mapping[str, object],
791
+ ) -> MultiModalKwargs:
792
+ """
793
+ Apply the HF processor on the multi-modal data only.
794
+
795
+ Since HF processor requires that text and multi-modal items
796
+ correspond to each other, we generate dummy text using
797
+ :class:`DummyInputsBuilder` to go along with the multi-modal data.
798
+ """
799
+ mm_counts = mm_items.get_all_counts()
800
+
801
dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
802
self.info.ctx.model_config.max_model_len,
784
- mm_missing_counts,
803
+ mm_counts,
804
)
805
787
- _, mm_missing_kwargs = self._apply_hf_processor(
806
+ _, mm_kwargs = self._apply_hf_processor_text_mm(
807
prompt_text=dummy_inputs.prompt_text,
789
- mm_items=mm_missing_data_items,
808
+ mm_items=mm_items,
809
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
810
+ )
811
+
812
+ return mm_kwargs
813
+
814
+ def _apply_hf_processor_main(
815
+ self,
816
+ prompt: Union[str, list[int]],
817
+ mm_items: MultiModalDataItems,
818
+ hf_processor_mm_kwargs: Mapping[str, object],
819
+ *,
820
+ enable_hf_prompt_replacement: bool,
821
+ ) -> tuple[list[int], MultiModalKwargs]:
822
+ """
823
+ Apply the HF processor on the prompt text and multi-modal data.
824
+
825
+ Note:
826
+ If :code:`enable_hf_prompt_replacement=False`, the prompt should
827
+ correspond to the multi-modal items.
828
+ """
829
+ if isinstance(prompt, str):
830
+ if enable_hf_prompt_replacement:
831
+ return self._apply_hf_processor_text_mm(
832
+ prompt_text=prompt,
833
+ mm_items=mm_items,
834
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
835
+ )
836
+
837
+ prompt_ids = self._apply_hf_processor_text_only(prompt)
838
+ else:
839
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt)
840
+
841
+ mm_missing_kwargs = self._apply_hf_processor_mm_only(
842
+ mm_items=mm_items,
843
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
844
)
845
@@ -794,7 +847,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
847
848
def _cached_apply_hf_processor(
849
self,
797
- prompt_text: str,
850
+ prompt: Union[str, list[int]],
851
mm_data_items: MultiModalDataItems,
852
hf_processor_mm_kwargs: Mapping[str, object],
853
) -> tuple[list[int], MultiModalKwargs]:
@@ -807,10 +860,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
860
861
_, passthrough_data = self._get_hf_mm_data(mm_data_items)
862
if cache is None or passthrough_data:
810
- return self._apply_hf_processor(
811
- prompt_text=prompt_text,
863
+ return self._apply_hf_processor_main(
864
+ prompt=prompt,
865
mm_items=mm_data_items,
866
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
867
+ enable_hf_prompt_replacement=True,
868
)
869
870
mm_maybe_cached_kw_items = {
@@ -832,10 +886,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
886
}
887
mm_missing_data_items = self._to_mm_items(mm_missing_data)
888
835
- prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
836
- prompt_text=prompt_text,
837
- mm_missing_data_items=mm_missing_data_items,
889
+ # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
890
+ # so we need to pass `enable_hf_prompt_replacement=False`
891
+ prompt_ids, mm_missing_kwargs = self._apply_hf_processor_main(
892
+ prompt=prompt,
893
+ mm_items=mm_missing_data_items,
894
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
895
+ enable_hf_prompt_replacement=False,
896
)
897
898
mm_missing_next_idx = {
@@ -1018,7 +1075,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1075
1076
def apply(
1077
self,
1021
- prompt_text: str,
1078
+ prompt: Union[str, list[int]],
1079
mm_data: MultiModalDataDict,
1080
hf_processor_mm_kwargs: Mapping[str, object],
1081
) -> MultiModalInputsV2:
@@ -1056,7 +1113,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1113
mm_hashes = None
1114
1115
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1059
- prompt_text,
1116
+ prompt,
1117
mm_items,
1118
hf_processor_mm_kwargs,
1119
)
@@ -1101,12 +1158,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1158
# there is no need for us to insert them
1159
if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1160
tokenizer = self.info.get_tokenizer()
1104
- prompt_text = decode_tokens(tokenizer, prompt_ids)
1161
+ prompt = decode_tokens(tokenizer, prompt_ids)
1162
mm_placeholders = hf_mm_placeholders
1163
else:
1164
(
1165
prompt_ids,
1109
- prompt_text,
1166
+ prompt,
1167
missing_mm_placeholders,
1168
) = self._apply_prompt_replacements(
1169
prompt_ids,
@@ -1125,7 +1182,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1182
1183
return MultiModalInputsV2(
1184
type="multimodal",
1128
- prompt=prompt_text,
1185
+ prompt=prompt,
1186
prompt_token_ids=prompt_ids,
1187
mm_kwargs=mm_kwargs,
1188
mm_hashes=mm_hashes,
vllm/multimodal/profiling.py
+1
-1
@@ -137,7 +137,7 @@ class MultiModalProfiler(Generic[_I]):
137
seq_len, mm_counts)
138
139
return self.processor.apply(
140
- prompt_text=processor_inputs.prompt_text,
140
+ prompt=processor_inputs.prompt_text,
141
mm_data=processor_inputs.mm_data,
142
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
143
)