A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Keep track of whether prompt replacements have been applied (#13215)
Cyrus Leung committed
Feb 14, 2025 at 20:20 UTC
4da1f667e95e000f80889850bfc9471722f08af8
10 files changed
+373
-329
vllm/model_executor/models/glm4v.py
+8
@@ -484,6 +484,14 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
484
485
class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
486
487
+ def _hf_processor_applies_repl(
488
+ self,
489
+ prompt_text: str,
490
+ mm_items: MultiModalDataItems,
491
+ hf_processor_mm_kwargs: Mapping[str, object],
492
+ ) -> bool:
493
+ return False
494
+
495
def _get_mm_fields_config(
496
self,
497
hf_inputs: BatchFeature,
vllm/model_executor/models/llava.py
+1
-2
@@ -294,7 +294,7 @@ class PixtralHFMultiModalProcessor(
294
pixel_values = processed_outputs.get("pixel_values")
295
if pixel_values is not None:
296
# Before/after https://github.com/huggingface/transformers/pull/35122
297
- if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
297
+ if Version(TRANSFORMERS_VERSION) <= Version("4.48.3"):
298
images = mm_data["images"]
299
assert isinstance(images, list)
300
@@ -819,7 +819,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
819
prompt_ids,
820
mm_item_counts,
821
)
822
-
822
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
823
824
mm_placeholder_ranges = {
vllm/model_executor/models/llava_onevision.py
+45
-12
@@ -299,36 +299,69 @@ class LlavaOnevisionMultiModalProcessor(
299
mm_kwargs=mm_kwargs,
300
)
301
302
+ # LLaVA-OneVision processor doesn't support multiple videos
303
+ # with different sizes when converting back to tensors
304
+ # So, we process each component separately
305
+ # NOTE: No prompt replacement is applied in this case
306
processor = self.info.get_hf_processor()
307
+ image_token = processor.image_token
308
video_token = processor.video_token
309
305
- # LLaVA-OneVision processor doesn't support multiple videos
306
- # with different sizes when converting back to tensors
307
- text_image_outputs = super()._call_hf_processor(
310
+ text_outputs = super()._call_hf_processor(
311
prompt=prompt,
309
- mm_data=mm_data,
312
+ mm_data={},
313
mm_kwargs=mm_kwargs,
314
)
315
316
+ images = mm_data.pop("images", [])
317
+ assert isinstance(images, list)
318
+ if images:
319
+ processor_outputs = super()._call_hf_processor(
320
+ prompt=image_token * len(images),
321
+ mm_data={"images": images},
322
+ mm_kwargs=mm_kwargs,
323
+ )
324
+ image_outputs = {
325
+ k: v
326
+ for k, v in processor_outputs.items()
327
+ if k in ("pixel_values", "image_sizes")
328
+ }
329
+ else:
330
+ image_outputs = {}
331
+
332
pixel_values_videos = []
333
for video in videos:
315
- item_processor_data = dict(prompt=video_token, videos=video)
316
-
334
item_outputs = super()._call_hf_processor(
318
- prompt=prompt,
319
- mm_data=item_processor_data,
335
+ prompt=video_token,
336
+ mm_data={"videos": video},
337
mm_kwargs=mm_kwargs,
338
)
339
323
- pixel_values_videos.append(
324
- item_outputs.pop("pixel_values_videos")[0])
340
+ pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
341
+
342
+ video_outputs = {"pixel_values_videos": pixel_values_videos}
343
344
combined_outputs = dict(
327
- **text_image_outputs,
328
- pixel_values_videos=pixel_values_videos,
345
+ text_outputs,
346
+ **image_outputs,
347
+ **video_outputs,
348
)
349
return BatchFeature(combined_outputs)
350
351
+ def _hf_processor_applies_repl(
352
+ self,
353
+ prompt_text: str,
354
+ mm_items: MultiModalDataItems,
355
+ hf_processor_mm_kwargs: Mapping[str, object],
356
+ ) -> bool:
357
+ base_result = super()._hf_processor_applies_repl(
358
+ prompt_text=prompt_text,
359
+ mm_items=mm_items,
360
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
361
+ )
362
+
363
+ return base_result and mm_items.get_count("video", strict=False) == 0
364
+
365
def _get_prompt_replacements(
366
self,
367
mm_items: MultiModalDataItems,
vllm/model_executor/models/minicpmo.py
+51
-39
@@ -27,8 +27,8 @@ from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
27
Tuple, TypedDict, Union)
28
29
import torch
30
-import torch.types
30
from torch import nn
31
+from transformers import BatchFeature
32
from transformers.modeling_outputs import BaseModelOutputWithPast
33
from transformers.models.whisper.modeling_whisper import (
34
ACT2FN, WHISPER_ATTENTION_CLASSES, WhisperConfig, WhisperEncoder)
@@ -37,23 +37,21 @@ from vllm.attention import AttentionMetadata
37
from vllm.config import VllmConfig
38
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39
from vllm.multimodal.inputs import MultiModalFieldConfig
40
-from vllm.multimodal.parse import (ModalityData, ModalityDataItems,
41
- MultiModalDataItems, MultiModalDataParser,
42
- VideoItem)
43
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
44
- PromptReplacement)
40
+from vllm.multimodal.parse import (AudioItem, DictEmbeddingItems, ModalityData,
41
+ ModalityDataItems, MultiModalDataItems,
42
+ MultiModalDataParser)
43
+from vllm.multimodal.processing import PromptReplacement
44
from vllm.multimodal.profiling import ProcessorInputs
45
from vllm.sequence import IntermediateTensors
46
47
from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
49
- MiniCPMVEmbeddingItems, MiniCPMVMultiModalDataParser,
50
- MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo)
48
+ MiniCPMVMultiModalDataParser,
49
+ MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo,
50
+ _minicpmv_field_config)
51
from .utils import AutoWeightsLoader, maybe_prefix
52
53
CPU_DEVICE = torch.device("cpu")
54
55
-MiniCPMOEmbeddingItems = MiniCPMVEmbeddingItems
56
-
55
56
class MiniCPMOAudioFeatureInputs(TypedDict):
57
type: Literal["audio_features"]
@@ -103,28 +101,49 @@ MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
101
MiniCPMOAudioEmbeddingInputs]
102
103
106
-class MiniCPMOAudioEmbeddingItems(MiniCPMOEmbeddingItems):
104
+def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
105
+ audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
106
+
107
+ return dict(
108
+ **_minicpmv_field_config(hf_inputs),
109
+ audio_features=MultiModalFieldConfig.flat_from_sizes(
110
+ "audio", audio_num_slices),
111
+ audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
112
+ "audio", audio_num_slices),
113
+ audio_num_slices=MultiModalFieldConfig.batched("audio"),
114
+ audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
115
+ audio_embeds=MultiModalFieldConfig.flat_from_sizes(
116
+ "audio", audio_num_slices),
117
+ )
118
+
119
108
- def __init__(self, data: Dict) -> None:
109
- super().__init__(data, "audio")
110
- audio_embeds = self.data.get("audio_embeds", None)
111
- if audio_embeds is None:
112
- raise ValueError("Incorrect type of video_embeds",
113
- "Got type: None")
114
- self.data["audio_embeds"] = audio_embeds
120
+class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
121
116
- def get(self, index: int) -> object:
117
- return self.data["audio_embeds"][index]
122
+ def __init__(
123
+ self,
124
+ data: Mapping[str, torch.Tensor],
125
+ fields_config: Mapping[str, MultiModalFieldConfig],
126
+ ) -> None:
127
+ super().__init__(
128
+ data,
129
+ modality="image",
130
+ fields_config=fields_config,
131
+ required_fields={"audio_embeds"},
132
+ )
133
134
135
class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
136
137
def _parse_audio_data(
138
self,
124
- data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
139
+ data: Union[dict[str, torch.Tensor], ModalityData[AudioItem]],
140
) -> ModalityDataItems[Any, Any]:
141
if isinstance(data, dict):
127
- return MiniCPMOAudioEmbeddingItems(data)
142
+ return MiniCPMOAudioEmbeddingItems(
143
+ data,
144
+ fields_config=_minicpmo_field_config(data),
145
+ )
146
+
147
return super()._parse_audio_data(data)
148
149
@@ -167,6 +186,10 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
186
def get_max_audio_chunks_with_most_features(self) -> int:
187
return 30
188
189
+ def get_max_audio_tokens(self) -> int:
190
+ return self.get_max_audio_tokens_per_chunk(
191
+ ) * self.get_max_audio_chunks_with_most_features()
192
+
193
def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
194
sampling_rate = self.get_default_audio_sampling_rate()
195
# exclude <audio> </audio>
@@ -194,7 +217,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
217
return num_frames
218
219
197
-class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder):
220
+class MiniCPMODummyInputsBuilder(
221
+ MiniCPMVDummyInputsBuilder[MiniCPMOProcessingInfo]):
222
223
def get_dummy_processor_inputs(
224
self, seq_len: int, mm_counts: Mapping[str,
@@ -222,8 +246,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder):
246
247
248
class MiniCPMOMultiModalProcessor(
225
- MiniCPMVMultiModalProcessor,
226
- BaseMultiModalProcessor[MiniCPMOProcessingInfo]):
249
+ MiniCPMVMultiModalProcessor[MiniCPMOProcessingInfo]):
250
251
def _get_data_parser(self) -> MultiModalDataParser:
252
return MiniCPMOMultiModalDataParser(
@@ -369,21 +392,10 @@ class MiniCPMOMultiModalProcessor(
392
393
def _get_mm_fields_config(
394
self,
372
- hf_inputs,
395
+ hf_inputs: BatchFeature,
396
hf_processor_mm_kwargs: Mapping[str, object],
397
) -> Mapping[str, MultiModalFieldConfig]:
375
- audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
376
-
377
- return dict(
378
- **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
379
- audio_features=MultiModalFieldConfig.flat_from_sizes(
380
- "audio", audio_num_slices),
381
- audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
382
- "audio", audio_num_slices),
383
- audio_num_slices=MultiModalFieldConfig.batched("audio"),
384
- audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
385
- audio_embeds=MultiModalFieldConfig.flat_from_sizes(
386
- "audio", audio_num_slices))
398
+ return _minicpmo_field_config(hf_inputs)
399
400
401
class MultiModalProjector(nn.Module):
@@ -406,7 +418,7 @@ class MultiModalProjector(nn.Module):
418
419
class MiniCPMWhisperEncoderLayer(nn.Module):
420
409
- def __init__(self, config: WhisperConfig, layer_idx: int = None):
421
+ def __init__(self, config: WhisperConfig, layer_idx: int):
422
super().__init__()
423
self.embed_dim = config.d_model
424
self.self_attn = WHISPER_ATTENTION_CLASSES[
vllm/model_executor/models/minicpmv.py
+99
-122
@@ -35,6 +35,7 @@ import torch.types
35
from PIL import Image
36
from torch import nn
37
from transformers import BatchFeature, PretrainedConfig
38
+from typing_extensions import TypeVar
39
40
from vllm.attention import AttentionMetadata
41
from vllm.config import VllmConfig
@@ -51,9 +52,10 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
53
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
54
MultiModalInputs, PlaceholderRange)
54
-from vllm.multimodal.parse import (ImageItem, ImageSize, ModalityData,
55
- ModalityDataItems, MultiModalDataItems,
56
- MultiModalDataParser, VideoItem)
55
+from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem, ImageSize,
56
+ ModalityData, ModalityDataItems,
57
+ MultiModalDataItems, MultiModalDataParser,
58
+ VideoItem)
59
from vllm.multimodal.processing import (BaseMultiModalProcessor,
60
BaseProcessingInfo, PromptReplacement)
61
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -115,93 +117,6 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
117
MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
118
MiniCPMVImageEmbeddingInputs]
119
118
-
119
-class MiniCPMVEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
120
- dict[str, torch.Tensor]]):
121
-
122
- def __init__(self, data: Dict, modality: str) -> None:
123
- super().__init__(data, modality)
124
-
125
- def get_processor_data(self) -> Mapping[str, object]:
126
- return self.data
127
-
128
- def get_passthrough_data(self) -> Mapping[str, object]:
129
- return {}
130
-
131
- def get_count(self) -> int:
132
- return len(self.data[f"{self.modality}_embeds"])
133
-
134
- def get(self, index: int) -> Dict[str, torch.Tensor]:
135
- out = {}
136
- for k, v in self.data.items():
137
- out[k] = v[index]
138
- return out
139
-
140
-
141
-class MiniCPMVImageEmbeddingItems(MiniCPMVEmbeddingItems):
142
-
143
- def __init__(self, data: Dict) -> None:
144
- super().__init__(data, "image")
145
- image_embeds = self.data.get("image_embeds", None)
146
- image_sizes = self.data.get("image_sizes", None)
147
- if image_embeds is None:
148
- raise ValueError("In correct type of image_embeds",
149
- "Got type: None")
150
- if not isinstance(image_embeds[0], torch.Tensor):
151
- raise ValueError("In correct type of image_embeds",
152
- f"Got type: {type(image_embeds[0])}")
153
- if image_sizes is None:
154
- raise ValueError(
155
- "In correct type of image_sizes", "Got type: None."
156
- "If you're using `image_size_list`, "
157
- "please rename it to `image_sizes`")
158
- if len(image_embeds[0].shape) == 2:
159
- image_embeds = [image_embeds]
160
- image_sizes = [image_sizes]
161
- self.data["image_embeds"] = image_embeds
162
- self.data["image_sizes"] = image_sizes
163
-
164
- def get_image_size(self, index: int) -> ImageSize:
165
- image_size = self.data["image_sizes"][index]
166
- return ImageSize(width=image_size[0], height=image_size[1])
167
-
168
-
169
-class MiniCPMVVideoEmbeddingItems(MiniCPMVEmbeddingItems):
170
-
171
- def __init__(self, data: Dict) -> None:
172
- super().__init__(data, "video")
173
- video_embeds = self.data.get("video_embeds", None)
174
- image_sizes = self.data.get("image_sizes", None)
175
- num_frames = self.data.get("num_frames", None)
176
- if video_embeds is None:
177
- raise ValueError("In correct type of video_embeds",
178
- "Got type: None")
179
- if not isinstance(video_embeds[0], torch.Tensor):
180
- raise ValueError("In correct type of video_embeds",
181
- f"Got type: {type(video_embeds[0])}")
182
- if image_sizes is None:
183
- raise ValueError(
184
- "In correct type of image_sizes", "Got type: None."
185
- "If you're using `image_size_list`, "
186
- "please rename it to `image_sizes`")
187
- if num_frames is None:
188
- raise ValueError("In correct type of numframes", "Got type: None")
189
- if len(video_embeds[0].shape) == 2:
190
- video_embeds = [video_embeds]
191
- image_sizes = [image_sizes]
192
- num_frames = [num_frames]
193
- self.data["video_embeds"] = video_embeds
194
- self.data["image_sizes"] = image_sizes
195
- self.data["num_frames"] = num_frames
196
-
197
- def get_frame_size(self, index: int) -> ImageSize:
198
- frame_size = self.data["image_sizes"][index]
199
- return ImageSize(width=frame_size[0], height=frame_size[1])
200
-
201
- def get_num_frames(self, index: int) -> int:
202
- return self.data["num_frames"][index]
203
-
204
-
120
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
121
122
@@ -311,6 +226,71 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
226
return tuple(int(x) for x in version_str.split("."))
227
228
229
+def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
230
+ image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
231
+ video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
232
+
233
+ return dict(
234
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
235
+ "image", image_num_slices),
236
+ image_sizes=MultiModalFieldConfig.batched("image"),
237
+ tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
238
+ "image", image_num_slices),
239
+ image_num_slices=MultiModalFieldConfig.batched("image"),
240
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
241
+ "image", image_num_slices),
242
+ video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
243
+ "video", video_num_slices),
244
+ video_image_sizes=MultiModalFieldConfig.batched("video"),
245
+ video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
246
+ "video", video_num_slices),
247
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
248
+ "video", video_num_slices),
249
+ video_num_slices=MultiModalFieldConfig.batched("video"),
250
+ )
251
+
252
+
253
+class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
254
+
255
+ def __init__(
256
+ self,
257
+ data: Mapping[str, torch.Tensor],
258
+ fields_config: Mapping[str, MultiModalFieldConfig],
259
+ ) -> None:
260
+ super().__init__(
261
+ data,
262
+ modality="image",
263
+ fields_config=fields_config,
264
+ required_fields={"image_embeds", "image_sizes"},
265
+ )
266
+
267
+ def get_image_size(self, index: int) -> ImageSize:
268
+ image_size = self.get(index)["image_sizes"].tolist()
269
+ return ImageSize(width=image_size[0], height=image_size[1])
270
+
271
+
272
+class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
273
+
274
+ def __init__(
275
+ self,
276
+ data: Mapping[str, torch.Tensor],
277
+ fields_config: Mapping[str, MultiModalFieldConfig],
278
+ ) -> None:
279
+ super().__init__(
280
+ data,
281
+ modality="video",
282
+ fields_config=fields_config,
283
+ required_fields={"video_embeds", "video_image_sizes"},
284
+ )
285
+
286
+ def get_frame_size(self, index: int) -> ImageSize:
287
+ frame_size = self.get(index)["video_image_sizes"].tolist()
288
+ return ImageSize(width=frame_size[0], height=frame_size[1])
289
+
290
+ def get_num_frames(self, index: int) -> int:
291
+ return len(self.get(index)["video_image_sizes"])
292
+
293
+
294
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
295
296
def _parse_image_data(
@@ -318,7 +298,11 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
298
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
299
) -> ModalityDataItems[Any, Any]:
300
if isinstance(data, dict):
321
- return MiniCPMVImageEmbeddingItems(data)
301
+ return MiniCPMVImageEmbeddingItems(
302
+ data,
303
+ fields_config=_minicpmv_field_config(data),
304
+ )
305
+
306
return super()._parse_image_data(data)
307
308
def _parse_video_data(
@@ -326,7 +310,11 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
310
data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
311
) -> ModalityDataItems[Any, Any]:
312
if isinstance(data, dict):
329
- return MiniCPMVVideoEmbeddingItems(data)
313
+ return MiniCPMVVideoEmbeddingItems(
314
+ data,
315
+ fields_config=_minicpmv_field_config(data),
316
+ )
317
+
318
return super()._parse_video_data(data)
319
320
@@ -392,10 +380,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
380
return self.get_max_video_frame_tokens(
381
) * self.get_num_frames_with_most_features(seq_len)
382
395
- def get_max_audio_tokens(self) -> int:
396
- return self.get_max_audio_tokens_per_chunk(
397
- ) * self.get_max_audio_chunks_with_most_features()
398
-
383
def get_slice_query_num(self) -> int:
384
hf_config = self.get_hf_config()
385
query_num = getattr(hf_config, "query_num", 64)
@@ -476,8 +460,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
460
return ImageSize(width=image_size, height=image_size * num_slices)
461
462
479
-class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
480
- ):
463
+_I = TypeVar("_I",
464
+ bound=MiniCPMVProcessingInfo,
465
+ default=MiniCPMVProcessingInfo)
466
+
467
+
468
+class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
469
470
def get_dummy_processor_inputs(
471
self,
@@ -514,8 +502,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
502
mm_data=mm_data)
503
504
517
-class MiniCPMVMultiModalProcessor(
518
- BaseMultiModalProcessor[MiniCPMVProcessingInfo]):
505
+class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
506
507
def _get_data_parser(self) -> MultiModalDataParser:
508
return MiniCPMVMultiModalDataParser()
@@ -675,7 +662,7 @@ class MiniCPMVMultiModalProcessor(
662
self.info.get_video_max_slice_num()
663
) * inputs[modality]["num_frames"][index]
664
else:
678
- raise ValueError(f"UnExpected modality: {modality}")
665
+ raise ValueError(f"Unexpected modality: {modality}")
666
667
def check_mm_inputs(self, inputs: Dict[str, object],
668
matches: List[str]) -> None:
@@ -700,7 +687,7 @@ class MiniCPMVMultiModalProcessor(
687
inputs["video"]["video_image_sizes"][index],
688
inputs["video"]["num_frames"][index])
689
else:
703
- raise ValueError(f"UnExpected modality: {modality}")
690
+ raise ValueError(f"Unexpected modality: {modality}")
691
692
def call_base_hf_processor(
693
self,
@@ -742,6 +729,14 @@ class MiniCPMVMultiModalProcessor(
729
}
730
}
731
732
+ def _hf_processor_applies_repl(
733
+ self,
734
+ prompt_text: str,
735
+ mm_items: MultiModalDataItems,
736
+ hf_processor_mm_kwargs: Mapping[str, object],
737
+ ) -> bool:
738
+ return False
739
+
740
def _get_prompt_replacements(
741
self, mm_items: MultiModalDataItems,
742
hf_processor_mm_kwargs: Mapping[str, Any],
@@ -770,28 +765,10 @@ class MiniCPMVMultiModalProcessor(
765
766
def _get_mm_fields_config(
767
self,
773
- hf_inputs,
768
+ hf_inputs: BatchFeature,
769
hf_processor_mm_kwargs: Mapping[str, object],
770
) -> Mapping[str, MultiModalFieldConfig]:
776
- image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
777
- video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
778
-
779
- return dict(pixel_values=MultiModalFieldConfig.flat_from_sizes(
780
- "image", image_num_slices),
781
- image_sizes=MultiModalFieldConfig.batched("image"),
782
- tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
783
- "image", image_num_slices),
784
- image_num_slices=MultiModalFieldConfig.batched("image"),
785
- image_embeds=MultiModalFieldConfig.flat_from_sizes(
786
- "image", image_num_slices),
787
- video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
788
- "video", video_num_slices),
789
- video_image_sizes=MultiModalFieldConfig.batched("video"),
790
- video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
791
- "video", video_num_slices),
792
- video_embeds=MultiModalFieldConfig.flat_from_sizes(
793
- "video", video_num_slices),
794
- video_num_slices=MultiModalFieldConfig.batched("video"))
771
+ return _minicpmv_field_config(hf_inputs)
772
773
def apply(
774
self,
vllm/model_executor/models/qwen2_audio.py
-10
@@ -243,16 +243,6 @@ class Qwen2AudioMultiModalProcessor(
243
)
244
]
245
246
- def _always_apply_prompt_replacements(self) -> bool:
247
- # Qwen2-Audio processor will start inserting placeholder tokens
248
- # in an upcoming release:
249
- # https://github.com/huggingface/transformers/pull/35534
250
- # NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
251
- # has already performed processing for multi-audio input when the input
252
- # audios are short (the corresponding placeholders may take up fewer
253
- # tokens than the number of audio items)
254
- return not hasattr(self.info.get_hf_processor(), "audio_token")
255
-
246
247
@MULTIMODAL_REGISTRY.register_processor(
248
Qwen2AudioMultiModalProcessor,
vllm/model_executor/models/qwen2_vl.py
+35
-65
@@ -58,8 +58,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
58
from vllm.multimodal.inputs import (ImageItem, ModalityData,
59
MultiModalFieldConfig, MultiModalKwargs,
60
VideoItem)
61
-from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
62
- MultiModalDataItems, MultiModalDataParser)
61
+from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
62
+ ModalityDataItems, MultiModalDataItems,
63
+ MultiModalDataParser)
64
from vllm.multimodal.processing import (BaseMultiModalProcessor,
65
BaseProcessingInfo, PromptReplacement)
66
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -657,49 +658,25 @@ class Qwen2VisionTransformer(nn.Module):
658
return loaded_params
659
660
660
-class Qwen2VLEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
661
- dict[str, torch.Tensor]]):
662
-
663
- def __init__(self, data: dict, modality: str) -> None:
664
- super().__init__(data, modality)
665
-
666
- grid_thw = data[f"{modality}_grid_thw"]
667
- slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
668
- self._slices = [
669
- slice(slice_idxs[i], slice_idxs[i + 1])
670
- for i in range(len(grid_thw))
671
- ]
672
-
673
- def get_count(self) -> int:
674
- return len(self.data[f"{self.modality}_grid_thw"])
675
-
676
- def get(self, index: int) -> dict[str, torch.Tensor]:
677
- out = {}
678
- for k, v in self.data.items():
679
- if v != f"{self.modality}_grid_thw":
680
- v = v[self._slices[index]]
681
-
682
- out[k] = v
683
-
684
- return out
685
-
686
- def get_processor_data(self) -> Mapping[str, object]:
687
- return {}
688
-
689
- def get_passthrough_data(self) -> Mapping[str, object]:
690
- return self.data
691
-
692
-
693
-class Qwen2VLImageEmbeddingItems(Qwen2VLEmbeddingItems):
694
-
695
- def __init__(self, data: dict) -> None:
696
- super().__init__(data, "image")
697
-
698
-
699
-class Qwen2VLVideoEmbeddingItems(Qwen2VLEmbeddingItems):
700
-
701
- def __init__(self, data: dict) -> None:
702
- super().__init__(data, "video")
661
+def _qwen2vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
662
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
663
+ image_grid_sizes = image_grid_thw.prod(-1)
664
+
665
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
666
+ video_grid_sizes = video_grid_thw.prod(-1)
667
+
668
+ return dict(
669
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
670
+ "image", image_grid_sizes),
671
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
672
+ "image", image_grid_sizes),
673
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
674
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
675
+ "video", video_grid_sizes),
676
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
677
+ "video", video_grid_sizes),
678
+ video_grid_thw=MultiModalFieldConfig.batched("video"),
679
+ )
680
681
682
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
@@ -709,7 +686,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
686
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
687
) -> ModalityDataItems[Any, Any]:
688
if isinstance(data, dict):
712
- return Qwen2VLEmbeddingItems(data, modality="image")
689
+ return DictEmbeddingItems(
690
+ data,
691
+ modality="image",
692
+ fields_config=_qwen2vl_field_config(data),
693
+ required_fields={"image_embeds", "image_grid_thw"},
694
+ )
695
696
return super()._parse_image_data(data)
697
@@ -718,7 +700,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
700
data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
701
) -> ModalityDataItems[Any, Any]:
702
if isinstance(data, dict):
721
- return Qwen2VLEmbeddingItems(data, modality="video")
703
+ return DictEmbeddingItems(
704
+ data,
705
+ modality="video",
706
+ fields_config=_qwen2vl_field_config(data),
707
+ required_fields={"video_embeds", "video_grid_thw"},
708
+ )
709
710
return super()._parse_video_data(data)
711
@@ -999,24 +986,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
986
hf_inputs: BatchFeature,
987
hf_processor_mm_kwargs: Mapping[str, object],
988
) -> Mapping[str, MultiModalFieldConfig]:
1002
- image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
1003
- image_grid_sizes = image_grid_thw.prod(-1)
1004
-
1005
- video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
1006
- video_grid_sizes = video_grid_thw.prod(-1)
1007
-
1008
- return dict(
1009
- pixel_values=MultiModalFieldConfig.flat_from_sizes(
1010
- "image", image_grid_sizes),
1011
- image_embeds=MultiModalFieldConfig.flat_from_sizes(
1012
- "image", image_grid_sizes),
1013
- image_grid_thw=MultiModalFieldConfig.batched("image"),
1014
- pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1015
- "video", video_grid_sizes),
1016
- video_embeds=MultiModalFieldConfig.flat_from_sizes(
1017
- "video", video_grid_sizes),
1018
- video_grid_thw=MultiModalFieldConfig.batched("video"),
1019
- )
989
+ return _qwen2vl_field_config(hf_inputs)
990
991
992
@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,
vllm/model_executor/models/qwen_vl.py
+9
-4
@@ -520,10 +520,7 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
520
return _get_tokenizer_without_image_pad(tokenizer)
521
522
def get_hf_processor(self) -> QwenVLProcessor:
523
- tokenizer = self.ctx.tokenizer
524
- assert isinstance(tokenizer, PreTrainedTokenizer)
525
-
526
- return QwenVLProcessor(self.get_hf_config(), tokenizer)
523
+ return QwenVLProcessor(self.get_hf_config(), self.get_tokenizer())
524
525
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
526
return {"image": None}
@@ -605,6 +602,14 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
602
mm_kwargs=mm_kwargs,
603
)
604
605
+ def _hf_processor_applies_repl(
606
+ self,
607
+ prompt_text: str,
608
+ mm_items: MultiModalDataItems,
609
+ hf_processor_mm_kwargs: Mapping[str, object],
610
+ ) -> bool:
611
+ return False
612
+
613
def _get_mm_fields_config(
614
self,
615
hf_inputs: BatchFeature,
vllm/multimodal/parse.py
+57
-1
@@ -9,13 +9,15 @@ from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar,
9
import numpy as np
10
import torch
11
from PIL.Image import Image
12
+from transformers import BatchFeature
13
from typing_extensions import TypeAlias, TypeGuard, assert_never
14
15
from vllm.utils import is_list_of
16
17
from .audio import resample_audio
18
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
18
- ImageItem, ModalityData, MultiModalDataDict, VideoItem)
19
+ ImageItem, ModalityData, MultiModalDataDict,
20
+ MultiModalFieldConfig, MultiModalKwargs, VideoItem)
21
22
_T = TypeVar("_T")
23
_I = TypeVar("_I")
@@ -111,6 +113,60 @@ class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
113
return len(self.get(item_idx))
114
115
116
+class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
117
+ Mapping[str, torch.Tensor]]):
118
+ """
119
+ Base class for data items that are expressed as a dictionary of tensors.
120
+
121
+ Usually, the dictionary keys correspond to the outputs of HF processor.
122
+ """
123
+
124
+ def __init__(
125
+ self,
126
+ data: Mapping[str, torch.Tensor],
127
+ modality: str,
128
+ fields_config: Mapping[str, MultiModalFieldConfig],
129
+ required_fields: set[str],
130
+ ) -> None:
131
+ super().__init__(data, modality)
132
+
133
+ missing_required_fields = required_fields - fields_config.keys()
134
+ if missing_required_fields:
135
+ fields = set(fields_config.keys())
136
+ msg = f"{required_fields=} should be a subset of {fields=}"
137
+ raise ValueError(msg)
138
+
139
+ missing_required_data_keys = required_fields - data.keys()
140
+ if missing_required_data_keys:
141
+ data_keys = set(data.keys())
142
+ msg = (f"The data should contain the fields: {required_fields}, "
143
+ f"but only found the following keys: {data_keys}")
144
+ raise ValueError(msg)
145
+
146
+ self.fields_config = fields_config
147
+ self.required_fields = required_fields
148
+
149
+ self._kwargs = MultiModalKwargs.from_hf_inputs(
150
+ BatchFeature(dict(data)),
151
+ fields_config,
152
+ )
153
+
154
+ def get_count(self) -> int:
155
+ return self._kwargs.get_item_count(self.modality)
156
+
157
+ def get(self, index: int) -> Mapping[str, torch.Tensor]:
158
+ return {
159
+ k: v.data
160
+ for k, v in self._kwargs.get_item(self.modality, index).items()
161
+ }
162
+
163
+ def get_processor_data(self) -> Mapping[str, object]:
164
+ return {}
165
+
166
+ def get_passthrough_data(self) -> Mapping[str, object]:
167
+ return self.data
168
+
169
+
170
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
171
172
def __init__(self, data: Sequence[HfAudioItem]) -> None:
vllm/multimodal/processing.py
+68
-74
@@ -23,7 +23,8 @@ from .hasher import MultiModalHasher
23
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
24
MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
25
MultiModalKwargsItem, PlaceholderRange)
26
-from .parse import MultiModalDataItems, MultiModalDataParser
26
+from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
27
+ MultiModalDataParser)
28
29
if TYPE_CHECKING:
30
from .profiling import BaseDummyInputsBuilder
@@ -830,15 +831,34 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
831
mm_kwargs,
832
)
833
834
+ def _hf_processor_applies_repl(
835
+ self,
836
+ prompt_text: str,
837
+ mm_items: MultiModalDataItems,
838
+ hf_processor_mm_kwargs: Mapping[str, object],
839
+ ) -> bool:
840
+ """
841
+ Return whether the HF processor applies prompt replacements.
842
+
843
+ For most HF processors, this should be :code:`True` when multi-modal
844
+ data items are passed, but :code:`False` when multi-modal embeddings
845
+ are passed.
846
+ """
847
+ return not any(
848
+ isinstance(items, (EmbeddingItems, DictEmbeddingItems))
849
+ for items in mm_items.values())
850
+
851
def _apply_hf_processor_text_mm(
852
self,
853
prompt_text: str,
854
mm_items: MultiModalDataItems,
855
hf_processor_mm_kwargs: Mapping[str, object],
838
- ) -> tuple[list[int], MultiModalKwargs]:
856
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
857
"""
858
Apply the HF processor on the prompt text and multi-modal data
859
together.
860
+
861
+ In addition, return whether prompt replacements have been applied.
862
"""
863
processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
864
@@ -856,7 +876,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
876
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
877
)
878
859
- return prompt_ids, mm_kwargs
879
+ is_repl_applied = self._hf_processor_applies_repl(
880
+ prompt_text=prompt_text,
881
+ mm_items=mm_items,
882
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
883
+ )
884
+
885
+ return prompt_ids, mm_kwargs, is_repl_applied
886
887
def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
888
"""
@@ -866,7 +892,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
892
correspond to each other, we create dummy multi-modal items
893
to go along with the text.
894
"""
869
- prompt_ids, _ = self._apply_hf_processor_text_mm(
895
+ prompt_ids, _, _ = self._apply_hf_processor_text_mm(
896
prompt_text=prompt_text,
897
mm_items=MultiModalDataItems({}),
898
hf_processor_mm_kwargs={},
@@ -908,7 +934,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
934
mm_counts,
935
)
936
911
- _, mm_kwargs = self._apply_hf_processor_text_mm(
937
+ _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
938
prompt_text=dummy_inputs.prompt_text,
939
mm_items=mm_items,
940
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -923,13 +949,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
949
hf_processor_mm_kwargs: Mapping[str, object],
950
*,
951
enable_hf_prompt_replacement: bool,
926
- ) -> tuple[list[int], MultiModalKwargs]:
952
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
953
"""
954
Apply the HF processor on the prompt text and multi-modal data.
955
956
+ In addition, return whether prompt replacements have been applied
957
+ (for most HF processors, this should be :code:`True`).
958
+
959
Note:
931
- If :code:`enable_hf_prompt_replacement=False`, the prompt should
932
- correspond to the multi-modal items.
960
+ If :code:`enable_hf_prompt_replacement=False`, we use HF processor
961
+ to perform prompt replacement if available; HF processor requires
962
+ that the prompt corresponds to multi-modal items.
963
"""
964
if isinstance(prompt, str):
965
if enable_hf_prompt_replacement:
@@ -943,19 +973,19 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
973
else:
974
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
975
946
- mm_missing_kwargs = self._apply_hf_processor_mm_only(
976
+ mm_kwargs = self._apply_hf_processor_mm_only(
977
mm_items=mm_items,
978
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
979
)
980
951
- return prompt_ids, mm_missing_kwargs
981
+ return prompt_ids, mm_kwargs, False
982
983
def _cached_apply_hf_processor(
984
self,
985
prompt: Union[str, list[int]],
986
mm_data_items: MultiModalDataItems,
987
hf_processor_mm_kwargs: Mapping[str, object],
958
- ) -> tuple[list[int], MultiModalKwargs]:
988
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
989
"""
990
Apply the HF processor on the full prompt text,
991
caching the results and reusing cached results.
@@ -992,8 +1022,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1022
mm_missing_data_items = self._to_mm_items(mm_missing_data)
1023
1024
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
995
- # so we need to pass `enable_hf_prompt_replacement=False`
996
- prompt_ids, mm_missing_kwargs = self._apply_hf_processor_main(
1025
+ # so we can't apply prompt replacements until the new multimodal
1026
+ # items are combined with the cached multimodal items
1027
+ (
1028
+ prompt_ids,
1029
+ mm_missing_kwargs,
1030
+ is_repl_applied,
1031
+ ) = self._apply_hf_processor_main(
1032
prompt=prompt,
1033
mm_items=mm_missing_data_items,
1034
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -1036,7 +1071,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1071
1072
mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1073
1039
- return prompt_ids, mm_kwargs
1074
+ return prompt_ids, mm_kwargs, is_repl_applied
1075
1076
def _bind_and_group_repls(
1077
self,
@@ -1047,18 +1082,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1082
it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
1083
return dict(full_groupby_modality(it))
1084
1050
- def _always_apply_prompt_replacements(self) -> bool:
1051
- """
1052
- A flag which can be overridden so that
1053
- :meth:`_apply_prompt_replacements` is always called even if we
1054
- detect that HF has performed processing via
1055
- :meth:`_find_placeholders_by_modality`.
1056
-
1057
- This is useful in cases where :meth:`_find_placeholders_by_modality`
1058
- cannot be reliably used to detect whether HF has performed processing.
1059
- """
1060
- return False
1061
-
1085
def _apply_prompt_replacements(
1086
self,
1087
token_ids: list[int],
@@ -1155,29 +1178,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1178
self,
1179
mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1180
mm_item_counts: Mapping[str, int],
1158
- *,
1159
- allow_missing: bool = False,
1160
- ) -> Mapping[str, int]:
1161
- missing_repl_counts = dict[str, int]()
1162
-
1181
+ ) -> None:
1182
for modality, item_count in mm_item_counts.items():
1183
placeholders = mm_placeholders.get(modality, [])
1184
1166
- if len(placeholders) != item_count and not allow_missing:
1185
+ if len(placeholders) != item_count:
1186
raise RuntimeError(
1187
f"Expected there to be {item_count} prompt replacements "
1169
- f"corresponding to {item_count} {modality} items, but only "
1170
- f"found {len(placeholders)} prompt replacements! Either "
1171
- "the prompt text has missing/incorrect tokens for "
1188
+ f"corresponding to {item_count} {modality} items, but "
1189
+ f"instead found {len(placeholders)} prompt replacements! "
1190
+ "Either the prompt text has missing/incorrect tokens for "
1191
"multi-modal inputs, or there is a problem with your "
1192
"implementation of merged multi-modal processor for this "
1193
"model (usually arising from an inconsistency between "
1194
"`_call_hf_processor` and `_get_prompt_replacements`).")
1195
1177
- missing_repl_counts[modality] = item_count - len(placeholders)
1178
-
1179
- return missing_repl_counts
1180
-
1196
def apply(
1197
self,
1198
prompt: Union[str, list[int]],
@@ -1217,7 +1232,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1232
else:
1233
mm_hashes = None
1234
1220
- prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1235
+ (
1236
+ prompt_ids,
1237
+ mm_kwargs,
1238
+ is_repl_applied,
1239
+ ) = self._cached_apply_hf_processor(
1240
prompt,
1241
mm_items,
1242
hf_processor_mm_kwargs,
@@ -1233,52 +1252,27 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1252
mm_item_counts = mm_items.get_all_counts()
1253
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1254
1236
- hf_mm_placeholders = self._find_mm_placeholders(
1237
- mm_prompt_repls,
1238
- prompt_ids,
1239
- mm_item_counts,
1240
- )
1241
-
1242
- if self._always_apply_prompt_replacements():
1243
- mm_missing_repl_counts = mm_item_counts
1244
- mm_missing_repls = dict(mm_prompt_repls)
1245
- else:
1246
- mm_missing_repl_counts = self._validate_mm_placeholders(
1247
- hf_mm_placeholders,
1255
+ if is_repl_applied:
1256
+ mm_placeholders = self._find_mm_placeholders(
1257
+ mm_prompt_repls,
1258
+ prompt_ids,
1259
mm_item_counts,
1249
- allow_missing=True,
1260
)
1261
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1262
1252
- mm_missing_repls = dict[str, list[BoundPromptReplacement]]()
1253
- for modality, missing_repl_count in mm_missing_repl_counts.items():
1254
- if missing_repl_count == 0:
1255
- mm_missing_repls[modality] = []
1256
- elif missing_repl_count == mm_item_counts.get(modality, 0):
1257
- mm_missing_repls[modality] = mm_prompt_repls[modality]
1258
- else:
1259
- raise ValueError("Partial prompt replacement within "
1260
- f"{modality=} is not supported")
1261
-
1262
- # If HF processor already inserts placeholder tokens,
1263
- # there is no need for us to insert them
1264
- if all(len(repls) == 0 for repls in mm_missing_repls.values()):
1263
tokenizer = self.info.get_tokenizer()
1264
prompt = decode_tokens(tokenizer, prompt_ids)
1267
- mm_placeholders = hf_mm_placeholders
1265
else:
1266
(
1267
prompt_ids,
1268
prompt,
1272
- missing_mm_placeholders,
1269
+ mm_placeholders,
1270
) = self._apply_prompt_replacements(
1271
prompt_ids,
1275
- mm_missing_repls,
1276
- mm_missing_repl_counts,
1272
+ mm_prompt_repls,
1273
+ mm_item_counts,
1274
)
1278
-
1279
- mm_placeholders = {**hf_mm_placeholders, **missing_mm_placeholders}
1280
-
1281
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1275
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1276
1277
mm_placeholder_ranges = {
1278
modality: [item.to_range() for item in placeholders]