@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4da1f667e9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Keep track of whether prompt replacements have been applied (#13215)

Cyrus Leung committed Feb 14, 2025 at 20:20 UTC 4da1f667e95e000f80889850bfc9471722f08af8
10 files changed +373 -329
vllm/model_executor/models/glm4v.py
+8
@@ -484,6 +484,14 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
484
485 class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
486
487 + def _hf_processor_applies_repl(
488 + self,
489 + prompt_text: str,
490 + mm_items: MultiModalDataItems,
491 + hf_processor_mm_kwargs: Mapping[str, object],
492 + ) -> bool:
493 + return False
494 +
495 def _get_mm_fields_config(
496 self,
497 hf_inputs: BatchFeature,
vllm/model_executor/models/llava.py
+1 -2
@@ -294,7 +294,7 @@ class PixtralHFMultiModalProcessor(
294 pixel_values = processed_outputs.get("pixel_values")
295 if pixel_values is not None:
296 # Before/after https://github.com/huggingface/transformers/pull/35122
297 - if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
297 + if Version(TRANSFORMERS_VERSION) <= Version("4.48.3"):
298 images = mm_data["images"]
299 assert isinstance(images, list)
300
@@ -819,7 +819,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
819 prompt_ids,
820 mm_item_counts,
821 )
822 -
822 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
823
824 mm_placeholder_ranges = {
vllm/model_executor/models/llava_onevision.py
+45 -12
@@ -299,36 +299,69 @@ class LlavaOnevisionMultiModalProcessor(
299 mm_kwargs=mm_kwargs,
300 )
301
302 + # LLaVA-OneVision processor doesn't support multiple videos
303 + # with different sizes when converting back to tensors
304 + # So, we process each component separately
305 + # NOTE: No prompt replacement is applied in this case
306 processor = self.info.get_hf_processor()
307 + image_token = processor.image_token
308 video_token = processor.video_token
309
305 - # LLaVA-OneVision processor doesn't support multiple videos
306 - # with different sizes when converting back to tensors
307 - text_image_outputs = super()._call_hf_processor(
310 + text_outputs = super()._call_hf_processor(
311 prompt=prompt,
309 - mm_data=mm_data,
312 + mm_data={},
313 mm_kwargs=mm_kwargs,
314 )
315
316 + images = mm_data.pop("images", [])
317 + assert isinstance(images, list)
318 + if images:
319 + processor_outputs = super()._call_hf_processor(
320 + prompt=image_token * len(images),
321 + mm_data={"images": images},
322 + mm_kwargs=mm_kwargs,
323 + )
324 + image_outputs = {
325 + k: v
326 + for k, v in processor_outputs.items()
327 + if k in ("pixel_values", "image_sizes")
328 + }
329 + else:
330 + image_outputs = {}
331 +
332 pixel_values_videos = []
333 for video in videos:
315 - item_processor_data = dict(prompt=video_token, videos=video)
316 -
334 item_outputs = super()._call_hf_processor(
318 - prompt=prompt,
319 - mm_data=item_processor_data,
335 + prompt=video_token,
336 + mm_data={"videos": video},
337 mm_kwargs=mm_kwargs,
338 )
339
323 - pixel_values_videos.append(
324 - item_outputs.pop("pixel_values_videos")[0])
340 + pixel_values_videos.append(item_outputs["pixel_values_videos"][0])
341 +
342 + video_outputs = {"pixel_values_videos": pixel_values_videos}
343
344 combined_outputs = dict(
327 - **text_image_outputs,
328 - pixel_values_videos=pixel_values_videos,
345 + text_outputs,
346 + **image_outputs,
347 + **video_outputs,
348 )
349 return BatchFeature(combined_outputs)
350
351 + def _hf_processor_applies_repl(
352 + self,
353 + prompt_text: str,
354 + mm_items: MultiModalDataItems,
355 + hf_processor_mm_kwargs: Mapping[str, object],
356 + ) -> bool:
357 + base_result = super()._hf_processor_applies_repl(
358 + prompt_text=prompt_text,
359 + mm_items=mm_items,
360 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
361 + )
362 +
363 + return base_result and mm_items.get_count("video", strict=False) == 0
364 +
365 def _get_prompt_replacements(
366 self,
367 mm_items: MultiModalDataItems,
vllm/model_executor/models/minicpmo.py
+51 -39
@@ -27,8 +27,8 @@ from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
27 Tuple, TypedDict, Union)
28
29 import torch
30 -import torch.types
30 from torch import nn
31 +from transformers import BatchFeature
32 from transformers.modeling_outputs import BaseModelOutputWithPast
33 from transformers.models.whisper.modeling_whisper import (
34 ACT2FN, WHISPER_ATTENTION_CLASSES, WhisperConfig, WhisperEncoder)
@@ -37,23 +37,21 @@ from vllm.attention import AttentionMetadata
37 from vllm.config import VllmConfig
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39 from vllm.multimodal.inputs import MultiModalFieldConfig
40 -from vllm.multimodal.parse import (ModalityData, ModalityDataItems,
41 - MultiModalDataItems, MultiModalDataParser,
42 - VideoItem)
43 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
44 - PromptReplacement)
40 +from vllm.multimodal.parse import (AudioItem, DictEmbeddingItems, ModalityData,
41 + ModalityDataItems, MultiModalDataItems,
42 + MultiModalDataParser)
43 +from vllm.multimodal.processing import PromptReplacement
44 from vllm.multimodal.profiling import ProcessorInputs
45 from vllm.sequence import IntermediateTensors
46
47 from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
49 - MiniCPMVEmbeddingItems, MiniCPMVMultiModalDataParser,
50 - MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo)
48 + MiniCPMVMultiModalDataParser,
49 + MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo,
50 + _minicpmv_field_config)
51 from .utils import AutoWeightsLoader, maybe_prefix
52
53 CPU_DEVICE = torch.device("cpu")
54
55 -MiniCPMOEmbeddingItems = MiniCPMVEmbeddingItems
56 -
55
56 class MiniCPMOAudioFeatureInputs(TypedDict):
57 type: Literal["audio_features"]
@@ -103,28 +101,49 @@ MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
101 MiniCPMOAudioEmbeddingInputs]
102
103
106 -class MiniCPMOAudioEmbeddingItems(MiniCPMOEmbeddingItems):
104 +def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
105 + audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
106 +
107 + return dict(
108 + **_minicpmv_field_config(hf_inputs),
109 + audio_features=MultiModalFieldConfig.flat_from_sizes(
110 + "audio", audio_num_slices),
111 + audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
112 + "audio", audio_num_slices),
113 + audio_num_slices=MultiModalFieldConfig.batched("audio"),
114 + audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
115 + audio_embeds=MultiModalFieldConfig.flat_from_sizes(
116 + "audio", audio_num_slices),
117 + )
118 +
119
108 - def __init__(self, data: Dict) -> None:
109 - super().__init__(data, "audio")
110 - audio_embeds = self.data.get("audio_embeds", None)
111 - if audio_embeds is None:
112 - raise ValueError("Incorrect type of video_embeds",
113 - "Got type: None")
114 - self.data["audio_embeds"] = audio_embeds
120 +class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
121
116 - def get(self, index: int) -> object:
117 - return self.data["audio_embeds"][index]
122 + def __init__(
123 + self,
124 + data: Mapping[str, torch.Tensor],
125 + fields_config: Mapping[str, MultiModalFieldConfig],
126 + ) -> None:
127 + super().__init__(
128 + data,
129 + modality="image",
130 + fields_config=fields_config,
131 + required_fields={"audio_embeds"},
132 + )
133
134
135 class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
136
137 def _parse_audio_data(
138 self,
124 - data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
139 + data: Union[dict[str, torch.Tensor], ModalityData[AudioItem]],
140 ) -> ModalityDataItems[Any, Any]:
141 if isinstance(data, dict):
127 - return MiniCPMOAudioEmbeddingItems(data)
142 + return MiniCPMOAudioEmbeddingItems(
143 + data,
144 + fields_config=_minicpmo_field_config(data),
145 + )
146 +
147 return super()._parse_audio_data(data)
148
149
@@ -167,6 +186,10 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
186 def get_max_audio_chunks_with_most_features(self) -> int:
187 return 30
188
189 + def get_max_audio_tokens(self) -> int:
190 + return self.get_max_audio_tokens_per_chunk(
191 + ) * self.get_max_audio_chunks_with_most_features()
192 +
193 def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
194 sampling_rate = self.get_default_audio_sampling_rate()
195 # exclude <audio> </audio>
@@ -194,7 +217,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
217 return num_frames
218
219
197 -class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder):
220 +class MiniCPMODummyInputsBuilder(
221 + MiniCPMVDummyInputsBuilder[MiniCPMOProcessingInfo]):
222
223 def get_dummy_processor_inputs(
224 self, seq_len: int, mm_counts: Mapping[str,
@@ -222,8 +246,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder):
246
247
248 class MiniCPMOMultiModalProcessor(
225 - MiniCPMVMultiModalProcessor,
226 - BaseMultiModalProcessor[MiniCPMOProcessingInfo]):
249 + MiniCPMVMultiModalProcessor[MiniCPMOProcessingInfo]):
250
251 def _get_data_parser(self) -> MultiModalDataParser:
252 return MiniCPMOMultiModalDataParser(
@@ -369,21 +392,10 @@ class MiniCPMOMultiModalProcessor(
392
393 def _get_mm_fields_config(
394 self,
372 - hf_inputs,
395 + hf_inputs: BatchFeature,
396 hf_processor_mm_kwargs: Mapping[str, object],
397 ) -> Mapping[str, MultiModalFieldConfig]:
375 - audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
376 -
377 - return dict(
378 - **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
379 - audio_features=MultiModalFieldConfig.flat_from_sizes(
380 - "audio", audio_num_slices),
381 - audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
382 - "audio", audio_num_slices),
383 - audio_num_slices=MultiModalFieldConfig.batched("audio"),
384 - audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
385 - audio_embeds=MultiModalFieldConfig.flat_from_sizes(
386 - "audio", audio_num_slices))
398 + return _minicpmo_field_config(hf_inputs)
399
400
401 class MultiModalProjector(nn.Module):
@@ -406,7 +418,7 @@ class MultiModalProjector(nn.Module):
418
419 class MiniCPMWhisperEncoderLayer(nn.Module):
420
409 - def __init__(self, config: WhisperConfig, layer_idx: int = None):
421 + def __init__(self, config: WhisperConfig, layer_idx: int):
422 super().__init__()
423 self.embed_dim = config.d_model
424 self.self_attn = WHISPER_ATTENTION_CLASSES[
vllm/model_executor/models/minicpmv.py
+99 -122
@@ -35,6 +35,7 @@ import torch.types
35 from PIL import Image
36 from torch import nn
37 from transformers import BatchFeature, PretrainedConfig
38 +from typing_extensions import TypeVar
39
40 from vllm.attention import AttentionMetadata
41 from vllm.config import VllmConfig
@@ -51,9 +52,10 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
53 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
54 MultiModalInputs, PlaceholderRange)
54 -from vllm.multimodal.parse import (ImageItem, ImageSize, ModalityData,
55 - ModalityDataItems, MultiModalDataItems,
56 - MultiModalDataParser, VideoItem)
55 +from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem, ImageSize,
56 + ModalityData, ModalityDataItems,
57 + MultiModalDataItems, MultiModalDataParser,
58 + VideoItem)
59 from vllm.multimodal.processing import (BaseMultiModalProcessor,
60 BaseProcessingInfo, PromptReplacement)
61 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -115,93 +117,6 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
117 MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
118 MiniCPMVImageEmbeddingInputs]
119
118 -
119 -class MiniCPMVEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
120 - dict[str, torch.Tensor]]):
121 -
122 - def __init__(self, data: Dict, modality: str) -> None:
123 - super().__init__(data, modality)
124 -
125 - def get_processor_data(self) -> Mapping[str, object]:
126 - return self.data
127 -
128 - def get_passthrough_data(self) -> Mapping[str, object]:
129 - return {}
130 -
131 - def get_count(self) -> int:
132 - return len(self.data[f"{self.modality}_embeds"])
133 -
134 - def get(self, index: int) -> Dict[str, torch.Tensor]:
135 - out = {}
136 - for k, v in self.data.items():
137 - out[k] = v[index]
138 - return out
139 -
140 -
141 -class MiniCPMVImageEmbeddingItems(MiniCPMVEmbeddingItems):
142 -
143 - def __init__(self, data: Dict) -> None:
144 - super().__init__(data, "image")
145 - image_embeds = self.data.get("image_embeds", None)
146 - image_sizes = self.data.get("image_sizes", None)
147 - if image_embeds is None:
148 - raise ValueError("In correct type of image_embeds",
149 - "Got type: None")
150 - if not isinstance(image_embeds[0], torch.Tensor):
151 - raise ValueError("In correct type of image_embeds",
152 - f"Got type: {type(image_embeds[0])}")
153 - if image_sizes is None:
154 - raise ValueError(
155 - "In correct type of image_sizes", "Got type: None."
156 - "If you're using `image_size_list`, "
157 - "please rename it to `image_sizes`")
158 - if len(image_embeds[0].shape) == 2:
159 - image_embeds = [image_embeds]
160 - image_sizes = [image_sizes]
161 - self.data["image_embeds"] = image_embeds
162 - self.data["image_sizes"] = image_sizes
163 -
164 - def get_image_size(self, index: int) -> ImageSize:
165 - image_size = self.data["image_sizes"][index]
166 - return ImageSize(width=image_size[0], height=image_size[1])
167 -
168 -
169 -class MiniCPMVVideoEmbeddingItems(MiniCPMVEmbeddingItems):
170 -
171 - def __init__(self, data: Dict) -> None:
172 - super().__init__(data, "video")
173 - video_embeds = self.data.get("video_embeds", None)
174 - image_sizes = self.data.get("image_sizes", None)
175 - num_frames = self.data.get("num_frames", None)
176 - if video_embeds is None:
177 - raise ValueError("In correct type of video_embeds",
178 - "Got type: None")
179 - if not isinstance(video_embeds[0], torch.Tensor):
180 - raise ValueError("In correct type of video_embeds",
181 - f"Got type: {type(video_embeds[0])}")
182 - if image_sizes is None:
183 - raise ValueError(
184 - "In correct type of image_sizes", "Got type: None."
185 - "If you're using `image_size_list`, "
186 - "please rename it to `image_sizes`")
187 - if num_frames is None:
188 - raise ValueError("In correct type of numframes", "Got type: None")
189 - if len(video_embeds[0].shape) == 2:
190 - video_embeds = [video_embeds]
191 - image_sizes = [image_sizes]
192 - num_frames = [num_frames]
193 - self.data["video_embeds"] = video_embeds
194 - self.data["image_sizes"] = image_sizes
195 - self.data["num_frames"] = num_frames
196 -
197 - def get_frame_size(self, index: int) -> ImageSize:
198 - frame_size = self.data["image_sizes"][index]
199 - return ImageSize(width=frame_size[0], height=frame_size[1])
200 -
201 - def get_num_frames(self, index: int) -> int:
202 - return self.data["num_frames"][index]
203 -
204 -
120 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
121
122
@@ -311,6 +226,71 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
226 return tuple(int(x) for x in version_str.split("."))
227
228
229 +def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
230 + image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
231 + video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
232 +
233 + return dict(
234 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
235 + "image", image_num_slices),
236 + image_sizes=MultiModalFieldConfig.batched("image"),
237 + tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
238 + "image", image_num_slices),
239 + image_num_slices=MultiModalFieldConfig.batched("image"),
240 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
241 + "image", image_num_slices),
242 + video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
243 + "video", video_num_slices),
244 + video_image_sizes=MultiModalFieldConfig.batched("video"),
245 + video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
246 + "video", video_num_slices),
247 + video_embeds=MultiModalFieldConfig.flat_from_sizes(
248 + "video", video_num_slices),
249 + video_num_slices=MultiModalFieldConfig.batched("video"),
250 + )
251 +
252 +
253 +class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
254 +
255 + def __init__(
256 + self,
257 + data: Mapping[str, torch.Tensor],
258 + fields_config: Mapping[str, MultiModalFieldConfig],
259 + ) -> None:
260 + super().__init__(
261 + data,
262 + modality="image",
263 + fields_config=fields_config,
264 + required_fields={"image_embeds", "image_sizes"},
265 + )
266 +
267 + def get_image_size(self, index: int) -> ImageSize:
268 + image_size = self.get(index)["image_sizes"].tolist()
269 + return ImageSize(width=image_size[0], height=image_size[1])
270 +
271 +
272 +class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
273 +
274 + def __init__(
275 + self,
276 + data: Mapping[str, torch.Tensor],
277 + fields_config: Mapping[str, MultiModalFieldConfig],
278 + ) -> None:
279 + super().__init__(
280 + data,
281 + modality="video",
282 + fields_config=fields_config,
283 + required_fields={"video_embeds", "video_image_sizes"},
284 + )
285 +
286 + def get_frame_size(self, index: int) -> ImageSize:
287 + frame_size = self.get(index)["video_image_sizes"].tolist()
288 + return ImageSize(width=frame_size[0], height=frame_size[1])
289 +
290 + def get_num_frames(self, index: int) -> int:
291 + return len(self.get(index)["video_image_sizes"])
292 +
293 +
294 class MiniCPMVMultiModalDataParser(MultiModalDataParser):
295
296 def _parse_image_data(
@@ -318,7 +298,11 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
298 data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
299 ) -> ModalityDataItems[Any, Any]:
300 if isinstance(data, dict):
321 - return MiniCPMVImageEmbeddingItems(data)
301 + return MiniCPMVImageEmbeddingItems(
302 + data,
303 + fields_config=_minicpmv_field_config(data),
304 + )
305 +
306 return super()._parse_image_data(data)
307
308 def _parse_video_data(
@@ -326,7 +310,11 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
310 data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
311 ) -> ModalityDataItems[Any, Any]:
312 if isinstance(data, dict):
329 - return MiniCPMVVideoEmbeddingItems(data)
313 + return MiniCPMVVideoEmbeddingItems(
314 + data,
315 + fields_config=_minicpmv_field_config(data),
316 + )
317 +
318 return super()._parse_video_data(data)
319
320
@@ -392,10 +380,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
380 return self.get_max_video_frame_tokens(
381 ) * self.get_num_frames_with_most_features(seq_len)
382
395 - def get_max_audio_tokens(self) -> int:
396 - return self.get_max_audio_tokens_per_chunk(
397 - ) * self.get_max_audio_chunks_with_most_features()
398 -
383 def get_slice_query_num(self) -> int:
384 hf_config = self.get_hf_config()
385 query_num = getattr(hf_config, "query_num", 64)
@@ -476,8 +460,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
460 return ImageSize(width=image_size, height=image_size * num_slices)
461
462
479 -class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
480 - ):
463 +_I = TypeVar("_I",
464 + bound=MiniCPMVProcessingInfo,
465 + default=MiniCPMVProcessingInfo)
466 +
467 +
468 +class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
469
470 def get_dummy_processor_inputs(
471 self,
@@ -514,8 +502,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
502 mm_data=mm_data)
503
504
517 -class MiniCPMVMultiModalProcessor(
518 - BaseMultiModalProcessor[MiniCPMVProcessingInfo]):
505 +class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
506
507 def _get_data_parser(self) -> MultiModalDataParser:
508 return MiniCPMVMultiModalDataParser()
@@ -675,7 +662,7 @@ class MiniCPMVMultiModalProcessor(
662 self.info.get_video_max_slice_num()
663 ) * inputs[modality]["num_frames"][index]
664 else:
678 - raise ValueError(f"UnExpected modality: {modality}")
665 + raise ValueError(f"Unexpected modality: {modality}")
666
667 def check_mm_inputs(self, inputs: Dict[str, object],
668 matches: List[str]) -> None:
@@ -700,7 +687,7 @@ class MiniCPMVMultiModalProcessor(
687 inputs["video"]["video_image_sizes"][index],
688 inputs["video"]["num_frames"][index])
689 else:
703 - raise ValueError(f"UnExpected modality: {modality}")
690 + raise ValueError(f"Unexpected modality: {modality}")
691
692 def call_base_hf_processor(
693 self,
@@ -742,6 +729,14 @@ class MiniCPMVMultiModalProcessor(
729 }
730 }
731
732 + def _hf_processor_applies_repl(
733 + self,
734 + prompt_text: str,
735 + mm_items: MultiModalDataItems,
736 + hf_processor_mm_kwargs: Mapping[str, object],
737 + ) -> bool:
738 + return False
739 +
740 def _get_prompt_replacements(
741 self, mm_items: MultiModalDataItems,
742 hf_processor_mm_kwargs: Mapping[str, Any],
@@ -770,28 +765,10 @@ class MiniCPMVMultiModalProcessor(
765
766 def _get_mm_fields_config(
767 self,
773 - hf_inputs,
768 + hf_inputs: BatchFeature,
769 hf_processor_mm_kwargs: Mapping[str, object],
770 ) -> Mapping[str, MultiModalFieldConfig]:
776 - image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
777 - video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
778 -
779 - return dict(pixel_values=MultiModalFieldConfig.flat_from_sizes(
780 - "image", image_num_slices),
781 - image_sizes=MultiModalFieldConfig.batched("image"),
782 - tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
783 - "image", image_num_slices),
784 - image_num_slices=MultiModalFieldConfig.batched("image"),
785 - image_embeds=MultiModalFieldConfig.flat_from_sizes(
786 - "image", image_num_slices),
787 - video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
788 - "video", video_num_slices),
789 - video_image_sizes=MultiModalFieldConfig.batched("video"),
790 - video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
791 - "video", video_num_slices),
792 - video_embeds=MultiModalFieldConfig.flat_from_sizes(
793 - "video", video_num_slices),
794 - video_num_slices=MultiModalFieldConfig.batched("video"))
771 + return _minicpmv_field_config(hf_inputs)
772
773 def apply(
774 self,
vllm/model_executor/models/qwen2_audio.py
-10
@@ -243,16 +243,6 @@ class Qwen2AudioMultiModalProcessor(
243 )
244 ]
245
246 - def _always_apply_prompt_replacements(self) -> bool:
247 - # Qwen2-Audio processor will start inserting placeholder tokens
248 - # in an upcoming release:
249 - # https://github.com/huggingface/transformers/pull/35534
250 - # NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
251 - # has already performed processing for multi-audio input when the input
252 - # audios are short (the corresponding placeholders may take up fewer
253 - # tokens than the number of audio items)
254 - return not hasattr(self.info.get_hf_processor(), "audio_token")
255 -
246
247 @MULTIMODAL_REGISTRY.register_processor(
248 Qwen2AudioMultiModalProcessor,
vllm/model_executor/models/qwen2_vl.py
+35 -65
@@ -58,8 +58,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
58 from vllm.multimodal.inputs import (ImageItem, ModalityData,
59 MultiModalFieldConfig, MultiModalKwargs,
60 VideoItem)
61 -from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
62 - MultiModalDataItems, MultiModalDataParser)
61 +from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
62 + ModalityDataItems, MultiModalDataItems,
63 + MultiModalDataParser)
64 from vllm.multimodal.processing import (BaseMultiModalProcessor,
65 BaseProcessingInfo, PromptReplacement)
66 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -657,49 +658,25 @@ class Qwen2VisionTransformer(nn.Module):
658 return loaded_params
659
660
660 -class Qwen2VLEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
661 - dict[str, torch.Tensor]]):
662 -
663 - def __init__(self, data: dict, modality: str) -> None:
664 - super().__init__(data, modality)
665 -
666 - grid_thw = data[f"{modality}_grid_thw"]
667 - slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
668 - self._slices = [
669 - slice(slice_idxs[i], slice_idxs[i + 1])
670 - for i in range(len(grid_thw))
671 - ]
672 -
673 - def get_count(self) -> int:
674 - return len(self.data[f"{self.modality}_grid_thw"])
675 -
676 - def get(self, index: int) -> dict[str, torch.Tensor]:
677 - out = {}
678 - for k, v in self.data.items():
679 - if v != f"{self.modality}_grid_thw":
680 - v = v[self._slices[index]]
681 -
682 - out[k] = v
683 -
684 - return out
685 -
686 - def get_processor_data(self) -> Mapping[str, object]:
687 - return {}
688 -
689 - def get_passthrough_data(self) -> Mapping[str, object]:
690 - return self.data
691 -
692 -
693 -class Qwen2VLImageEmbeddingItems(Qwen2VLEmbeddingItems):
694 -
695 - def __init__(self, data: dict) -> None:
696 - super().__init__(data, "image")
697 -
698 -
699 -class Qwen2VLVideoEmbeddingItems(Qwen2VLEmbeddingItems):
700 -
701 - def __init__(self, data: dict) -> None:
702 - super().__init__(data, "video")
661 +def _qwen2vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
662 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
663 + image_grid_sizes = image_grid_thw.prod(-1)
664 +
665 + video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
666 + video_grid_sizes = video_grid_thw.prod(-1)
667 +
668 + return dict(
669 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
670 + "image", image_grid_sizes),
671 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
672 + "image", image_grid_sizes),
673 + image_grid_thw=MultiModalFieldConfig.batched("image"),
674 + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
675 + "video", video_grid_sizes),
676 + video_embeds=MultiModalFieldConfig.flat_from_sizes(
677 + "video", video_grid_sizes),
678 + video_grid_thw=MultiModalFieldConfig.batched("video"),
679 + )
680
681
682 class Qwen2VLMultiModalDataParser(MultiModalDataParser):
@@ -709,7 +686,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
686 data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
687 ) -> ModalityDataItems[Any, Any]:
688 if isinstance(data, dict):
712 - return Qwen2VLEmbeddingItems(data, modality="image")
689 + return DictEmbeddingItems(
690 + data,
691 + modality="image",
692 + fields_config=_qwen2vl_field_config(data),
693 + required_fields={"image_embeds", "image_grid_thw"},
694 + )
695
696 return super()._parse_image_data(data)
697
@@ -718,7 +700,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
700 data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
701 ) -> ModalityDataItems[Any, Any]:
702 if isinstance(data, dict):
721 - return Qwen2VLEmbeddingItems(data, modality="video")
703 + return DictEmbeddingItems(
704 + data,
705 + modality="video",
706 + fields_config=_qwen2vl_field_config(data),
707 + required_fields={"video_embeds", "video_grid_thw"},
708 + )
709
710 return super()._parse_video_data(data)
711
@@ -999,24 +986,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
986 hf_inputs: BatchFeature,
987 hf_processor_mm_kwargs: Mapping[str, object],
988 ) -> Mapping[str, MultiModalFieldConfig]:
1002 - image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
1003 - image_grid_sizes = image_grid_thw.prod(-1)
1004 -
1005 - video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
1006 - video_grid_sizes = video_grid_thw.prod(-1)
1007 -
1008 - return dict(
1009 - pixel_values=MultiModalFieldConfig.flat_from_sizes(
1010 - "image", image_grid_sizes),
1011 - image_embeds=MultiModalFieldConfig.flat_from_sizes(
1012 - "image", image_grid_sizes),
1013 - image_grid_thw=MultiModalFieldConfig.batched("image"),
1014 - pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1015 - "video", video_grid_sizes),
1016 - video_embeds=MultiModalFieldConfig.flat_from_sizes(
1017 - "video", video_grid_sizes),
1018 - video_grid_thw=MultiModalFieldConfig.batched("video"),
1019 - )
989 + return _qwen2vl_field_config(hf_inputs)
990
991
992 @MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,
vllm/model_executor/models/qwen_vl.py
+9 -4
@@ -520,10 +520,7 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
520 return _get_tokenizer_without_image_pad(tokenizer)
521
522 def get_hf_processor(self) -> QwenVLProcessor:
523 - tokenizer = self.ctx.tokenizer
524 - assert isinstance(tokenizer, PreTrainedTokenizer)
525 -
526 - return QwenVLProcessor(self.get_hf_config(), tokenizer)
523 + return QwenVLProcessor(self.get_hf_config(), self.get_tokenizer())
524
525 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
526 return {"image": None}
@@ -605,6 +602,14 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
602 mm_kwargs=mm_kwargs,
603 )
604
605 + def _hf_processor_applies_repl(
606 + self,
607 + prompt_text: str,
608 + mm_items: MultiModalDataItems,
609 + hf_processor_mm_kwargs: Mapping[str, object],
610 + ) -> bool:
611 + return False
612 +
613 def _get_mm_fields_config(
614 self,
615 hf_inputs: BatchFeature,
vllm/multimodal/parse.py
+57 -1
@@ -9,13 +9,15 @@ from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar,
9 import numpy as np
10 import torch
11 from PIL.Image import Image
12 +from transformers import BatchFeature
13 from typing_extensions import TypeAlias, TypeGuard, assert_never
14
15 from vllm.utils import is_list_of
16
17 from .audio import resample_audio
18 from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
18 - ImageItem, ModalityData, MultiModalDataDict, VideoItem)
19 + ImageItem, ModalityData, MultiModalDataDict,
20 + MultiModalFieldConfig, MultiModalKwargs, VideoItem)
21
22 _T = TypeVar("_T")
23 _I = TypeVar("_I")
@@ -111,6 +113,60 @@ class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
113 return len(self.get(item_idx))
114
115
116 +class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
117 + Mapping[str, torch.Tensor]]):
118 + """
119 + Base class for data items that are expressed as a dictionary of tensors.
120 +
121 + Usually, the dictionary keys correspond to the outputs of HF processor.
122 + """
123 +
124 + def __init__(
125 + self,
126 + data: Mapping[str, torch.Tensor],
127 + modality: str,
128 + fields_config: Mapping[str, MultiModalFieldConfig],
129 + required_fields: set[str],
130 + ) -> None:
131 + super().__init__(data, modality)
132 +
133 + missing_required_fields = required_fields - fields_config.keys()
134 + if missing_required_fields:
135 + fields = set(fields_config.keys())
136 + msg = f"{required_fields=} should be a subset of {fields=}"
137 + raise ValueError(msg)
138 +
139 + missing_required_data_keys = required_fields - data.keys()
140 + if missing_required_data_keys:
141 + data_keys = set(data.keys())
142 + msg = (f"The data should contain the fields: {required_fields}, "
143 + f"but only found the following keys: {data_keys}")
144 + raise ValueError(msg)
145 +
146 + self.fields_config = fields_config
147 + self.required_fields = required_fields
148 +
149 + self._kwargs = MultiModalKwargs.from_hf_inputs(
150 + BatchFeature(dict(data)),
151 + fields_config,
152 + )
153 +
154 + def get_count(self) -> int:
155 + return self._kwargs.get_item_count(self.modality)
156 +
157 + def get(self, index: int) -> Mapping[str, torch.Tensor]:
158 + return {
159 + k: v.data
160 + for k, v in self._kwargs.get_item(self.modality, index).items()
161 + }
162 +
163 + def get_processor_data(self) -> Mapping[str, object]:
164 + return {}
165 +
166 + def get_passthrough_data(self) -> Mapping[str, object]:
167 + return self.data
168 +
169 +
170 class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
171
172 def __init__(self, data: Sequence[HfAudioItem]) -> None:
vllm/multimodal/processing.py
+68 -74
@@ -23,7 +23,8 @@ from .hasher import MultiModalHasher
23 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
24 MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
25 MultiModalKwargsItem, PlaceholderRange)
26 -from .parse import MultiModalDataItems, MultiModalDataParser
26 +from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
27 + MultiModalDataParser)
28
29 if TYPE_CHECKING:
30 from .profiling import BaseDummyInputsBuilder
@@ -830,15 +831,34 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
831 mm_kwargs,
832 )
833
834 + def _hf_processor_applies_repl(
835 + self,
836 + prompt_text: str,
837 + mm_items: MultiModalDataItems,
838 + hf_processor_mm_kwargs: Mapping[str, object],
839 + ) -> bool:
840 + """
841 + Return whether the HF processor applies prompt replacements.
842 +
843 + For most HF processors, this should be :code:`True` when multi-modal
844 + data items are passed, but :code:`False` when multi-modal embeddings
845 + are passed.
846 + """
847 + return not any(
848 + isinstance(items, (EmbeddingItems, DictEmbeddingItems))
849 + for items in mm_items.values())
850 +
851 def _apply_hf_processor_text_mm(
852 self,
853 prompt_text: str,
854 mm_items: MultiModalDataItems,
855 hf_processor_mm_kwargs: Mapping[str, object],
838 - ) -> tuple[list[int], MultiModalKwargs]:
856 + ) -> tuple[list[int], MultiModalKwargs, bool]:
857 """
858 Apply the HF processor on the prompt text and multi-modal data
859 together.
860 +
861 + In addition, return whether prompt replacements have been applied.
862 """
863 processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
864
@@ -856,7 +876,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
876 self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
877 )
878
859 - return prompt_ids, mm_kwargs
879 + is_repl_applied = self._hf_processor_applies_repl(
880 + prompt_text=prompt_text,
881 + mm_items=mm_items,
882 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
883 + )
884 +
885 + return prompt_ids, mm_kwargs, is_repl_applied
886
887 def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
888 """
@@ -866,7 +892,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
892 correspond to each other, we create dummy multi-modal items
893 to go along with the text.
894 """
869 - prompt_ids, _ = self._apply_hf_processor_text_mm(
895 + prompt_ids, _, _ = self._apply_hf_processor_text_mm(
896 prompt_text=prompt_text,
897 mm_items=MultiModalDataItems({}),
898 hf_processor_mm_kwargs={},
@@ -908,7 +934,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
934 mm_counts,
935 )
936
911 - _, mm_kwargs = self._apply_hf_processor_text_mm(
937 + _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
938 prompt_text=dummy_inputs.prompt_text,
939 mm_items=mm_items,
940 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -923,13 +949,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
949 hf_processor_mm_kwargs: Mapping[str, object],
950 *,
951 enable_hf_prompt_replacement: bool,
926 - ) -> tuple[list[int], MultiModalKwargs]:
952 + ) -> tuple[list[int], MultiModalKwargs, bool]:
953 """
954 Apply the HF processor on the prompt text and multi-modal data.
955
956 + In addition, return whether prompt replacements have been applied
957 + (for most HF processors, this should be :code:`True`).
958 +
959 Note:
931 - If :code:`enable_hf_prompt_replacement=False`, the prompt should
932 - correspond to the multi-modal items.
960 + If :code:`enable_hf_prompt_replacement=False`, we use HF processor
961 + to perform prompt replacement if available; HF processor requires
962 + that the prompt corresponds to multi-modal items.
963 """
964 if isinstance(prompt, str):
965 if enable_hf_prompt_replacement:
@@ -943,19 +973,19 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
973 else:
974 prompt_ids = self._apply_hf_processor_tokens_only(prompt)
975
946 - mm_missing_kwargs = self._apply_hf_processor_mm_only(
976 + mm_kwargs = self._apply_hf_processor_mm_only(
977 mm_items=mm_items,
978 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
979 )
980
951 - return prompt_ids, mm_missing_kwargs
981 + return prompt_ids, mm_kwargs, False
982
983 def _cached_apply_hf_processor(
984 self,
985 prompt: Union[str, list[int]],
986 mm_data_items: MultiModalDataItems,
987 hf_processor_mm_kwargs: Mapping[str, object],
958 - ) -> tuple[list[int], MultiModalKwargs]:
988 + ) -> tuple[list[int], MultiModalKwargs, bool]:
989 """
990 Apply the HF processor on the full prompt text,
991 caching the results and reusing cached results.
@@ -992,8 +1022,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1022 mm_missing_data_items = self._to_mm_items(mm_missing_data)
1023
1024 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
995 - # so we need to pass `enable_hf_prompt_replacement=False`
996 - prompt_ids, mm_missing_kwargs = self._apply_hf_processor_main(
1025 + # so we can't apply prompt replacements until the new multimodal
1026 + # items are combined with the cached multimodal items
1027 + (
1028 + prompt_ids,
1029 + mm_missing_kwargs,
1030 + is_repl_applied,
1031 + ) = self._apply_hf_processor_main(
1032 prompt=prompt,
1033 mm_items=mm_missing_data_items,
1034 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -1036,7 +1071,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1071
1072 mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1073
1039 - return prompt_ids, mm_kwargs
1074 + return prompt_ids, mm_kwargs, is_repl_applied
1075
1076 def _bind_and_group_repls(
1077 self,
@@ -1047,18 +1082,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1082 it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
1083 return dict(full_groupby_modality(it))
1084
1050 - def _always_apply_prompt_replacements(self) -> bool:
1051 - """
1052 - A flag which can be overridden so that
1053 - :meth:`_apply_prompt_replacements` is always called even if we
1054 - detect that HF has performed processing via
1055 - :meth:`_find_placeholders_by_modality`.
1056 -
1057 - This is useful in cases where :meth:`_find_placeholders_by_modality`
1058 - cannot be reliably used to detect whether HF has performed processing.
1059 - """
1060 - return False
1061 -
1085 def _apply_prompt_replacements(
1086 self,
1087 token_ids: list[int],
@@ -1155,29 +1178,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1178 self,
1179 mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1180 mm_item_counts: Mapping[str, int],
1158 - *,
1159 - allow_missing: bool = False,
1160 - ) -> Mapping[str, int]:
1161 - missing_repl_counts = dict[str, int]()
1162 -
1181 + ) -> None:
1182 for modality, item_count in mm_item_counts.items():
1183 placeholders = mm_placeholders.get(modality, [])
1184
1166 - if len(placeholders) != item_count and not allow_missing:
1185 + if len(placeholders) != item_count:
1186 raise RuntimeError(
1187 f"Expected there to be {item_count} prompt replacements "
1169 - f"corresponding to {item_count} {modality} items, but only "
1170 - f"found {len(placeholders)} prompt replacements! Either "
1171 - "the prompt text has missing/incorrect tokens for "
1188 + f"corresponding to {item_count} {modality} items, but "
1189 + f"instead found {len(placeholders)} prompt replacements! "
1190 + "Either the prompt text has missing/incorrect tokens for "
1191 "multi-modal inputs, or there is a problem with your "
1192 "implementation of merged multi-modal processor for this "
1193 "model (usually arising from an inconsistency between "
1194 "`_call_hf_processor` and `_get_prompt_replacements`).")
1195
1177 - missing_repl_counts[modality] = item_count - len(placeholders)
1178 -
1179 - return missing_repl_counts
1180 -
1196 def apply(
1197 self,
1198 prompt: Union[str, list[int]],
@@ -1217,7 +1232,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1232 else:
1233 mm_hashes = None
1234
1220 - prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1235 + (
1236 + prompt_ids,
1237 + mm_kwargs,
1238 + is_repl_applied,
1239 + ) = self._cached_apply_hf_processor(
1240 prompt,
1241 mm_items,
1242 hf_processor_mm_kwargs,
@@ -1233,52 +1252,27 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1252 mm_item_counts = mm_items.get_all_counts()
1253 self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1254
1236 - hf_mm_placeholders = self._find_mm_placeholders(
1237 - mm_prompt_repls,
1238 - prompt_ids,
1239 - mm_item_counts,
1240 - )
1241 -
1242 - if self._always_apply_prompt_replacements():
1243 - mm_missing_repl_counts = mm_item_counts
1244 - mm_missing_repls = dict(mm_prompt_repls)
1245 - else:
1246 - mm_missing_repl_counts = self._validate_mm_placeholders(
1247 - hf_mm_placeholders,
1255 + if is_repl_applied:
1256 + mm_placeholders = self._find_mm_placeholders(
1257 + mm_prompt_repls,
1258 + prompt_ids,
1259 mm_item_counts,
1249 - allow_missing=True,
1260 )
1261 + self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1262
1252 - mm_missing_repls = dict[str, list[BoundPromptReplacement]]()
1253 - for modality, missing_repl_count in mm_missing_repl_counts.items():
1254 - if missing_repl_count == 0:
1255 - mm_missing_repls[modality] = []
1256 - elif missing_repl_count == mm_item_counts.get(modality, 0):
1257 - mm_missing_repls[modality] = mm_prompt_repls[modality]
1258 - else:
1259 - raise ValueError("Partial prompt replacement within "
1260 - f"{modality=} is not supported")
1261 -
1262 - # If HF processor already inserts placeholder tokens,
1263 - # there is no need for us to insert them
1264 - if all(len(repls) == 0 for repls in mm_missing_repls.values()):
1263 tokenizer = self.info.get_tokenizer()
1264 prompt = decode_tokens(tokenizer, prompt_ids)
1267 - mm_placeholders = hf_mm_placeholders
1265 else:
1266 (
1267 prompt_ids,
1268 prompt,
1272 - missing_mm_placeholders,
1269 + mm_placeholders,
1270 ) = self._apply_prompt_replacements(
1271 prompt_ids,
1275 - mm_missing_repls,
1276 - mm_missing_repl_counts,
1272 + mm_prompt_repls,
1273 + mm_item_counts,
1274 )
1278 -
1279 - mm_placeholders = {**hf_mm_placeholders, **missing_mm_placeholders}
1280 -
1281 - self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1275 + self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1276
1277 mm_placeholder_ranges = {
1278 modality: [item.to_range() for item in placeholders]