A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Model] MiniCPM-V/O supports V1 (#15487)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 27, 2025 at 21:07 UTC
ac5bc615b0adac4038e5574b446c8ac64c241caf
4 files changed
+577
-598
docs/source/models/supported_models.md
+2
-2
@@ -836,14 +836,14 @@ See [this page](#generative-models) for more information on how to use generativ
836
* `openbmb/MiniCPM-o-2_6`, etc.
837
* ✅︎
838
* ✅︎
839
- *
839
+ * ✅︎
840
- * `MiniCPMV`
841
* MiniCPM-V
842
* T + I<sup>E+</sup> + V<sup>E+</sup>
843
* `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, etc.
844
* ✅︎
845
* ✅︎
846
- *
846
+ * ✅︎
847
- * `MllamaForConditionalGeneration`
848
* Llama 3.2
849
* T + I<sup>+</sup>
vllm/model_executor/models/minicpmo.py
+214
-211
@@ -23,8 +23,8 @@
23
# limitations under the License.
24
"""Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25
from collections.abc import Iterable, Mapping, Sequence
26
-from typing import (Any, Callable, Dict, Literal, Optional, Set, Tuple,
27
- TypedDict, Union)
26
+from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,
27
+ Union)
28
29
import torch
30
from torch import nn
@@ -42,8 +42,6 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
42
MultiModalDataParser)
43
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
44
from vllm.multimodal.profiling import ProcessorInputs
45
-from vllm.sequence import IntermediateTensors
46
-from vllm.utils import flatten_2d_lists
45
46
from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
47
MiniCPMVMultiModalDataParser,
@@ -51,13 +49,14 @@ from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
49
_minicpmv_field_config)
50
from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn,
51
maybe_prefix)
52
+from .vision import scatter_patch_features
53
54
CPU_DEVICE = torch.device("cpu")
55
56
57
class MiniCPMOAudioFeatureInputs(TypedDict):
58
type: Literal["audio_features"]
60
- audio_features: torch.Tensor
59
+ audio_features: Union[torch.Tensor, list[torch.Tensor]]
60
"""
61
Shape: `(batch_size * num_audios * num_slices, num_channels, length)`
62
Slice here means chunk. Audio that is too long will be split into slices,
@@ -65,37 +64,40 @@ class MiniCPMOAudioFeatureInputs(TypedDict):
64
Padding is used therefore `audio_features` is `torch.Tensor`.
65
"""
66
68
- audio_feature_lens: torch.Tensor
67
+ audio_feature_lens: Union[torch.Tensor, list[torch.Tensor]]
68
"""
70
- Shape: `(batch_size * num_audios * num_slices)`
69
+ Shape: `(batch_size * num_audios, num_slices)`
70
71
This should be feature length of each audio slice,
72
which equals to `audio_features.shape[-1]`
73
"""
74
76
- audio_bounds: torch.Tensor
75
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
76
"""
78
- Shape: `(batch_size * num_audios * num_slices, 2)`
77
+ A boolean mask indicating which audio embeddings correspond
78
+ to patch tokens.
79
80
- This should be in `(start, stop)` format.
80
+ Shape: `(batch_size * num_audios, num_embeds)`
81
"""
82
83
84
class MiniCPMOAudioEmbeddingInputs(TypedDict):
85
type: Literal["audio_embeds"]
86
- audio_embeds: torch.Tensor
86
+ audio_embeds: Union[torch.Tensor, list[torch.Tensor]]
87
"""
88
- Shape: `(batch_size * num_images * num_slices, hidden_size)`
88
+ Shape: `(batch_size * num_audios, num_slices, hidden_size)`
89
90
`hidden_size` must match the hidden size of language model backbone.
91
instead of a batched tensor.
92
Length of each slice may vary, so pass it as a list.
93
"""
94
- audio_bounds: torch.Tensor
94
+
95
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
96
"""
96
- Shape: `(batch_size * num_audios * num_slices, 2)`
97
+ A boolean mask indicating which audio embeddings correspond
98
+ to patch tokens.
99
98
- This should be in `(start, stop)` format.
100
+ Shape: `(batch_size * num_audios, num_embeds)`
101
"""
102
103
@@ -104,11 +106,16 @@ MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
106
107
108
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
109
+ audio_features = hf_inputs.get("audio_features", torch.empty(0))
110
+ num_audios = len(audio_features)
111
+
112
return dict(
113
**_minicpmv_field_config(hf_inputs),
114
audio_features=MultiModalFieldConfig.batched("audio"),
115
audio_feature_lens=MultiModalFieldConfig.batched("audio"),
116
audio_embeds=MultiModalFieldConfig.batched("audio"),
117
+ audio_embed_is_patch=MultiModalFieldConfig.batched("audio"),
118
+ audio_token_id=MultiModalFieldConfig.shared("audio", num_audios),
119
)
120
121
@@ -149,7 +156,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
156
audio_pattern = "(<audio>./</audio>)"
157
158
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
152
- return {"image": None, "video": None, "audio": None}
159
+ return {**super().get_supported_mm_limits(), "audio": None}
160
161
def get_mm_max_tokens_per_item(
162
self,
@@ -157,11 +164,25 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
164
mm_counts: Mapping[str, int],
165
) -> Mapping[str, int]:
166
return {
160
- "image": self.get_max_image_tokens(),
161
- "audio": self.get_max_audio_tokens(),
162
- "video": self.get_max_video_tokens(seq_len),
167
+ **super().get_mm_max_tokens_per_item(seq_len, mm_counts),
168
+ "audio":
169
+ self.get_max_audio_tokens(),
170
}
171
172
+ def get_audio_placeholder(
173
+ self,
174
+ audio_lens: int,
175
+ chunk_input: bool = True,
176
+ chunk_length: int = 1,
177
+ ) -> str:
178
+ hf_processor = self.get_hf_processor()
179
+
180
+ return hf_processor.get_audio_placeholder(
181
+ audio_lens,
182
+ chunk_input=chunk_input,
183
+ chunk_length=chunk_length,
184
+ )
185
+
186
def get_default_audio_pool_step(self) -> int:
187
return 2
188
@@ -197,12 +218,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
218
max_videos = mm_config.get_limit_per_prompt("video")
219
max_audios = mm_config.get_limit_per_prompt("audio")
220
200
- # count <image_idx></image_idx> tokens
201
- # which are not in get_max_image_tokens
202
- max_image_tokens = self.get_max_image_tokens(
203
- ) * max_images + 4 * max_images
204
- max_audio_tokens = self.get_max_audio_tokens(
205
- ) * max_audios + 2 * max_audios
221
+ max_image_tokens = self.get_max_image_tokens() * max_images
222
+ max_audio_tokens = self.get_max_audio_tokens() * max_audios
223
max_total_frames = self.get_max_video_frames(seq_len -
224
max_image_tokens -
225
max_audio_tokens)
@@ -224,20 +241,20 @@ class MiniCPMODummyInputsBuilder(
241
242
processor_inputs = super().get_dummy_processor_inputs(
243
seq_len, mm_counts)
227
- mm_data = {
228
- "image":
229
- processor_inputs.mm_data["image"],
230
- "video":
231
- processor_inputs.mm_data["video"],
244
+
245
+ audio_prompt_texts = self.info.audio_pattern * num_audios
246
+ audio_mm_data = {
247
"audio":
248
self._get_dummy_audios(length=audio_len, num_audios=num_audios)
249
}
250
236
- audio_prompt_texts = self.info.audio_pattern * num_audios
237
-
238
- return ProcessorInputs(prompt_text=processor_inputs.prompt_text + \
239
- audio_prompt_texts,
240
- mm_data=mm_data)
251
+ return ProcessorInputs(
252
+ prompt_text=processor_inputs.prompt_text + audio_prompt_texts,
253
+ mm_data={
254
+ **processor_inputs.mm_data,
255
+ **audio_mm_data,
256
+ },
257
+ )
258
259
260
class MiniCPMOMultiModalProcessor(
@@ -247,22 +264,17 @@ class MiniCPMOMultiModalProcessor(
264
return MiniCPMOMultiModalDataParser(
265
target_sr=self.info.get_default_audio_sampling_rate())
266
250
- def get_audio_prompt_texts(self,
251
- audio_lens: int,
252
- chunk_input: bool = True,
253
- chunk_length: int = 1) -> str:
254
- return self.info.get_hf_processor().get_audio_placeholder(
255
- audio_lens, chunk_input, chunk_length)
256
-
257
- def get_special_tokens(self) -> Dict[str, torch.Tensor]:
258
- tokenizer = self.info.get_tokenizer()
259
- special_tokens = super().get_special_tokens()
260
- if hasattr(tokenizer, "audio_start_id"):
261
- special_tokens["audio_start_id"] = torch.tensor(
262
- tokenizer.audio_start_id)
263
- special_tokens["audio_end_id"] = torch.tensor(
264
- tokenizer.audio_end_id)
265
- return special_tokens
267
+ def get_audio_prompt_texts(
268
+ self,
269
+ audio_lens: int,
270
+ chunk_input: bool = True,
271
+ chunk_length: int = 1,
272
+ ) -> str:
273
+ return self.info.get_audio_placeholder(
274
+ audio_lens,
275
+ chunk_input=chunk_input,
276
+ chunk_length=chunk_length,
277
+ )
278
279
def process_audios(
280
self,
@@ -274,31 +286,64 @@ class MiniCPMOMultiModalProcessor(
286
287
parsed_audios = (self._get_data_parser().parse_mm_data({
288
"audio": audios
277
- }).get_items("audio", AudioProcessorItems))
289
+ }).get_items("audio",
290
+ (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)))
291
279
- audio_inputs = self._base_call_hf_processor(
280
- prompts=[self.info.audio_pattern] * len(parsed_audios),
281
- mm_data={"audios": [[audio] for audio in parsed_audios]},
282
- mm_kwargs={
283
- **mm_kwargs, "chunk_input": True
284
- },
285
- out_keys={"audio_features", "audio_feature_lens"},
286
- )
292
+ if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
293
+ audio_inputs = {}
294
288
- # Avoid padding since we need the output for each audio to be
289
- # independent of other audios for the cache to work correctly
290
- unpadded_audio_features = [
291
- feat[:, :feature_len] for feat, feature_len in zip(
292
- audio_inputs["audio_features"],
293
- audio_inputs["audio_feature_lens"],
295
+ audio_lens = [
296
+ self.info.get_audio_len_by_num_chunks(
297
+ sum(map(len,
298
+ parsed_audios.get(i)["audio_embeds"])))
299
+ for i in range(len(parsed_audios))
300
+ ]
301
+ else:
302
+ audio_inputs = self._base_call_hf_processor(
303
+ prompts=[self.info.audio_pattern] * len(parsed_audios),
304
+ mm_data={"audios": [[audio] for audio in parsed_audios]},
305
+ mm_kwargs={
306
+ **mm_kwargs,
307
+ "chunk_input": True,
308
+ },
309
+ out_keys={"audio_features", "audio_feature_lens"},
310
)
311
+
312
+ # Avoid padding since we need the output for each audio to be
313
+ # independent of other audios for the cache to work correctly
314
+ unpadded_audio_features = [
315
+ feat[:, :feature_len] for feat, feature_len in zip(
316
+ audio_inputs["audio_features"],
317
+ audio_inputs["audio_feature_lens"],
318
+ )
319
+ ]
320
+ audio_inputs["audio_features"] = unpadded_audio_features
321
+
322
+ audio_lens = [
323
+ parsed_audios.get_audio_length(i)
324
+ for i in range(len(parsed_audios))
325
+ ]
326
+
327
+ audio_repl_features = [
328
+ self.get_audio_prompt_texts(audio_len) for audio_len in audio_lens
329
]
296
- audio_inputs["audio_features"] = unpadded_audio_features
330
298
- return audio_inputs
331
+ tokenizer = self.info.get_tokenizer()
332
+ audio_repls_feature_tokens = [
333
+ tokenizer.encode(audio_repl, add_special_tokens=False)
334
+ for audio_repl in audio_repl_features
335
+ ]
336
+
337
+ embed_is_patch = [
338
+ self.get_embed_is_patch(audio_repl_tokens)
339
+ for audio_repl_tokens in audio_repls_feature_tokens
340
+ ]
341
+ audio_inputs["audio_embed_is_patch"] = embed_is_patch
342
300
- def get_placeholder_match_pattern(self) -> str:
301
- return r"\(<(image|video|audio)>./</\1>\)"
343
+ unk_token_id = tokenizer.get_vocab()["<unk>"]
344
+ audio_inputs["audio_token_id"] = torch.tensor(unk_token_id)
345
+
346
+ return audio_inputs
347
348
def process_mm_inputs(
349
self,
@@ -331,8 +376,7 @@ class MiniCPMOMultiModalProcessor(
376
if isinstance(audios, MiniCPMOAudioEmbeddingItems):
377
single_audio_embeds = audios.get(item_idx)["audio_embeds"]
378
audio_len = self.info.get_audio_len_by_num_chunks(
334
- sum(chunk_embeds.shape[0]
335
- for chunk_embeds in single_audio_embeds))
379
+ sum(map(len, single_audio_embeds)))
380
else:
381
audio_len = audios.get_audio_length(item_idx)
382
@@ -514,6 +558,8 @@ class MiniCPMO(MiniCPMV2_6):
558
self.apm = self.init_audio_module(vllm_config=vllm_config,
559
prefix=maybe_prefix(prefix, "apm"))
560
561
+ self.audio_token_id = None
562
+
563
def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""):
564
# Do not use parameters temporarily
565
audio_config = self.config.audio_config
@@ -563,18 +609,30 @@ class MiniCPMO(MiniCPMV2_6):
609
610
return input_lengths_after_cnn, input_lengths_after_pooling
611
566
- # Copied from HF repo of MiniCPM-o-2_6,
567
- # designed for batched inputs and outputs
568
- def get_audio_hidden_states(self, data: MiniCPMOAudioInputs,
569
- chunk_length: int) -> list[torch.Tensor]:
570
- wavforms = data.get(
571
- "audio_features",
572
- []) # (bs, 80, frames) or [], multi audios need filled in advance
573
- audio_feature_lens_raw = [data.get("audio_feature_lens",
574
- [])] # list, [[x1, x2], [y1], [z1]]
612
+ def get_audio_hidden_states(
613
+ self, data: MiniCPMOAudioFeatureInputs) -> list[torch.Tensor]:
614
+ chunk_length = self.config.audio_chunk_length
615
+
616
+ # (bs, 80, frames) or [], multi audios need filled in advance
617
+ wavforms_raw = data["audio_features"]
618
+ if isinstance(wavforms_raw, list):
619
+ B = len(wavforms_raw)
620
+ C = wavforms_raw[0].shape[-2]
621
+ L = max(item.shape[-1] for item in wavforms_raw)
622
+ device = wavforms_raw[0].device
623
+ dtype = wavforms_raw[0].dtype
624
+
625
+ wavforms = torch.zeros((B, C, L), dtype=dtype, device=device)
626
+ for i, wavforms_item in enumerate(wavforms_raw):
627
+ L_item = wavforms_item.shape[-1]
628
+ wavforms[i, ..., :L_item] = wavforms_item
629
+ else:
630
+ wavforms = wavforms_raw
631
576
- if len(wavforms) == 0:
577
- return []
632
+ # list, [[x1, x2], [y1], [z1]]
633
+ audio_feature_lens_raw = data["audio_feature_lens"]
634
+ if isinstance(audio_feature_lens_raw, torch.Tensor):
635
+ audio_feature_lens_raw = audio_feature_lens_raw.unbind(0)
636
637
audio_feature_lens = torch.hstack(audio_feature_lens_raw)
638
batch_size, _, max_mel_seq_len = wavforms.shape
@@ -625,159 +683,104 @@ class MiniCPMO(MiniCPMV2_6):
683
684
num_audio_tokens = feature_lens_after_pooling
685
628
- final_audio_embeds = []
686
+ final_audio_embeds = list[torch.Tensor]()
687
idx = 0
688
for i in range(len(audio_feature_lens_raw)):
631
- target_audio_embeds = []
689
+ target_audio_embeds_lst = list[torch.Tensor]()
690
for _ in range(len(audio_feature_lens_raw[i])):
633
- target_audio_embeds.append(
691
+ target_audio_embeds_lst.append(
692
audio_embeds[idx, :num_audio_tokens[idx], :])
693
idx += 1
636
- final_audio_embeds.append(target_audio_embeds)
637
- return final_audio_embeds
694
639
- def get_embedding_with_audios(self, vlm_embedding: torch.Tensor,
640
- audio_inputs: MiniCPMOAudioInputs,
641
- chunk_length: int) -> torch.Tensor:
642
- device, dtype = vlm_embedding.device, vlm_embedding.dtype
643
- if audio_inputs["type"] == "audio_embeds":
644
- audio_embeddings = [
645
- item.to(device=device, dtype=dtype)
646
- for item in audio_inputs["audio_embeds"]
647
- ]
648
- else:
649
- audio_embeddings = self.get_audio_hidden_states(
650
- audio_inputs, chunk_length)[0]
651
- if audio_embeddings is None or len(audio_embeddings) == 0:
652
- return vlm_embedding
653
- audio_bounds = audio_inputs["audio_bounds"]
654
- if self.config.chunk_input:
655
- audio_embs = torch.cat(audio_embeddings, dim=0).to(device=device,
656
- dtype=dtype)
657
- audio_start_pos = 0
658
- for bound in audio_bounds:
659
- audio_len = bound[1] - bound[0]
660
- vlm_embedding[bound[0]:bound[1]] = audio_embs[
661
- audio_start_pos:audio_start_pos + audio_len, :]
662
- audio_start_pos += audio_len
663
- else:
664
- for embs, bound in zip(audio_embeddings, audio_bounds):
665
- audio_indices = torch.arange(bound[0],
666
- bound[1],
667
- dtype=torch.long).to(device)
668
-
669
- if embs.shape[0] != len(audio_indices):
670
- raise ValueError(
671
- "Shape mismatch: Trying to assign embeddings "
672
- f"of shape {embs.shape} "
673
- f"to input indices of length {len(audio_indices)}")
674
- vlm_embedding[audio_indices] = embs.to(dtype)
675
- return vlm_embedding
676
-
677
- def _get_audio_bounds(self, input_ids: torch.Tensor,
678
- audio_start_id: torch.Tensor,
679
- audio_end_id: torch.Tensor) -> torch.Tensor:
680
- audio_start_tokens, = torch.where(input_ids == audio_start_id[0])
681
- audio_start_tokens += 1
682
- audio_end_tokens, = torch.where(input_ids == audio_end_id[0])
683
- valid_audio_nums = max(len(audio_start_tokens), len(audio_end_tokens))
684
- return torch.hstack([
685
- audio_start_tokens[:valid_audio_nums].unsqueeze(-1),
686
- audio_end_tokens[:valid_audio_nums].unsqueeze(-1)
687
- ])
695
+ final_audio_embeds.append(torch.cat(target_audio_embeds_lst))
696
689
- def _parse_and_validate_audio_inputs(
690
- self, input_ids: torch.Tensor,
691
- **kwargs: object) -> Optional[MiniCPMOAudioInputs]:
697
+ return final_audio_embeds
698
+
699
+ def _parse_and_validate_audio_input(
700
+ self, **kwargs: object) -> Optional[MiniCPMOAudioInputs]:
701
audio_features = kwargs.pop("audio_features", None)
702
audio_embeds = kwargs.pop("audio_embeds", None)
703
704
if audio_features is None and audio_embeds is None:
705
return None
706
698
- audio_start_id = kwargs.pop("audio_start_id")
699
- if not isinstance(audio_start_id, torch.Tensor):
700
- raise ValueError("Incorrect type of audio_start_id. "
701
- f"Got type: {type(audio_start_id)}")
707
+ audio_token_id = kwargs.pop("audio_token_id")
708
+ if audio_token_id is not None:
709
+ assert isinstance(audio_token_id, torch.Tensor)
710
+ self.mm_token_ids.add(audio_token_id.flatten().unique().item())
711
+
712
+ audio_embed_is_patch = kwargs.pop("audio_embed_is_patch")
713
+ if not isinstance(audio_embed_is_patch, (torch.Tensor, list)):
714
+ raise ValueError("Incorrect type of audio_embed_is_patch. "
715
+ f"Got type: {type(audio_embed_is_patch)}")
716
703
- audio_end_id = kwargs.pop("audio_end_id")
704
- if not isinstance(audio_end_id, torch.Tensor):
705
- raise ValueError("Incorrect type of audio_end_id. "
706
- f"Got type: {type(audio_end_id)}")
717
+ audio_embed_is_patch = flatten_bn(audio_embed_is_patch)
718
719
if audio_embeds is not None:
720
if not isinstance(audio_embeds, (torch.Tensor, list)):
721
raise ValueError("Incorrect type of audio_embeds. "
722
f"Got type: {type(audio_embeds)}")
723
724
+ audio_embeds_flat = flatten_bn(audio_embeds)
725
+
726
return MiniCPMOAudioEmbeddingInputs(
727
type="audio_embeds",
715
- audio_embeds=flatten_bn(flatten_2d_lists(audio_embeds),
716
- concat=True),
717
- audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
718
- audio_end_id),
728
+ audio_embeds=audio_embeds_flat,
729
+ embed_is_patch=audio_embed_is_patch,
730
)
731
721
- if audio_features is not None:
722
- if not isinstance(audio_features, (torch.Tensor, list)):
723
- raise ValueError("Incorrect type of audio_features. "
724
- f"Got type: {type(audio_features)}")
725
-
726
- audio_feature_lens = kwargs.pop("audio_feature_lens")
727
- if not isinstance(audio_feature_lens, (torch.Tensor, list)):
728
- raise ValueError("Incorrect type of audio_feature_lens. "
729
- f"Got type: {type(audio_feature_lens)}")
730
-
731
- return MiniCPMOAudioFeatureInputs(
732
- type="audio_features",
733
- audio_features=flatten_bn(audio_features, concat=True),
734
- audio_feature_lens=flatten_bn(
735
- flatten_2d_lists(audio_feature_lens), concat=True),
736
- audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
737
- audio_end_id),
738
- )
732
+ if not isinstance(audio_features, (torch.Tensor, list)):
733
+ raise ValueError("Incorrect type of audio_features. "
734
+ f"Got type: {type(audio_features)}")
735
740
- raise AssertionError("This line should be unreachable.")
736
+ audio_feature_lens = kwargs.pop("audio_feature_lens")
737
+ if not isinstance(audio_feature_lens, (torch.Tensor, list)):
738
+ raise ValueError("Incorrect type of audio_feature_lens. "
739
+ f"Got type: {type(audio_feature_lens)}")
740
742
- def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
743
- **kwargs: object):
744
- image_inputs = self._parse_and_validate_image_inputs(
745
- input_ids, **kwargs)
746
- if not any("audio" in key for key in kwargs):
747
- return image_inputs, None
748
- audio_inputs = self._parse_and_validate_audio_inputs(
749
- input_ids, **kwargs)
750
- return image_inputs, audio_inputs
741
+ audio_features_flat = flatten_bn(audio_features)
742
+ audio_feature_lens_flat = flatten_bn(audio_feature_lens)
743
752
- def forward(
753
- self,
754
- input_ids: torch.Tensor,
755
- positions: torch.Tensor,
756
- intermediate_tensors: Optional[IntermediateTensors] = None,
757
- **kwargs: Any,
758
- ) -> torch.Tensor:
759
- if intermediate_tensors is not None:
760
- vlm_embeddings = None
761
- else:
762
- image_inputs, audio_inputs = \
763
- self._parse_and_validate_inputs(input_ids, **kwargs)
764
- vlm_embeddings = self.get_embedding_with_vision(
765
- input_ids, image_inputs)
766
-
767
- if audio_inputs is not None:
768
- vlm_embeddings = self.get_embedding_with_audios(
769
- vlm_embeddings, audio_inputs,
770
- self.config.audio_chunk_length)
771
-
772
- # always pass the input via `inputs_embeds`
773
- # to make sure the computation graph is consistent
774
- # for `torch.compile` integration
775
- input_ids = None
776
-
777
- output = self.llm.model(
778
- input_ids=input_ids,
779
- positions=positions,
780
- intermediate_tensors=intermediate_tensors,
781
- inputs_embeds=vlm_embeddings,
744
+ return MiniCPMOAudioFeatureInputs(
745
+ type="audio_features",
746
+ audio_features=audio_features_flat,
747
+ audio_feature_lens=audio_feature_lens_flat,
748
+ embed_is_patch=audio_embed_is_patch,
749
)
783
- return output
750
+
751
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
752
+ modalities = super()._parse_and_validate_multimodal_inputs(**kwargs)
753
+
754
+ # Preserve the order of modalities if there are multiple of them
755
+ # from the order of kwargs.
756
+ for input_key in kwargs:
757
+ if input_key in ("audio_features",
758
+ "audio_embeds") and "audios" not in modalities:
759
+ modalities["audios"] = self._parse_and_validate_audio_input(
760
+ **kwargs)
761
+
762
+ return modalities
763
+
764
+ def _process_audio_input(
765
+ self,
766
+ audio_input: MiniCPMOAudioInputs,
767
+ ) -> Union[torch.Tensor, list[torch.Tensor]]:
768
+ if audio_input["type"] == "audio_embeds":
769
+ return audio_input["audio_embeds"]
770
+
771
+ return self.get_audio_hidden_states(audio_input)
772
+
773
+ def _process_multimodal_inputs(self, modalities: dict):
774
+ multimodal_embeddings = super()._process_multimodal_inputs(modalities)
775
+
776
+ for modality in modalities:
777
+ if modality == "audios":
778
+ audio_input = modalities["audios"]
779
+ audio_features = self._process_audio_input(audio_input)
780
+ multimodal_embeddings += tuple(
781
+ scatter_patch_features(
782
+ audio_features,
783
+ audio_input["embed_is_patch"],
784
+ ))
785
+
786
+ return multimodal_embeddings
vllm/model_executor/models/minicpmv.py
+343
-361
@@ -23,17 +23,15 @@
23
# limitations under the License.
24
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
25
import math
26
-import re
26
from collections import defaultdict
27
from collections.abc import Iterable, Mapping, Sequence
28
from functools import cached_property, partial
30
-from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,
31
- TypedDict, Union)
29
+from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,
30
+ Union)
31
32
import numpy as np
33
import torch
34
import torch.types
36
-from PIL import Image
35
from torch import nn
36
from transformers import BatchFeature, PretrainedConfig
37
from typing_extensions import TypeVar
@@ -50,9 +48,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
48
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
53
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
54
- MultiModalInputs, NestedTensors,
55
- PlaceholderRange)
51
+from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
52
from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
53
ImageProcessorItems, ImageSize,
54
ModalityData, ModalityDataItems,
@@ -67,13 +63,11 @@ from vllm.sequence import IntermediateTensors
63
from vllm.utils import flatten_2d_lists
64
65
from .idefics2_vision_model import Idefics2VisionTransformer
70
-from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,
71
- SupportsV0Only)
72
-from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix
73
-
74
-CPU_DEVICE = torch.device("cpu")
75
-
76
-RawImageType = Union[Image.Image, torch.Tensor]
66
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
67
+ SupportsMultiModal, SupportsPP)
68
+from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
69
+ merge_multimodal_embeddings)
70
+from .vision import scatter_patch_features, select_patch_features
71
72
73
class MiniCPMVImagePixelInputs(TypedDict):
@@ -86,37 +80,41 @@ class MiniCPMVImagePixelInputs(TypedDict):
80
instead of a batched tensor.
81
"""
82
89
- image_bounds: torch.Tensor
83
+ tgt_sizes: torch.Tensor
84
"""
85
Shape: `(batch_size * num_images * num_slices, 2)`
86
93
- This should be in `(start, stop)` format.
87
+ This should be in `(height, width)` format.
88
"""
89
96
- tgt_sizes: torch.Tensor
90
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
91
"""
98
- Shape: `(batch_size * num_images * num_slices, 2)`
92
+ A boolean mask indicating which image embeddings correspond
93
+ to patch tokens.
94
100
- This should be in `(height, width)` format.
95
+ Shape: `(batch_size * num_images, num_embeds)`
96
"""
97
98
+ num_slices: torch.Tensor
99
+ """Shape: `(batch_size * num_images)`"""
100
+
101
102
class MiniCPMVImageEmbeddingInputs(TypedDict):
103
type: Literal["image_embeds"]
106
- image_embeds: torch.Tensor
104
+ image_embeds: Union[torch.Tensor, list[torch.Tensor]]
105
"""
108
- Shape: `(batch_size * num_images * num_slices,
109
- image_feature_size, hidden_size)`
106
+ Shape: `(batch_size * num_images, num_slices, hidden_size)`
107
108
`hidden_size` must match the hidden size of language model backbone.
109
instead of a batched tensor.
110
"""
111
115
- image_bounds: torch.Tensor
112
+ embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
113
"""
117
- Shape: `(batch_size * num_images * num_slices, 2)`
114
+ A boolean mask indicating which image embeddings correspond
115
+ to patch tokens.
116
119
- This should be in `(start, stop)` format.
117
+ Shape: `(batch_size * num_images, num_embeds)`
118
"""
119
120
@@ -233,15 +231,25 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
231
232
233
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
234
+ pixel_values = hf_inputs.get("pixel_values", torch.empty(0))
235
+ num_images = len(pixel_values)
236
+
237
+ video_pixel_values = hf_inputs.get("video_pixel_values", torch.empty(0))
238
+ num_videos = len(video_pixel_values)
239
+
240
return dict(
241
pixel_values=MultiModalFieldConfig.batched("image"),
242
image_sizes=MultiModalFieldConfig.batched("image"),
243
tgt_sizes=MultiModalFieldConfig.batched("image"),
244
image_embeds=MultiModalFieldConfig.batched("image"),
245
+ embed_is_patch=MultiModalFieldConfig.batched("image"),
246
video_pixel_values=MultiModalFieldConfig.batched("video"),
247
video_image_sizes=MultiModalFieldConfig.batched("video"),
248
video_tgt_sizes=MultiModalFieldConfig.batched("video"),
249
video_embeds=MultiModalFieldConfig.batched("video"),
250
+ video_embed_is_patch=MultiModalFieldConfig.batched("video"),
251
+ image_token_id=MultiModalFieldConfig.shared("image", num_images),
252
+ video_token_id=MultiModalFieldConfig.shared("video", num_videos),
253
)
254
255
@@ -348,10 +356,11 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
356
return get_version_by_config(self.get_hf_config())
357
358
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
359
+ mm_limits = {"image": None}
360
if self.get_model_version() == (2, 6):
352
- return {"image": None, "video": None}
353
- else:
354
- return {"image": None}
361
+ mm_limits["video"] = None
362
+
363
+ return mm_limits
364
365
def get_mm_max_tokens_per_item(
366
self,
@@ -361,70 +370,79 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
370
mm_max_tokens = {"image": self.get_max_image_tokens()}
371
if self.get_model_version() == (2, 6):
372
mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)
373
+
374
return mm_max_tokens
375
366
- def get_max_video_frame_tokens(self) -> int:
367
- frame_size = self.get_video_frame_size_with_most_features()
368
- return self.get_num_image_tokens(frame_size,
369
- self.get_video_max_slice_num())
376
+ def get_slice_image_placeholder(
377
+ self,
378
+ image_size: ImageSize,
379
+ # For MiniCPM V/O 2.6
380
+ image_idx: int = 0,
381
+ max_slice_nums: Optional[int] = None,
382
+ use_image_id: bool = True,
383
+ ) -> str:
384
+ image_processor = self.get_image_processor()
385
+ version = self.get_model_version()
386
371
- def get_max_video_tokens(self, seq_len: int) -> int:
372
- return self.get_max_video_frame_tokens(
373
- ) * self.get_num_frames_with_most_features(seq_len)
387
+ if version == (2, 0) or version == (2, 5):
388
+ return image_processor.get_slice_image_placeholder(image_size)
389
375
- def get_slice_query_num(self) -> int:
376
- hf_config = self.get_hf_config()
377
- query_num = getattr(hf_config, "query_num", 64)
378
- return query_num
390
+ return image_processor.get_slice_image_placeholder(
391
+ image_size,
392
+ image_idx=image_idx,
393
+ max_slice_nums=max_slice_nums,
394
+ use_image_id=use_image_id,
395
+ )
396
380
- def get_max_slice_num(self) -> int:
381
- hf_config = self.get_hf_config()
382
- max_slice_num = getattr(hf_config, "max_slice_num", 9)
383
- return max_slice_num
397
+ def get_num_image_tokens(
398
+ self,
399
+ image_size: ImageSize,
400
+ max_slice_nums: Optional[int] = None,
401
+ use_image_id: bool = True,
402
+ ) -> int:
403
+ tokenizer = self.get_tokenizer()
404
+ image_placeholders = self.get_slice_image_placeholder(
405
+ image_size,
406
+ max_slice_nums=max_slice_nums,
407
+ use_image_id=use_image_id,
408
+ )
409
+ image_token_ids = tokenizer.encode(image_placeholders,
410
+ add_special_tokens=False)
411
385
- def get_sliced_grid(self, image_size: ImageSize,
386
- max_slice_num: int) -> Tuple[int, int]:
387
- if self.get_model_version() == (2, 6):
388
- slice_grid = self.get_image_processor().get_sliced_grid(
389
- image_size, max_slice_num)
390
- else:
391
- slice_grid = self.get_image_processor().get_sliced_grid(image_size)
392
- return slice_grid
393
-
394
- def get_num_image_tokens(self, image_size: ImageSize,
395
- max_slice_num: int) -> int:
396
- slice_grid = self.get_sliced_grid(image_size, max_slice_num)
397
- num_tokens = self.get_slice_query_num(
398
- ) + 2 # <image>(<unk> * query_num)</image>
399
- if slice_grid is not None:
400
- if self.get_model_version() == (2, 6):
401
- num_additional_tokens = 0
402
- else:
403
- # <slice><image>(<unk> * query_num)</image></slice>
404
- num_additional_tokens = 2
405
- num_tokens += ((self.get_slice_query_num() + 2) \
406
- * slice_grid[0] * slice_grid[1]) \
407
- + slice_grid[1] - 1 + num_additional_tokens
408
- return num_tokens
409
-
410
- def get_image_slice_nums(self, image_size: torch.Tensor,
411
- max_slice_nums: int) -> int:
412
- grid = self.get_sliced_grid(image_size, max_slice_nums)
413
- return 1 if grid is None else grid[0] * grid[1] + 1
412
+ return len(image_token_ids)
413
414
def get_max_image_tokens(self) -> int:
415
image_size = self.get_image_size_with_most_features()
417
- return self.get_num_image_tokens(image_size, self.get_max_slice_num())
416
+ return self.get_num_image_tokens(image_size)
417
+
418
+ def get_image_max_slice_num(self) -> int:
419
+ return getattr(self.get_hf_config(), "max_slice_num", 9)
420
421
def get_image_size_with_most_features(self) -> ImageSize:
420
- # Result in the max possible feature size (h:w = 9:1)
421
- return self.get_default_image_sizes(self.get_max_slice_num())
422
+ image_size = getattr(self.get_hf_config(), "image_size", 448)
423
+ max_slice_num = self.get_image_max_slice_num()
424
+ return ImageSize(width=image_size, height=image_size * max_slice_num)
425
+
426
+ def get_max_video_frame_tokens(self) -> int:
427
+ frame_size = self.get_video_frame_size_with_most_features()
428
+
429
+ return self.get_num_image_tokens(
430
+ frame_size,
431
+ max_slice_nums=self.get_video_max_slice_num(),
432
+ use_image_id=False,
433
+ )
434
+
435
+ def get_max_video_tokens(self, seq_len: int) -> int:
436
+ return self.get_max_video_frame_tokens(
437
+ ) * self.get_num_frames_with_most_features(seq_len)
438
439
def get_video_max_slice_num(self) -> int:
440
return 1
441
442
def get_video_frame_size_with_most_features(self) -> ImageSize:
427
- return self.get_default_image_sizes(self.get_video_max_slice_num())
443
+ image_size = getattr(self.get_hf_config(), "image_size", 448)
444
+ max_slice_num = self.get_video_max_slice_num()
445
+ return ImageSize(width=image_size, height=image_size * max_slice_num)
446
447
def get_max_video_frames(self, max_tokens: int) -> int:
448
num_frame_tokens = self.get_max_video_frame_tokens()
@@ -436,10 +454,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
454
max_images = mm_config.get_limit_per_prompt("image")
455
max_videos = mm_config.get_limit_per_prompt("video")
456
439
- # count <image_idx></image_idx> tokens
440
- # which are not in get_max_image_tokens
441
- max_image_tokens = self.get_max_image_tokens(
442
- ) * max_images + 4 * max_images
457
+ max_image_tokens = self.get_max_image_tokens() * max_images
458
max_total_frames = self.get_max_video_frames(seq_len -
459
max_image_tokens)
460
@@ -447,10 +462,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
462
463
return num_frames
464
450
- def get_default_image_sizes(self, num_slices: int) -> ImageSize:
451
- image_size = getattr(self.get_hf_config(), "image_size", 448)
452
- return ImageSize(width=image_size, height=image_size * num_slices)
453
-
465
466
_I = TypeVar("_I",
467
bound=MiniCPMVProcessingInfo,
@@ -499,42 +510,30 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
510
def _get_data_parser(self) -> MultiModalDataParser:
511
return MiniCPMVMultiModalDataParser()
512
502
- def get_slice_image_placeholder(self, image_size: ImageSize,
503
- **kwargs) -> str:
504
- image_processor = self.info.get_image_processor()
505
- version = self.info.get_model_version()
506
- if version == (2, 0) or version == (2, 5):
507
- return image_processor.get_slice_image_placeholder(image_size)
508
- return image_processor.get_slice_image_placeholder(
509
- image_size, **kwargs)
510
-
513
def get_image_prompt_texts(self,
514
image_size: ImageSize,
515
image_idx: int = 0) -> str:
514
- return self.get_slice_image_placeholder(image_size,
515
- image_idx=image_idx)
516
+ return self.info.get_slice_image_placeholder(
517
+ image_size,
518
+ image_idx=image_idx,
519
+ )
520
521
def get_video_prompt_texts(self, image_size: ImageSize,
522
num_frames: int) -> str:
519
- return self.get_slice_image_placeholder(
523
+ return self.info.get_slice_image_placeholder(
524
image_size=image_size,
525
image_idx=0,
526
max_slice_nums=self.info.get_video_max_slice_num(),
527
use_image_id=False,
528
) * num_frames
529
526
- def get_special_tokens(self) -> Dict[str, torch.Tensor]:
530
+ def get_embed_is_patch(
531
+ self,
532
+ input_ids: list[int],
533
+ ) -> torch.Tensor:
534
tokenizer = self.info.get_tokenizer()
528
-
529
- special_tokens = {
530
- "im_start_id": tokenizer.im_start_id,
531
- "im_end_id": tokenizer.im_end_id,
532
- }
533
- if hasattr(tokenizer, "slice_start_id"):
534
- special_tokens["slice_start_id"] = tokenizer.slice_start_id
535
- special_tokens["slice_end_id"] = tokenizer.slice_end_id
536
-
537
- return {k: torch.tensor(v) for k, v in special_tokens.items()}
535
+ unk_token_id = tokenizer.get_vocab()["<unk>"]
536
+ return torch.tensor(input_ids) == unk_token_id
537
538
def process_images(
539
self,
@@ -546,14 +545,43 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
545
546
parsed_images = (self._get_data_parser().parse_mm_data({
547
"image": images
549
- }).get_items("image", ImageProcessorItems))
548
+ }).get_items("image",
549
+ (MiniCPMVImageEmbeddingItems, ImageProcessorItems)))
550
551
- return self._base_call_hf_processor(
552
- prompts=[self.info.image_pattern] * len(parsed_images),
553
- mm_data={"images": [[image] for image in parsed_images]},
554
- mm_kwargs=mm_kwargs,
555
- out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
556
- )
551
+ if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
552
+ image_inputs = {}
553
+ else:
554
+ image_inputs = self._base_call_hf_processor(
555
+ prompts=[self.info.image_pattern] * len(parsed_images),
556
+ mm_data={"images": [[image] for image in parsed_images]},
557
+ mm_kwargs=mm_kwargs,
558
+ out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
559
+ )
560
+
561
+ image_sizes = [
562
+ parsed_images.get_image_size(i) for i in range(len(parsed_images))
563
+ ]
564
+ image_repl_features = [
565
+ self.get_image_prompt_texts(size, idx)
566
+ for idx, size in enumerate(image_sizes)
567
+ ]
568
+
569
+ tokenizer = self.info.get_tokenizer()
570
+ image_repls_feature_tokens = [
571
+ tokenizer.encode(image_repl, add_special_tokens=False)
572
+ for image_repl in image_repl_features
573
+ ]
574
+
575
+ embed_is_patch = [
576
+ self.get_embed_is_patch(image_repl_tokens)
577
+ for image_repl_tokens in image_repls_feature_tokens
578
+ ]
579
+ image_inputs["embed_is_patch"] = embed_is_patch
580
+
581
+ unk_token_id = tokenizer.get_vocab()["<unk>"]
582
+ image_inputs["image_token_id"] = torch.tensor(unk_token_id)
583
+
584
+ return image_inputs
585
586
def process_videos(
587
self,
@@ -565,25 +593,55 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
593
594
parsed_videos = (self._get_data_parser().parse_mm_data({
595
"video": videos
568
- }).get_items("video", VideoProcessorItems))
569
-
570
- max_slice_num = self.info.get_video_max_slice_num()
571
-
572
- video_inputs = self._base_call_hf_processor(
573
- prompts=[
574
- self.info.image_pattern * len(video) for video in parsed_videos
575
- ],
576
- mm_data={"images": list(parsed_videos)},
577
- mm_kwargs={
578
- **mm_kwargs, "max_slice_nums": max_slice_num
579
- },
580
- out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
581
- )
596
+ }).get_items("video",
597
+ (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)))
598
+
599
+ if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
600
+ video_inputs = {}
601
+ else:
602
+ video_inputs = self._base_call_hf_processor(
603
+ prompts=[
604
+ self.info.image_pattern * len(video)
605
+ for video in parsed_videos
606
+ ],
607
+ mm_data={"images": list(parsed_videos)},
608
+ mm_kwargs={
609
+ **mm_kwargs,
610
+ "max_slice_nums":
611
+ self.info.get_video_max_slice_num(),
612
+ },
613
+ out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
614
+ )
615
+
616
+ frame_sizes = [
617
+ parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))
618
+ ]
619
+ num_frames = [
620
+ parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))
621
+ ]
622
+ video_repl_features = [
623
+ self.get_video_prompt_texts(size, nframes)
624
+ for size, nframes in zip(frame_sizes, num_frames)
625
+ ]
626
+
627
+ tokenizer = self.info.get_tokenizer()
628
+ video_repls_feature_tokens = [
629
+ tokenizer.encode(video_repl, add_special_tokens=False)
630
+ for video_repl in video_repl_features
631
+ ]
632
+
633
+ embed_is_patch = [
634
+ self.get_embed_is_patch(video_repl_tokens)
635
+ for video_repl_tokens in video_repls_feature_tokens
636
+ ]
637
+ video_inputs["embed_is_patch"] = embed_is_patch
638
583
- return {f"video_{k}": v for k, v in video_inputs.items()}
639
+ video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
640
585
- def get_placeholder_match_pattern(self) -> str:
586
- return r"\(<(image|video)>./</\1>\)"
641
+ unk_token_id = tokenizer.get_vocab()["<unk>"]
642
+ video_inputs["video_token_id"] = torch.tensor(unk_token_id)
643
+
644
+ return video_inputs
645
646
def process_mm_inputs(
647
self,
@@ -602,7 +660,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
660
mm_kwargs: Mapping[str, object],
661
*,
662
out_keys: set[str],
605
- ) -> Mapping[str, NestedTensors]:
663
+ ) -> dict[str, NestedTensors]:
664
# This processor supports zipping prompt and mm_data together
665
if self.info.get_model_version() == (2, 6):
666
inputs = super()._call_hf_processor(
@@ -635,14 +693,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
693
mm_data: Mapping[str, object],
694
mm_kwargs: Mapping[str, object],
695
) -> BatchFeature:
638
- # Do not support combination inputs of images and videos for now
639
- # Try to handle interleaved multimodal data
696
tokenizer = self.info.get_tokenizer()
697
+
698
+ input_ids = torch.tensor([tokenizer.encode(prompt)])
699
mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
700
701
return BatchFeature({
644
- "input_ids":
645
- torch.tensor([tokenizer.encode(prompt)]),
702
+ "input_ids": input_ids,
703
**mm_inputs,
704
})
705
@@ -701,39 +758,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
758
) -> Mapping[str, MultiModalFieldConfig]:
759
return _minicpmv_field_config(hf_inputs)
760
704
- def apply(
705
- self,
706
- prompt: Union[str, List[int]],
707
- mm_data: MultiModalDataDict,
708
- hf_processor_mm_kwargs: Mapping[str, object],
709
- return_mm_hashes: bool = False,
710
- ) -> MultiModalInputs:
711
- if isinstance(prompt, list):
712
- prompt = self.info.get_tokenizer().decode(prompt)
713
- matches = re.findall(self.get_placeholder_match_pattern(), prompt)
714
- mm_orders = {
715
- f"{modality}_orders":
716
- torch.tensor(
717
- [index for index, m in enumerate(matches) if m == modality])
718
- for modality in self.info.get_supported_mm_limits()
719
- }
720
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
721
- return_mm_hashes)
722
- # Exclude <image_id>x</image_id> from placeholders
723
- if "image" in result["mm_placeholders"] and \
724
- self.info.get_model_version() == (2, 6):
725
- result["mm_placeholders"]["image"] = [
726
- PlaceholderRange(offset=p["offset"] + 3 + idx // 10,
727
- length=p["length"] - 3 - idx // 10)
728
- for idx, p in enumerate(result["mm_placeholders"]["image"])
729
- ]
730
- result["mm_kwargs"].update(**mm_orders)
731
- result["mm_kwargs"].update(**self.get_special_tokens())
732
- return result
733
-
734
-
735
-class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
736
- SupportsV0Only):
761
+
762
+class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
763
"""
764
The abstract class of MiniCPMV can only be inherited, but cannot be
765
instantiated.
@@ -767,6 +793,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
793
prefix=maybe_prefix(
794
prefix, "resampler"))
795
796
+ self.mm_token_ids = set[int]()
797
self.make_empty_intermediate_tensors = (
798
self.llm.make_empty_intermediate_tensors)
799
@@ -777,233 +804,191 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
804
805
return get_sampler()
806
780
- def get_embedding_with_vision(
781
- self,
782
- input_ids: torch.Tensor,
783
- image_inputs: Optional[MiniCPMVImageInputs],
784
- ) -> torch.Tensor:
785
- vlm_embedding: torch.Tensor = self.llm.get_input_embeddings(input_ids)
786
-
787
- if image_inputs is None:
788
- return vlm_embedding
789
-
790
- if image_inputs["type"] == "image_embeds":
791
- vision_hidden_states = image_inputs["image_embeds"].to(
792
- device=vlm_embedding.device,
793
- dtype=vlm_embedding.dtype,
794
- )
795
- else:
796
- vision_hidden_states = self.get_vision_hidden_states(image_inputs)
797
-
798
- # See NOTE in _parse_and_validate_inputs
799
- image_bounds = image_inputs["image_bounds"]
800
- if len(image_bounds) > 0:
801
- image_indices = torch.stack([
802
- torch.arange(start, end, dtype=torch.long)
803
- for start, end in image_bounds.tolist()
804
- ]).to(vlm_embedding.device)
805
-
806
- vlm_embedding.scatter_(
807
- 0,
808
- image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
809
- vision_hidden_states.view(-1, vision_hidden_states.shape[-1]),
810
- )
811
-
812
- return vlm_embedding
813
-
814
- def _get_image_bounds(
815
- self,
816
- input_ids: torch.Tensor,
817
- im_start_id: torch.Tensor,
818
- im_end_id: torch.Tensor,
819
- slice_start_id: Optional[torch.Tensor] = None,
820
- slice_end_id: Optional[torch.Tensor] = None) -> torch.Tensor:
821
- # All the images in the batch should share the same special image
822
- # bound token ids.
823
- start_cond = input_ids == im_start_id[0]
824
- end_cond = input_ids == im_end_id[0]
825
- if slice_start_id is not None:
826
- start_cond |= (input_ids == slice_start_id[0])
827
- end_cond |= (input_ids == slice_end_id[0])
828
-
829
- image_start_tokens, = torch.where(start_cond)
830
- image_start_tokens += 1
831
- image_end_tokens, = torch.where(end_cond)
832
- valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
833
-
834
- if valid_image_nums == 0:
835
- return torch.zeros((0, 2), device=input_ids.device)
836
-
837
- return torch.hstack([
838
- image_start_tokens[:valid_image_nums].unsqueeze(-1),
839
- image_end_tokens[:valid_image_nums].unsqueeze(-1),
840
- ])
841
-
842
- def _parse_and_validate_image_inputs(
807
+ def _parse_and_validate_vision_input(
808
self,
844
- input_ids: torch.Tensor,
809
+ modality: str,
810
**kwargs: object,
811
) -> Optional[MiniCPMVImageInputs]:
847
- image_keys = {"pixel_values", "tgt_sizes"}
848
- pixel_data = {
849
- "image": {
850
- key: kwargs.pop(key, None)
851
- for key in image_keys
852
- },
853
- "video": {
854
- key: kwargs.pop("video_" + key, None)
855
- for key in image_keys
856
- }
857
- }
858
- embed_data = {
859
- "image": kwargs.pop("image_embeds", None),
860
- "video": kwargs.pop("video_embeds", None),
861
- }
812
+ pixel_values = kwargs.pop("pixel_values", None)
813
+ image_embeds = kwargs.pop("image_embeds", None)
814
863
- all_pixel_data = [
864
- v for vs in pixel_data.values() for v in vs.values()
865
- if v is not None
866
- ]
867
- all_embed_data = [v for v in embed_data.values() if v is not None]
868
- if len(all_pixel_data) == 0 and len(all_embed_data) == 0:
815
+ if pixel_values is None and image_embeds is None:
816
return None
817
871
- im_start_id = kwargs.pop("im_start_id")
872
- if not isinstance(im_start_id, torch.Tensor):
873
- raise ValueError("Incorrect type of im_start_id. "
874
- f"Got type: {type(im_start_id)}")
875
-
876
- im_end_id = kwargs.pop("im_end_id")
877
- if not isinstance(im_end_id, torch.Tensor):
878
- raise ValueError("Incorrect type of im_end_id. "
879
- f"Got type: {type(im_end_id)}")
880
-
881
- slice_start_id = kwargs.pop("slice_start_id", None)
882
- if slice_start_id is not None and not isinstance(
883
- slice_start_id, torch.Tensor):
884
- raise ValueError("Incorrect type of slice_start_id. "
885
- f"Got type: {type(slice_start_id)}")
886
-
887
- slice_end_id = kwargs.pop("slice_end_id", None)
888
- if slice_end_id is not None and not isinstance(slice_end_id,
889
- torch.Tensor):
890
- raise ValueError("Incorrect type of slice_end_id. "
891
- f"Got type: {type(slice_end_id)}")
892
-
893
- if len(all_embed_data) > 0:
894
- if len(all_embed_data) > 1:
895
- raise ValueError("Incorrect inputs for vision embeddings. "
896
- "Image embeds and video embeds can not "
897
- "exist simultaneously.")
898
-
899
- vision_embeds, = all_embed_data
900
- if not isinstance(vision_embeds, (torch.Tensor, list)):
901
- raise ValueError(f"Incorrect type of vision_embeds. "
902
- f"Got type: {type(vision_embeds)}")
818
+ image_token_id = kwargs.pop("image_token_id")
819
+ if image_token_id is not None:
820
+ assert isinstance(image_token_id, torch.Tensor)
821
+ self.mm_token_ids.add(image_token_id.flatten().unique().item())
822
+
823
+ embed_is_patch = kwargs.pop("embed_is_patch")
824
+ if not isinstance(embed_is_patch, (torch.Tensor, list)):
825
+ raise ValueError(
826
+ f"Incorrect type of embed_is_patch for {modality=}. "
827
+ f"Got type: {type(embed_is_patch)}")
828
+
829
+ embed_is_patch = flatten_bn(embed_is_patch)
830
+
831
+ if image_embeds is not None:
832
+ if not isinstance(image_embeds, (torch.Tensor, list)):
833
+ raise ValueError(
834
+ f"Incorrect type of image_embeds for {modality=}. "
835
+ f"Got type: {type(image_embeds)}")
836
+
837
+ image_embeds_flat = flatten_bn(image_embeds)
838
839
return MiniCPMVImageEmbeddingInputs(
840
type="image_embeds",
906
- image_embeds=flatten_bn(flatten_2d_lists(vision_embeds),
907
- concat=True),
908
- image_bounds=self._get_image_bounds(input_ids, im_start_id,
909
- im_end_id, slice_start_id,
910
- slice_end_id),
841
+ image_embeds=image_embeds_flat,
842
+ embed_is_patch=embed_is_patch,
843
)
844
913
- order_data = dict[str, Union[torch.Tensor, list[torch.Tensor]]]()
914
- for modality in ("image", "video"):
915
- modality_orders = kwargs.pop(f"{modality}_orders", None)
916
- if modality_orders is not None:
917
- if not isinstance(modality_orders, (torch.Tensor, list)):
918
- raise ValueError(f"Incorrect type of {modality}_orders. "
919
- f"Got type: {type(modality_orders)}")
845
+ if not isinstance(pixel_values, (torch.Tensor, list)):
846
+ raise ValueError(
847
+ f"Incorrect type of pixel_values for {modality=}. "
848
+ f"Got type: {type(pixel_values)}")
849
+
850
+ tgt_sizes = kwargs.pop("tgt_sizes")
851
+ if not isinstance(tgt_sizes, (torch.Tensor, list)):
852
+ raise ValueError(f"Incorrect type of tgt_sizes for {modality=}. "
853
+ f"Got type: {type(tgt_sizes)}")
854
921
- order_data[modality] = modality_orders
855
+ num_slices = [[len(p) for p in ps] for ps in pixel_values]
856
+ num_slices_flat = flatten_bn(torch.tensor(num_slices))
857
+
858
+ pixel_values_flat = flatten_bn(flatten_2d_lists(pixel_values))
859
+ tgt_sizes_flat = flatten_bn(flatten_2d_lists(tgt_sizes), concat=True)
860
923
- batch_sizes = {
924
- modality: len(modality_orders)
925
- for modality, modality_orders in order_data.items()
926
- }
927
- unique_batch_sizes = set(batch_sizes.values())
928
- assert len(unique_batch_sizes) == 1, (
929
- f"Found inconsistent batch sizes: {batch_sizes}")
930
- batch_size, = unique_batch_sizes
931
-
932
- pixel_values_flat = list[torch.Tensor]()
933
- tgt_sizes_flat = list[torch.Tensor]()
934
- for b in range(batch_size):
935
- mm_orders_b = [(idx_b.item(), modality)
936
- for modality, modality_orders in order_data.items()
937
- for idx_b in modality_orders[b]]
938
-
939
- for _, modality in sorted(mm_orders_b, key=lambda x: x[0]):
940
- modality_pixel_data = pixel_data[modality]
941
-
942
- modality_pixel_values = modality_pixel_data["pixel_values"]
943
- if not isinstance(modality_pixel_values, (torch.Tensor, list)):
944
- raise ValueError(
945
- f"Incorrect type of pixel_values for {modality=}. "
946
- f"Got type: {type(modality_pixel_values)}")
947
-
948
- modality_tgt_sizes = modality_pixel_data["tgt_sizes"]
949
- if not isinstance(modality_tgt_sizes, (torch.Tensor, list)):
950
- raise ValueError(
951
- f"Incorrect type of tgt_sizes for {modality=}. "
952
- f"Got type: {type(modality_tgt_sizes)}")
953
-
954
- pixel_values_flat += flatten_2d_lists(modality_pixel_values[b])
955
- tgt_sizes_flat += flatten_2d_lists(modality_tgt_sizes[b])
956
-
957
- # NOTE: Input IDs does not contain image tokens during memory profiling,
958
- # so we allow it to be empty
861
if len(pixel_values_flat) != len(tgt_sizes_flat):
862
raise ValueError("Inconsistent flattened lengths, found: "
863
f"{len(pixel_values_flat)} vs. "
864
f"{len(tgt_sizes_flat)}")
865
964
- if len(pixel_values_flat) == 0:
965
- return None
966
-
866
return MiniCPMVImagePixelInputs(
867
type="pixel_values",
868
pixel_values=pixel_values_flat,
970
- tgt_sizes=torch.stack(tgt_sizes_flat),
971
- image_bounds=self._get_image_bounds(input_ids, im_start_id,
972
- im_end_id, slice_start_id,
973
- slice_end_id),
869
+ tgt_sizes=tgt_sizes_flat,
870
+ embed_is_patch=embed_is_patch,
871
+ num_slices=num_slices_flat,
872
)
873
976
- def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
977
- **kwargs: object):
978
- return self._parse_and_validate_image_inputs(input_ids, **kwargs)
874
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
875
+ modalities = {}
876
+
877
+ # Preserve the order of modalities if there are multiple of them
878
+ # from the order of kwargs.
879
+ for input_key in kwargs:
880
+ if input_key in ("pixel_values",
881
+ "image_embeds") and "images" not in modalities:
882
+ modalities["images"] = self._parse_and_validate_vision_input(
883
+ "images", **kwargs)
884
+ if input_key in ("video_pixel_values",
885
+ "video_embeds") and "videos" not in modalities:
886
+
887
+ def _image_key(video_key: str):
888
+ if video_key == "video_token_id":
889
+ return "image_token_id"
890
+
891
+ return video_key.removeprefix("video_")
892
+
893
+ modalities["videos"] = self._parse_and_validate_vision_input(
894
+ "videos", **{
895
+ _image_key(k): v
896
+ for k, v in kwargs.items()
897
+ })
898
+
899
+ return modalities
900
+
901
+ def _process_vision_input(
902
+ self,
903
+ image_input: MiniCPMVImageInputs,
904
+ ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]:
905
+ if image_input["type"] == "image_embeds":
906
+ return image_input["image_embeds"]
907
+
908
+ image_features_flat = self.get_vision_hidden_states(image_input)
909
+
910
+ # Reconstruct the batch dimension
911
+ return image_features_flat.split(image_input["num_slices"].tolist())
912
+
913
+ def _process_multimodal_inputs(self, modalities: dict):
914
+ # The result multimodal_embeddings is tuple of tensors, with each
915
+ # tensor correspoending to a multimodal data item (image or video).
916
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
917
+
918
+ # NOTE: It is important to iterate over the keys in this dictionary
919
+ # to preserve the order of the modalities.
920
+ for modality in modalities:
921
+ if modality == "images":
922
+ image_input = modalities["images"]
923
+ image_features = self._process_vision_input(image_input)
924
+ multimodal_embeddings += tuple(
925
+ scatter_patch_features(
926
+ image_features,
927
+ image_input["embed_is_patch"],
928
+ ))
929
+ if modality == "videos":
930
+ video_input = modalities["videos"]
931
+ video_features = self._process_vision_input(video_input)
932
+ multimodal_embeddings += tuple(
933
+ scatter_patch_features(
934
+ video_features,
935
+ video_input["embed_is_patch"],
936
+ ))
937
+
938
+ return multimodal_embeddings
939
+
940
+ def get_multimodal_embeddings(
941
+ self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
942
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
943
+ if not modalities:
944
+ return None
945
+
946
+ return self._process_multimodal_inputs(modalities)
947
+
948
+ def get_input_embeddings(
949
+ self,
950
+ input_ids: torch.Tensor,
951
+ multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
952
+ ) -> torch.Tensor:
953
+ inputs_embeds = self.llm.get_input_embeddings(input_ids)
954
+ if multimodal_embeddings is not None:
955
+ assert len(self.mm_token_ids) > 0
956
+ inputs_embeds = merge_multimodal_embeddings(
957
+ input_ids,
958
+ inputs_embeds,
959
+ select_patch_features(multimodal_embeddings),
960
+ list(self.mm_token_ids),
961
+ )
962
+ return inputs_embeds
963
964
def forward(
965
self,
966
input_ids: torch.Tensor,
967
positions: torch.Tensor,
968
intermediate_tensors: Optional[IntermediateTensors] = None,
969
+ inputs_embeds: Optional[torch.Tensor] = None,
970
**kwargs: Any,
971
) -> torch.Tensor:
972
if intermediate_tensors is not None:
988
- vlm_embeddings = None
989
- else:
990
- image_inputs = \
991
- self._parse_and_validate_inputs(input_ids, **kwargs)
992
- vlm_embeddings = self.get_embedding_with_vision(
993
- input_ids, image_inputs)
973
+ inputs_embeds = None
974
995
- # always pass the input via `inputs_embeds`
996
- # to make sure the computation graph is consistent
997
- # for `torch.compile` integration
998
- input_ids = None
975
+ # NOTE: In v1, inputs_embeds is always generated at model runner from
976
+ # `get_multimodal_embeddings` and `get_input_embeddings`, this
977
+ # condition is only for v0 compatibility.
978
+ elif inputs_embeds is None:
979
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
980
1000
- output = self.llm.model(
981
+ inputs_embeds = self.get_input_embeddings(input_ids,
982
+ vision_embeddings)
983
+ input_ids = None
984
+
985
+ hidden_states = self.llm.model(
986
input_ids=input_ids,
987
positions=positions,
988
intermediate_tensors=intermediate_tensors,
1004
- inputs_embeds=vlm_embeddings,
989
+ inputs_embeds=inputs_embeds,
990
)
1006
- return output
991
+ return hidden_states
992
993
def compute_logits(
994
self,
@@ -1105,9 +1090,6 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
1090
1091
return model
1092
1108
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
1109
- return self.model.embed_tokens(input_ids)
1110
-
1093
def init_resampler(self,
1094
embed_dim: int,
1095
vision_dim: int,
vllm/model_executor/models/molmo.py
+18
-24
@@ -92,8 +92,8 @@ class MolmoImageInputs(TypedDict):
92
Shape: `(batch_size * num_images, num_embeds)`
93
"""
94
95
- num_crops: Union[torch.Tensor, list[torch.Tensor]]
96
- """Shape: `(batch_size, num_images)`"""
95
+ num_crops: torch.Tensor
96
+ """Shape: `(batch_size * num_images)`"""
97
98
99
@dataclass
@@ -1492,6 +1492,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1492
self.img_patch_id = img_patch_id.flatten().unique().item()
1493
1494
embed_is_patch = flatten_bn(embed_is_patch)
1495
+ num_crops = flatten_bn(num_crops, concat=True)
1496
1497
return MolmoImageInputs(
1498
images=images,
@@ -1510,31 +1511,24 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1511
feat_is_patch = image_input["feat_is_patch"]
1512
num_crops = image_input["num_crops"]
1513
1513
- if isinstance(images, list):
1514
- # Call the vision backbone on the whole batch at once
1515
- images_flat = flatten_bn(images, concat=True)
1516
- image_masks_flat = (None if image_masks is None else flatten_bn(
1517
- image_masks, concat=True))
1518
-
1519
- image_features_flat = self.vision_backbone(
1520
- images=images_flat.unsqueeze(0),
1521
- image_masks=(None if image_masks_flat is None else
1522
- image_masks_flat.unsqueeze(0)),
1523
- ).squeeze(0)
1524
-
1525
- # Reconstruct the batch dimension
1526
- num_crops_per_image = [nc.sum().item() for nc in num_crops]
1527
- image_features = image_features_flat.split(num_crops_per_image)
1528
- else:
1529
- image_features = self.vision_backbone(
1530
- images=images,
1531
- image_masks=image_masks,
1532
- )
1514
+ # Call the vision backbone on the whole batch at once
1515
+ images_flat = flatten_bn(images, concat=True)
1516
+ image_masks_flat = (None if image_masks is None else flatten_bn(
1517
+ image_masks, concat=True))
1518
+ feat_is_patch_flat = flatten_bn(feat_is_patch, concat=True)
1519
+
1520
+ image_features_flat = self.vision_backbone(
1521
+ images=images_flat.unsqueeze(0),
1522
+ image_masks=(None if image_masks_flat is None else
1523
+ image_masks_flat.unsqueeze(0)),
1524
+ ).squeeze(0)
1525
1526
# Only the features corresponding to patch tokens are relevant
1527
return [
1536
- feats[f_is_patch]
1537
- for feats, f_is_patch in zip(image_features, feat_is_patch)
1528
+ feats[f_is_patch] for feats, f_is_patch in zip(
1529
+ image_features_flat.split(num_crops.tolist()),
1530
+ feat_is_patch_flat.split(num_crops.tolist()),
1531
+ )
1532
]
1533
1534
def get_multimodal_embeddings(