A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Clean up MiniCPM-V/O code (#15337)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 25, 2025 at 18:22 UTC
a9e879b3167be9d04148ff6ee42313f71e0fd4c0
7 files changed
+531
-661
examples/offline_inference/vision_language.py
+1
@@ -361,6 +361,7 @@ def run_llava_next_video(questions: list[str],
361
engine_args = EngineArgs(
362
model="llava-hf/LLaVA-NeXT-Video-7B-hf",
363
max_model_len=8192,
364
+ max_num_seqs=2,
365
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
366
)
367
tests/models/decoder_only/vision_language/test_models.py
+45
-20
@@ -163,24 +163,24 @@ VLM_TEST_SETTINGS = {
163
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
164
),
165
#### Extended model tests
166
- # "aria": VLMTestInfo(
167
- # models=["rhymes-ai/Aria"],
168
- # test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
169
- # prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
170
- # img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
171
- # max_model_len=4096,
172
- # max_num_seqs=2,
173
- # auto_cls=AutoModelForImageTextToText,
174
- # single_image_prompts=IMAGE_ASSETS.prompts({
175
- # "stop_sign": "<vlm_image>Please describe the image shortly.",
176
- # "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
177
- # }),
178
- # multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
179
- # stop_str=["<|im_end|>"],
180
- # image_size_factors=[(0.10, 0.15)],
181
- # max_tokens=64,
182
- # marks=[large_gpu_mark(min_gb=64)],
183
- # ),
166
+ "aria": VLMTestInfo(
167
+ models=["rhymes-ai/Aria"],
168
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
169
+ prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
170
+ img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
171
+ max_model_len=4096,
172
+ max_num_seqs=2,
173
+ auto_cls=AutoModelForImageTextToText,
174
+ single_image_prompts=IMAGE_ASSETS.prompts({
175
+ "stop_sign": "<vlm_image>Please describe the image shortly.",
176
+ "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
177
+ }),
178
+ multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
179
+ stop_str=["<|im_end|>"],
180
+ image_size_factors=[(0.10, 0.15)],
181
+ max_tokens=64,
182
+ marks=[large_gpu_mark(min_gb=64)],
183
+ ),
184
"blip2": VLMTestInfo(
185
models=["Salesforce/blip2-opt-2.7b"],
186
test_type=VLMTestType.IMAGE,
@@ -352,6 +352,7 @@ VLM_TEST_SETTINGS = {
352
prompt_formatter=lambda vid_prompt: f"USER: {vid_prompt} ASSISTANT:",
353
num_video_frames=16,
354
max_model_len=4096,
355
+ max_num_seqs=2,
356
auto_cls=AutoModelForVision2Seq,
357
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
358
),
@@ -384,7 +385,18 @@ VLM_TEST_SETTINGS = {
385
),
386
"minicpmo_26": VLMTestInfo(
387
models=["openbmb/MiniCPM-o-2_6"],
387
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
388
+ test_type=(VLMTestType.IMAGE),
389
+ prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
390
+ img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
391
+ max_model_len=4096,
392
+ max_num_seqs=2,
393
+ get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
394
+ hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
395
+ patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
396
+ ),
397
+ "minicpmo_26_multi_image": VLMTestInfo(
398
+ models=["openbmb/MiniCPM-o-2_6"],
399
+ test_type=(VLMTestType.MULTI_IMAGE),
400
prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
401
img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
402
max_model_len=4096,
@@ -392,10 +404,22 @@ VLM_TEST_SETTINGS = {
404
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
405
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
406
patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
407
+ marks=[large_gpu_mark(min_gb=32)],
408
),
409
"minicpmv_26": VLMTestInfo(
410
models=["openbmb/MiniCPM-V-2_6"],
398
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
411
+ test_type=(VLMTestType.IMAGE),
412
+ prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
413
+ img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
414
+ max_model_len=4096,
415
+ max_num_seqs=2,
416
+ get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
417
+ hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
418
+ patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
419
+ ),
420
+ "minicpmv_26_multi_image": VLMTestInfo(
421
+ models=["openbmb/MiniCPM-V-2_6"],
422
+ test_type=(VLMTestType.MULTI_IMAGE),
423
prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
424
img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
425
max_model_len=4096,
@@ -403,6 +427,7 @@ VLM_TEST_SETTINGS = {
427
get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
428
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
429
patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
430
+ marks=[large_gpu_mark(min_gb=32)],
431
),
432
"molmo": VLMTestInfo(
433
models=["allenai/Molmo-7B-D-0924"],
tests/models/multimodal/processing/test_common.py
+41
-49
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import copy
3
from functools import partial
4
from typing import Optional, Union
5
@@ -29,7 +28,7 @@ def _test_processing_correctness(
28
hit_rate: float,
29
num_batches: int,
30
simplify_rate: float,
32
- ignore_mm_keys: Optional[list[str]] = None,
31
+ ignore_mm_keys: Optional[set[str]] = None,
32
):
33
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
34
model_info.check_available_online(on_fail="skip")
@@ -145,7 +144,7 @@ def _test_processing_correctness_hf(
144
baseline_processor: BaseMultiModalProcessor,
145
cached_processor: BaseMultiModalProcessor,
146
batch_idx: int,
148
- ignore_mm_keys: Optional[list[str]] = None,
147
+ ignore_mm_keys: Optional[set[str]] = None,
148
):
149
if model_config.hf_config.model_type in ("mllama", "whisper", "ultravox"):
150
# For some multimodal models, tokenizer will always add bos_token
@@ -167,11 +166,12 @@ def _test_processing_correctness_hf(
166
hf_processor_mm_kwargs={},
167
)
168
170
- assert _inputs_equal(
169
+ _assert_inputs_equal(
170
baseline_result,
171
cached_result,
173
- ignore_mm_keys,
174
- ), f"Failed ({batch_idx=}, {prompt=}, {mm_data=})"
172
+ ignore_mm_keys=ignore_mm_keys,
173
+ msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
174
+ )
175
176
baseline_tokenized_result = baseline_processor.apply(
177
token_prompt,
@@ -179,11 +179,12 @@ def _test_processing_correctness_hf(
179
hf_processor_mm_kwargs={},
180
)
181
182
- assert _inputs_equal(
182
+ _assert_inputs_equal(
183
baseline_result,
184
baseline_tokenized_result,
185
- ignore_mm_keys,
186
- ), f"Failed ({batch_idx=}, {prompt=}, {mm_data=})"
185
+ ignore_mm_keys=ignore_mm_keys,
186
+ msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
187
+ )
188
189
cached_tokenized_result = cached_processor.apply(
190
token_prompt,
@@ -191,11 +192,12 @@ def _test_processing_correctness_hf(
192
hf_processor_mm_kwargs={},
193
)
194
194
- assert _inputs_equal(
195
+ _assert_inputs_equal(
196
cached_result,
197
cached_tokenized_result,
197
- ignore_mm_keys,
198
- ), f"Failed ({batch_idx=}, {prompt=}, {mm_data=})"
198
+ ignore_mm_keys=ignore_mm_keys,
199
+ msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
200
+ )
201
202
203
def _test_processing_correctness_mistral(
@@ -206,7 +208,7 @@ def _test_processing_correctness_mistral(
208
baseline_processor: BaseMultiModalProcessor,
209
cached_processor: BaseMultiModalProcessor,
210
batch_idx: int,
209
- ignore_mm_keys: Optional[list[str]] = None,
211
+ ignore_mm_keys: Optional[set[str]] = None,
212
):
213
images = mm_data.get("image", [])
214
if not isinstance(images, list):
@@ -233,11 +235,12 @@ def _test_processing_correctness_mistral(
235
hf_processor_mm_kwargs={},
236
)
237
236
- assert _inputs_equal(
238
+ _assert_inputs_equal(
239
baseline_tokenized_result,
240
cached_tokenized_result,
239
- ignore_mm_keys,
240
- ), f"Failed ({batch_idx=}, {prompt=}, {mm_data=})"
241
+ ignore_mm_keys=ignore_mm_keys,
242
+ msg=f"Failed ({batch_idx=}, {prompt=}, {mm_data=})",
243
+ )
244
245
246
# yapf: disable
@@ -261,6 +264,7 @@ def _test_processing_correctness_mistral(
264
"TIGER-Lab/Mantis-8B-siglip-llama3",
265
"mistralai/Pixtral-12B-2409",
266
"mistral-community/pixtral-12b",
267
+ "openbmb/MiniCPM-Llama3-V-2_5",
268
"openbmb/MiniCPM-o-2_6",
269
"openbmb/MiniCPM-V-2_6",
270
"allenai/Molmo-7B-D-0924",
@@ -290,7 +294,7 @@ def test_processing_correctness(
294
# In Ultravox, the audio_features can be different depending on padding
295
# The slight difference should not be a problem though, since
296
# attention_mask lets us ignore the difference.
293
- ignore_mm_keys = ['audio_features']
297
+ ignore_mm_keys = {"audio_features"}
298
299
_test_processing_correctness(
300
model_id,
@@ -328,38 +332,26 @@ def test_processing_correctness_phi3v(
332
)
333
334
331
-def _inputs_equal(
335
+def _assert_inputs_equal(
336
a: MultiModalInputs,
337
b: MultiModalInputs,
334
- ignore_mm_keys: Optional[list[str]] = None,
338
+ *,
339
+ ignore_mm_keys: Optional[set[str]] = None,
340
+ msg: str = "",
341
):
336
- return _drop_mm_kwargs_keys(a, ignore_mm_keys) == _drop_mm_kwargs_keys(
337
- b, ignore_mm_keys)
338
-
339
-
340
-def _drop_mm_kwargs_keys(
341
- result: MultiModalInputs,
342
- ignore_mm_keys: Optional[list[str]] = None,
343
-) -> MultiModalInputs:
344
- """Drop specified keys from result['mm_kwargs'].
345
-
346
- This is mainly to avoid doing exact match of audio_features in ultravox.
347
-
348
- Args:
349
- result: Result to drop keys from
350
- ignore_mm_keys: List of keys to ignore, e.g. ['audio_features']
351
- """
352
- if not ignore_mm_keys:
353
- return result
354
-
355
- if 'mm_kwargs' in result:
356
- result = copy.deepcopy(result)
357
- mm_kwargs = result['mm_kwargs']
358
- for key in ignore_mm_keys:
359
- mm_kwargs.pop(key, None)
360
- for items in mm_kwargs._items_by_modality.values():
361
- for item in items:
362
- for key in ignore_mm_keys:
363
- item.pop(key, None)
364
-
365
- return result
342
+ if ignore_mm_keys is None:
343
+ ignore_mm_keys = set()
344
+
345
+ if msg is None:
346
+ assert "mm_kwargs" in a and "mm_kwargs" in b
347
+ else:
348
+ assert "mm_kwargs" in a and "mm_kwargs" in b, msg
349
+
350
+ for key in ignore_mm_keys:
351
+ a["mm_kwargs"].pop(key, None)
352
+ b["mm_kwargs"].pop(key, None)
353
+
354
+ if msg is None:
355
+ assert a == b
356
+ else:
357
+ assert a == b, msg
vllm/model_executor/models/gemma3_mm.py
-2
@@ -295,8 +295,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
295
296
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
297
if (images := mm_data.get("images")) is not None:
298
- assert isinstance(images, list)
299
-
298
parsed_images = (self._get_data_parser().parse_mm_data({
299
"image":
300
images
vllm/model_executor/models/minicpmo.py
+149
-191
@@ -23,7 +23,7 @@
23
# limitations under the License.
24
"""Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25
from collections.abc import Iterable, Mapping, Sequence
26
-from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,
26
+from typing import (Any, Callable, Dict, Literal, Optional, Set, Tuple,
27
TypedDict, Union)
28
29
import torch
@@ -43,24 +43,26 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
43
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
44
from vllm.multimodal.profiling import ProcessorInputs
45
from vllm.sequence import IntermediateTensors
46
+from vllm.utils import flatten_2d_lists
47
48
from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
49
MiniCPMVMultiModalDataParser,
50
MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo,
51
_minicpmv_field_config)
51
-from .utils import AutoWeightsLoader, cast_overflow_tensors, maybe_prefix
52
+from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn,
53
+ maybe_prefix)
54
55
CPU_DEVICE = torch.device("cpu")
56
57
58
class MiniCPMOAudioFeatureInputs(TypedDict):
59
type: Literal["audio_features"]
58
- data: torch.Tensor
60
+ audio_features: torch.Tensor
61
"""
62
Shape: `(batch_size * num_audios * num_slices, num_channels, length)`
63
Slice here means chunk. Audio that is too long will be split into slices,
64
which is the same as image.
63
- Padding is used therefore `data` is `torch.Tensor`.
65
+ Padding is used therefore `audio_features` is `torch.Tensor`.
66
"""
67
68
audio_feature_lens: torch.Tensor
@@ -68,7 +70,7 @@ class MiniCPMOAudioFeatureInputs(TypedDict):
70
Shape: `(batch_size * num_audios * num_slices)`
71
72
This should be feature length of each audio slice,
71
- which equals to `data.shape[-1]`
73
+ which equals to `audio_features.shape[-1]`
74
"""
75
76
audio_bounds: torch.Tensor
@@ -81,7 +83,7 @@ class MiniCPMOAudioFeatureInputs(TypedDict):
83
84
class MiniCPMOAudioEmbeddingInputs(TypedDict):
85
type: Literal["audio_embeds"]
84
- data: List[torch.Tensor]
86
+ audio_embeds: torch.Tensor
87
"""
88
Shape: `(batch_size * num_images * num_slices, hidden_size)`
89
@@ -102,18 +104,11 @@ MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
104
105
106
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
105
- audio_num_slices = hf_inputs.get("audio_num_slices", torch.empty(0))
106
-
107
return dict(
108
**_minicpmv_field_config(hf_inputs),
109
- audio_features=MultiModalFieldConfig.flat_from_sizes(
110
- "audio", audio_num_slices),
111
- audio_feature_lens=MultiModalFieldConfig.flat_from_sizes(
112
- "audio", audio_num_slices),
113
- audio_num_slices=MultiModalFieldConfig.batched("audio"),
114
- audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
115
- audio_embeds=MultiModalFieldConfig.flat_from_sizes(
116
- "audio", audio_num_slices),
109
+ audio_features=MultiModalFieldConfig.batched("audio"),
110
+ audio_feature_lens=MultiModalFieldConfig.batched("audio"),
111
+ audio_embeds=MultiModalFieldConfig.batched("audio"),
112
)
113
114
@@ -153,9 +148,6 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
148
class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
149
audio_pattern = "(<audio>./</audio>)"
150
156
- def get_supported_mm_modalities(self) -> List[str]:
157
- return ["image", "video", "audio"]
158
-
151
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
152
return {"image": None, "video": None, "audio": None}
153
@@ -277,95 +269,47 @@ class MiniCPMOMultiModalProcessor(
269
mm_data: Mapping[str, object],
270
mm_kwargs: Mapping[str, object],
271
) -> Mapping[str, NestedTensors]:
280
- mm_data = dict(mm_data)
281
-
282
- audios = mm_data.pop("audios", [])
283
- audio_embeds = mm_data.pop("audio_embeds", [])
284
- if isinstance(audios, (list, torch.Tensor)) and len(audios) > 0:
285
- audio_outputs = {
286
- "audio_lens": [],
287
- "audio_features": [],
288
- "audio_feature_lens": [],
289
- "audio_num_segments": []
290
- }
291
- for audio in audios:
292
- single_audio_outputs = super().call_base_hf_processor(
293
- prompt=self.info.audio_pattern,
294
- mm_data={
295
- "audios": audio,
296
- "chunk_input": True
297
- },
298
- mm_kwargs=mm_kwargs)
299
- audio_outputs["audio_lens"].append(len(audio))
300
- audio_outputs["audio_features"].append(
301
- single_audio_outputs["audio_features"])
302
- audio_outputs["audio_num_segments"].append(
303
- len(single_audio_outputs["audio_feature_lens"][0]))
304
- audio_outputs["audio_feature_lens"] += \
305
- single_audio_outputs["audio_feature_lens"]
306
- audio_outputs["audio_features"] = [
307
- audio_feature for single_audio_features in \
308
- audio_outputs["audio_features"]
309
- for audio_feature in single_audio_features
310
- ]
311
- audio_outputs["audio_feature_lens"] = torch.cat(
312
- audio_outputs["audio_feature_lens"])
313
- elif len(audio_embeds):
314
- audio_outputs = {
315
- "audio_lens": [
316
- self.info.get_audio_len_by_num_chunks(
317
- sum(chunk_embeds.shape[0]
318
- for chunk_embeds in single_audio_embeds))
319
- for single_audio_embeds in audio_embeds
320
- ],
321
- "audio_embeds": [
322
- chunk_embeds for single_audio_embeds in audio_embeds
323
- for chunk_embeds in single_audio_embeds
324
- ],
325
- "audio_num_segments": [
326
- len(single_audio_embeds)
327
- for single_audio_embeds in audio_embeds
328
- ]
329
- }
330
- else:
331
- audio_outputs = {}
332
- return audio_outputs
272
+ if (audios := mm_data.get("audios")) is None:
273
+ return {}
274
+
275
+ parsed_audios = (self._get_data_parser().parse_mm_data({
276
+ "audio": audios
277
+ }).get_items("audio", AudioProcessorItems))
278
+
279
+ audio_inputs = self._base_call_hf_processor(
280
+ prompts=[self.info.audio_pattern] * len(parsed_audios),
281
+ mm_data={"audios": [[audio] for audio in parsed_audios]},
282
+ mm_kwargs={
283
+ **mm_kwargs, "chunk_input": True
284
+ },
285
+ out_keys={"audio_features", "audio_feature_lens"},
286
+ )
287
+
288
+ # Avoid padding since we need the output for each audio to be
289
+ # independent of other audios for the cache to work correctly
290
+ unpadded_audio_features = [
291
+ feat[:, :feature_len] for feat, feature_len in zip(
292
+ audio_inputs["audio_features"],
293
+ audio_inputs["audio_feature_lens"],
294
+ )
295
+ ]
296
+ audio_inputs["audio_features"] = unpadded_audio_features
297
+
298
+ return audio_inputs
299
300
def get_placeholder_match_pattern(self) -> str:
301
return r"\(<(image|video|audio)>./</\1>\)"
302
337
- def get_placeholder_split_pattern(self) -> str:
338
- return r"\(<(?:image|video|audio)>./</(?:image|video|audio)>\)"
339
-
303
def process_mm_inputs(
304
self,
305
mm_data: Mapping[str, object],
306
mm_kwargs: Mapping[str, object],
344
- ) -> Mapping[str, Mapping[str, NestedTensors]]:
307
+ ) -> Mapping[str, NestedTensors]:
308
return {
346
- "image": self.process_images(mm_data, mm_kwargs),
347
- "video": self.process_videos(mm_data, mm_kwargs),
348
- "audio": self.process_audios(mm_data, mm_kwargs),
309
+ **super().process_mm_inputs(mm_data, mm_kwargs),
310
+ **self.process_audios(mm_data, mm_kwargs),
311
}
312
351
- def get_modality_num_counter(self, modality: str) -> str:
352
- if modality == "audio":
353
- return "audio_lens"
354
- return super().get_modality_num_counter(modality)
355
-
356
- def get_num_slices_by_modality(self, inputs: Dict[str, object],
357
- modality: str, index: int) -> int:
358
- if modality == "audio":
359
- return inputs["audio"]["audio_num_segments"][index]
360
- return super().get_num_slices_by_modality(inputs, modality, index)
361
-
362
- def get_prompt_texts_by_modality(self, inputs: Dict[str, object],
363
- modality: str, index: int) -> str:
364
- if modality == "audio":
365
- return self.get_audio_prompt_texts(
366
- inputs["audio"]["audio_lens"][index])
367
- return super().get_prompt_texts_by_modality(inputs, modality, index)
368
-
313
def _get_prompt_updates(
314
self,
315
mm_items: MultiModalDataItems,
@@ -622,86 +566,84 @@ class MiniCPMO(MiniCPMV2_6):
566
# Copied from HF repo of MiniCPM-o-2_6,
567
# designed for batched inputs and outputs
568
def get_audio_hidden_states(self, data: MiniCPMOAudioInputs,
625
- chunk_length: int) -> torch.Tensor:
569
+ chunk_length: int) -> list[torch.Tensor]:
570
wavforms = data.get(
627
- "data",
571
+ "audio_features",
572
[]) # (bs, 80, frames) or [], multi audios need filled in advance
573
audio_feature_lens_raw = [data.get("audio_feature_lens",
574
[])] # list, [[x1, x2], [y1], [z1]]
575
632
- # exist audio
633
- if len(wavforms) > 0:
634
- audio_feature_lens = torch.hstack(audio_feature_lens_raw)
635
- batch_size, _, max_mel_seq_len = wavforms.shape
636
- max_seq_len = (max_mel_seq_len - 1) // 2 + 1
637
-
638
- # Create a sequence tensor of shape (batch_size, max_seq_len)
639
- seq_range = (torch.arange(
640
- 0,
641
- max_seq_len,
642
- dtype=audio_feature_lens.dtype,
643
- device=audio_feature_lens.device).unsqueeze(0).expand(
644
- batch_size, max_seq_len))
645
- lengths_expand = audio_feature_lens.unsqueeze(1).expand(
646
- batch_size, max_seq_len)
647
- # Create mask
648
- padding_mask = seq_range >= lengths_expand # 1 for padded values
649
-
650
- audio_attention_mask_ = padding_mask.view(
651
- batch_size, 1, 1, max_seq_len).expand(batch_size, 1,
652
- max_seq_len, max_seq_len)
653
- audio_attention_mask = audio_attention_mask_.to(
654
- dtype=self.apm.conv1.weight.dtype,
655
- device=self.apm.conv1.weight.device)
656
-
657
- if chunk_length > 0:
658
- chunk_num_frame = int(chunk_length * 50)
659
- chunk_mask = self.subsequent_chunk_mask(
660
- size=max_seq_len,
661
- chunk_size=chunk_num_frame,
662
- num_left_chunks=-1,
663
- device=audio_attention_mask_.device,
664
- )
665
- audio_attention_mask_ = torch.logical_or(
666
- audio_attention_mask_, torch.logical_not(chunk_mask))
667
-
668
- audio_attention_mask[audio_attention_mask_] = float("-inf")
669
- audio_states = self.apm(
670
- wavforms, attention_mask=audio_attention_mask).hidden_states[
671
- self.audio_encoder_layer]
672
- audio_embeds = self.audio_projection_layer(audio_states)
673
-
674
- audio_embeds = audio_embeds.transpose(1, 2)
675
- audio_embeds = self.audio_avg_pooler(audio_embeds)
676
- audio_embeds = audio_embeds.transpose(1, 2)
677
-
678
- _, feature_lens_after_pooling = \
679
- self._get_feat_extract_output_lengths(audio_feature_lens)
680
-
681
- num_audio_tokens = feature_lens_after_pooling
682
-
683
- final_audio_embeds = []
684
- idx = 0
685
- for i in range(len(audio_feature_lens_raw)):
686
- target_audio_embeds = []
687
- for _ in range(len(audio_feature_lens_raw[i])):
688
- target_audio_embeds.append(
689
- audio_embeds[idx, :num_audio_tokens[idx], :])
690
- idx += 1
691
- final_audio_embeds.append(target_audio_embeds)
692
- return final_audio_embeds
693
- else:
576
+ if len(wavforms) == 0:
577
return []
578
579
+ audio_feature_lens = torch.hstack(audio_feature_lens_raw)
580
+ batch_size, _, max_mel_seq_len = wavforms.shape
581
+ max_seq_len = (max_mel_seq_len - 1) // 2 + 1
582
+
583
+ # Create a sequence tensor of shape (batch_size, max_seq_len)
584
+ seq_range = (torch.arange(
585
+ 0,
586
+ max_seq_len,
587
+ dtype=audio_feature_lens.dtype,
588
+ device=audio_feature_lens.device).unsqueeze(0).expand(
589
+ batch_size, max_seq_len))
590
+ lengths_expand = audio_feature_lens.unsqueeze(1).expand(
591
+ batch_size, max_seq_len)
592
+ # Create mask
593
+ padding_mask = seq_range >= lengths_expand # 1 for padded values
594
+
595
+ audio_attention_mask_ = padding_mask.view(
596
+ batch_size, 1, 1, max_seq_len).expand(batch_size, 1, max_seq_len,
597
+ max_seq_len)
598
+ audio_attention_mask = audio_attention_mask_.to(
599
+ dtype=self.apm.conv1.weight.dtype,
600
+ device=self.apm.conv1.weight.device)
601
+
602
+ if chunk_length > 0:
603
+ chunk_num_frame = int(chunk_length * 50)
604
+ chunk_mask = self.subsequent_chunk_mask(
605
+ size=max_seq_len,
606
+ chunk_size=chunk_num_frame,
607
+ num_left_chunks=-1,
608
+ device=audio_attention_mask_.device,
609
+ )
610
+ audio_attention_mask_ = torch.logical_or(
611
+ audio_attention_mask_, torch.logical_not(chunk_mask))
612
+
613
+ audio_attention_mask[audio_attention_mask_] = float("-inf")
614
+ audio_states = self.apm(
615
+ wavforms, attention_mask=audio_attention_mask).hidden_states[
616
+ self.audio_encoder_layer]
617
+ audio_embeds = self.audio_projection_layer(audio_states)
618
+
619
+ audio_embeds = audio_embeds.transpose(1, 2)
620
+ audio_embeds = self.audio_avg_pooler(audio_embeds)
621
+ audio_embeds = audio_embeds.transpose(1, 2)
622
+
623
+ _, feature_lens_after_pooling = \
624
+ self._get_feat_extract_output_lengths(audio_feature_lens)
625
+
626
+ num_audio_tokens = feature_lens_after_pooling
627
+
628
+ final_audio_embeds = []
629
+ idx = 0
630
+ for i in range(len(audio_feature_lens_raw)):
631
+ target_audio_embeds = []
632
+ for _ in range(len(audio_feature_lens_raw[i])):
633
+ target_audio_embeds.append(
634
+ audio_embeds[idx, :num_audio_tokens[idx], :])
635
+ idx += 1
636
+ final_audio_embeds.append(target_audio_embeds)
637
+ return final_audio_embeds
638
+
639
def get_embedding_with_audios(self, vlm_embedding: torch.Tensor,
697
- audio_inputs: Optional[MiniCPMOAudioInputs],
640
+ audio_inputs: MiniCPMOAudioInputs,
641
chunk_length: int) -> torch.Tensor:
642
device, dtype = vlm_embedding.device, vlm_embedding.dtype
643
if audio_inputs["type"] == "audio_embeds":
701
- audio_embeddings = audio_inputs["data"]
644
audio_embeddings = [
703
- audio_embeddings[i].to(device=device, dtype=dtype)
704
- for i in range(len(audio_embeddings))
645
+ item.to(device=device, dtype=dtype)
646
+ for item in audio_inputs["audio_embeds"]
647
]
648
else:
649
audio_embeddings = self.get_audio_hidden_states(
@@ -746,40 +688,56 @@ class MiniCPMO(MiniCPMV2_6):
688
689
def _parse_and_validate_audio_inputs(
690
self, input_ids: torch.Tensor,
749
- **kwargs: object) -> Tuple[MiniCPMOAudioInputs]:
750
- audio_features = kwargs.pop("audio_features", [])
751
- audio_feature_lens = kwargs.pop("audio_feature_lens", [])
691
+ **kwargs: object) -> Optional[MiniCPMOAudioInputs]:
692
+ audio_features = kwargs.pop("audio_features", None)
693
audio_embeds = kwargs.pop("audio_embeds", None)
753
- audio_start_id = kwargs.pop("audio_start_id", None)
754
- audio_end_id = kwargs.pop("audio_end_id", None)
694
+
695
+ if audio_features is None and audio_embeds is None:
696
+ return None
697
+
698
+ audio_start_id = kwargs.pop("audio_start_id")
699
+ if not isinstance(audio_start_id, torch.Tensor):
700
+ raise ValueError("Incorrect type of audio_start_id. "
701
+ f"Got type: {type(audio_start_id)}")
702
+
703
+ audio_end_id = kwargs.pop("audio_end_id")
704
+ if not isinstance(audio_end_id, torch.Tensor):
705
+ raise ValueError("Incorrect type of audio_end_id. "
706
+ f"Got type: {type(audio_end_id)}")
707
+
708
if audio_embeds is not None:
756
- audio_embeds = [
757
- audio_embeds[i][j] for i in range(len(audio_embeds))
758
- for j in range(len(audio_embeds[i]))
759
- ]
709
+ if not isinstance(audio_embeds, (torch.Tensor, list)):
710
+ raise ValueError("Incorrect type of audio_embeds. "
711
+ f"Got type: {type(audio_embeds)}")
712
+
713
return MiniCPMOAudioEmbeddingInputs(
714
+ type="audio_embeds",
715
+ audio_embeds=flatten_bn(flatten_2d_lists(audio_embeds),
716
+ concat=True),
717
audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
718
audio_end_id),
763
- data=audio_embeds,
764
- type="audio_embeds")
765
- if len(audio_features) > 0:
766
- audio_features_all = [
767
- i.permute(1, 0) for audio_feature in audio_features
768
- for i in audio_feature
769
- ]
770
- audio_features = torch.nn.utils.rnn.pad_sequence(
771
- audio_features_all, batch_first=True,
772
- padding_value=0.0).permute(0, 2, 1)
773
- audio_feature_lens = torch.cat(
774
- [item for item in audio_feature_lens])
719
+ )
720
+
721
+ if audio_features is not None:
722
+ if not isinstance(audio_features, (torch.Tensor, list)):
723
+ raise ValueError("Incorrect type of audio_features. "
724
+ f"Got type: {type(audio_features)}")
725
+
726
+ audio_feature_lens = kwargs.pop("audio_feature_lens")
727
+ if not isinstance(audio_feature_lens, (torch.Tensor, list)):
728
+ raise ValueError("Incorrect type of audio_feature_lens. "
729
+ f"Got type: {type(audio_feature_lens)}")
730
731
return MiniCPMOAudioFeatureInputs(
732
+ type="audio_features",
733
+ audio_features=flatten_bn(audio_features, concat=True),
734
+ audio_feature_lens=flatten_bn(
735
+ flatten_2d_lists(audio_feature_lens), concat=True),
736
audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
737
audio_end_id),
779
- data=audio_features,
780
- audio_feature_lens=audio_feature_lens,
781
- type="audio_features")
782
- return None
738
+ )
739
+
740
+ raise AssertionError("This line should be unreachable.")
741
742
def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
743
**kwargs: object):
@@ -803,7 +761,7 @@ class MiniCPMO(MiniCPMV2_6):
761
else:
762
image_inputs, audio_inputs = \
763
self._parse_and_validate_inputs(input_ids, **kwargs)
806
- vlm_embeddings, _ = self.get_embedding_with_vision(
764
+ vlm_embeddings = self.get_embedding_with_vision(
765
input_ids, image_inputs)
766
767
if audio_inputs is not None:
vllm/model_executor/models/minicpmv.py
+288
-399
@@ -24,6 +24,7 @@
24
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
25
import math
26
import re
27
+from collections import defaultdict
28
from collections.abc import Iterable, Mapping, Sequence
29
from functools import cached_property, partial
30
from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,
@@ -63,11 +64,12 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
64
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
65
from vllm.platforms import current_platform
66
from vllm.sequence import IntermediateTensors
67
+from vllm.utils import flatten_2d_lists
68
69
from .idefics2_vision_model import Idefics2VisionTransformer
70
from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,
71
SupportsV0Only)
70
-from .utils import AutoWeightsLoader, maybe_prefix
72
+from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix
73
74
CPU_DEVICE = torch.device("cpu")
75
@@ -76,7 +78,7 @@ RawImageType = Union[Image.Image, torch.Tensor]
78
79
class MiniCPMVImagePixelInputs(TypedDict):
80
type: Literal["pixel_values"]
79
- data: List[torch.Tensor]
81
+ pixel_values: list[torch.Tensor]
82
"""
83
Shape: `(batch_size * num_images * num_slices, num_channels, height, width)`
84
@@ -101,7 +103,7 @@ class MiniCPMVImagePixelInputs(TypedDict):
103
104
class MiniCPMVImageEmbeddingInputs(TypedDict):
105
type: Literal["image_embeds"]
104
- data: torch.Tensor
106
+ image_embeds: torch.Tensor
107
"""
108
Shape: `(batch_size * num_images * num_slices,
109
image_feature_size, hidden_size)`
@@ -231,26 +233,15 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
233
234
235
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
234
- image_num_slices = hf_inputs.get("image_num_slices", torch.empty(0))
235
- video_num_slices = hf_inputs.get("video_num_slices", torch.empty(0))
236
-
236
return dict(
238
- pixel_values=MultiModalFieldConfig.flat_from_sizes(
239
- "image", image_num_slices),
237
+ pixel_values=MultiModalFieldConfig.batched("image"),
238
image_sizes=MultiModalFieldConfig.batched("image"),
241
- tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
242
- "image", image_num_slices),
243
- image_num_slices=MultiModalFieldConfig.batched("image"),
244
- image_embeds=MultiModalFieldConfig.flat_from_sizes(
245
- "image", image_num_slices),
246
- video_pixel_values=MultiModalFieldConfig.flat_from_sizes(
247
- "video", video_num_slices),
239
+ tgt_sizes=MultiModalFieldConfig.batched("image"),
240
+ image_embeds=MultiModalFieldConfig.batched("image"),
241
+ video_pixel_values=MultiModalFieldConfig.batched("video"),
242
video_image_sizes=MultiModalFieldConfig.batched("video"),
249
- video_tgt_sizes=MultiModalFieldConfig.flat_from_sizes(
250
- "video", video_num_slices),
251
- video_embeds=MultiModalFieldConfig.flat_from_sizes(
252
- "video", video_num_slices),
253
- video_num_slices=MultiModalFieldConfig.batched("video"),
243
+ video_tgt_sizes=MultiModalFieldConfig.batched("video"),
244
+ video_embeds=MultiModalFieldConfig.batched("video"),
245
)
246
247
@@ -356,12 +347,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
347
def get_model_version(self):
348
return get_version_by_config(self.get_hf_config())
349
359
- def get_supported_mm_modalities(self) -> List[str]:
360
- if self.get_model_version() == (2, 6):
361
- return ["image", "video"]
362
- else:
363
- return ["image"]
364
-
350
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
351
if self.get_model_version() == (2, 6):
352
return {"image": None, "video": None}
@@ -526,187 +511,123 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
511
def get_image_prompt_texts(self,
512
image_size: ImageSize,
513
image_idx: int = 0) -> str:
529
- prompt_texts = self.get_slice_image_placeholder(image_size,
530
- image_idx=image_idx)
531
- return prompt_texts
514
+ return self.get_slice_image_placeholder(image_size,
515
+ image_idx=image_idx)
516
517
def get_video_prompt_texts(self, image_size: ImageSize,
518
num_frames: int) -> str:
535
- prompt_texts = "".join(
536
- self.get_slice_image_placeholder(
537
- image_size=image_size,
538
- image_idx=0,
539
- max_slice_nums=self.info.get_video_max_slice_num(),
540
- use_image_id=False) for image_idx in range(num_frames))
541
- return prompt_texts
519
+ return self.get_slice_image_placeholder(
520
+ image_size=image_size,
521
+ image_idx=0,
522
+ max_slice_nums=self.info.get_video_max_slice_num(),
523
+ use_image_id=False,
524
+ ) * num_frames
525
526
def get_special_tokens(self) -> Dict[str, torch.Tensor]:
527
tokenizer = self.info.get_tokenizer()
528
+
529
special_tokens = {
546
- "im_start_id": torch.tensor(tokenizer.im_start_id),
547
- "im_end_id": torch.tensor(tokenizer.im_end_id)
530
+ "im_start_id": tokenizer.im_start_id,
531
+ "im_end_id": tokenizer.im_end_id,
532
}
533
if hasattr(tokenizer, "slice_start_id"):
550
- special_tokens["slice_start_id"] = torch.tensor(
551
- tokenizer.slice_start_id)
552
- special_tokens["slice_end_id"] = torch.tensor(
553
- tokenizer.slice_end_id)
554
- return special_tokens
555
-
556
- @staticmethod
557
- def repack_processor_outputs(outputs: Any) -> BatchFeature:
558
- valid_keys = ["pixel_values", "image_sizes", "tgt_sizes"]
559
- outputs = {key: outputs[key][0] for key in valid_keys}
560
- return outputs
534
+ special_tokens["slice_start_id"] = tokenizer.slice_start_id
535
+ special_tokens["slice_end_id"] = tokenizer.slice_end_id
536
+
537
+ return {k: torch.tensor(v) for k, v in special_tokens.items()}
538
539
def process_images(
540
self,
541
mm_data: Mapping[str, object],
542
mm_kwargs: Mapping[str, object],
543
) -> Mapping[str, NestedTensors]:
567
- mm_data = dict(mm_data)
568
-
569
- images = mm_data.pop("images", [])
570
- image_embeds = mm_data.pop("image_embeds", [])
571
- if isinstance(images, Image.Image):
572
- images = [images]
573
- if isinstance(images, (list, torch.Tensor)) and len(images) > 0:
574
- image_outputs = super()._call_hf_processor(
575
- prompt=self.info.image_pattern * len(images),
576
- mm_data={"images": images},
577
- mm_kwargs=mm_kwargs)
578
- image_outputs = self.repack_processor_outputs(image_outputs)
579
- elif len(image_embeds) > 0:
580
- image_sizes = mm_data.pop("image_sizes", None)
581
- image_outputs = {
582
- "image_embeds": torch.cat(image_embeds),
583
- "image_sizes": image_sizes
584
- }
585
- else:
586
- image_outputs = {}
587
- return image_outputs
544
+ if (images := mm_data.get("images")) is None:
545
+ return {}
546
+
547
+ parsed_images = (self._get_data_parser().parse_mm_data({
548
+ "image": images
549
+ }).get_items("image", ImageProcessorItems))
550
+
551
+ return self._base_call_hf_processor(
552
+ prompts=[self.info.image_pattern] * len(parsed_images),
553
+ mm_data={"images": [[image] for image in parsed_images]},
554
+ mm_kwargs=mm_kwargs,
555
+ out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
556
+ )
557
558
def process_videos(
559
self,
560
mm_data: Mapping[str, object],
561
mm_kwargs: Mapping[str, object],
562
) -> Mapping[str, NestedTensors]:
594
- mm_data = dict(mm_data)
595
-
596
- videos = mm_data.pop("videos", [])
597
- video_embeds = mm_data.pop("video_embeds", [])
598
- if len(videos) > 0 and isinstance(videos[0], Image.Image):
599
- videos = [videos]
600
- if isinstance(videos, list) and len(videos) > 0:
601
- video_outputs = {
602
- "video_pixel_values": [],
603
- "video_image_sizes": [],
604
- "video_tgt_sizes": [],
605
- "num_frames": []
606
- }
607
- for video in videos:
608
- parsed_video = []
609
- for frame in video:
610
- if isinstance(frame, np.ndarray):
611
- parsed_video.append(Image.fromarray(frame))
612
- else:
613
- parsed_video.append(frame)
614
- video = parsed_video
615
- single_video_outputs = super()._call_hf_processor(
616
- prompt=self.info.image_pattern * len(video),
617
- mm_data={"images": video},
618
- mm_kwargs={
619
- **mm_kwargs, "max_slice_nums":
620
- self.info.get_video_max_slice_num()
621
- })
622
- video_outputs["num_frames"].append(len(video))
623
- for key in single_video_outputs:
624
- if "video_" + key in video_outputs:
625
- if key == "image_sizes":
626
- video_outputs["video_" + key].append(
627
- single_video_outputs[key][0][0])
628
- else:
629
- video_outputs["video_" +
630
- key] += single_video_outputs[key][0]
631
- elif len(video_embeds):
632
- image_sizes = mm_data.pop("image_sizes", None)
633
- num_frames = mm_data.pop("num_frames", None)
634
- video_outputs = {
635
- "video_embeds": torch.cat(video_embeds),
636
- "video_image_sizes": image_sizes,
637
- "num_frames": num_frames
638
- }
639
- else:
640
- video_outputs = {}
641
- return video_outputs
563
+ if (videos := mm_data.get("videos")) is None:
564
+ return {}
565
+
566
+ parsed_videos = (self._get_data_parser().parse_mm_data({
567
+ "video": videos
568
+ }).get_items("video", VideoProcessorItems))
569
+
570
+ max_slice_num = self.info.get_video_max_slice_num()
571
+
572
+ video_inputs = self._base_call_hf_processor(
573
+ prompts=[
574
+ self.info.image_pattern * len(video) for video in parsed_videos
575
+ ],
576
+ mm_data={"images": list(parsed_videos)},
577
+ mm_kwargs={
578
+ **mm_kwargs, "max_slice_nums": max_slice_num
579
+ },
580
+ out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
581
+ )
582
+
583
+ return {f"video_{k}": v for k, v in video_inputs.items()}
584
585
def get_placeholder_match_pattern(self) -> str:
586
return r"\(<(image|video)>./</\1>\)"
587
646
- def get_placeholder_split_pattern(self) -> str:
647
- return r"\(<(?:image|video)>./</(?:image|video)>\)"
648
-
588
def process_mm_inputs(
589
self,
590
mm_data: Mapping[str, object],
591
mm_kwargs: Mapping[str, object],
653
- ) -> Mapping[str, Mapping[str, NestedTensors]]:
592
+ ) -> Mapping[str, NestedTensors]:
593
return {
655
- "image": self.process_images(mm_data, mm_kwargs),
656
- "video": self.process_videos(mm_data, mm_kwargs),
594
+ **self.process_images(mm_data, mm_kwargs),
595
+ **self.process_videos(mm_data, mm_kwargs),
596
}
597
659
- def get_input_modalities(self, mm_data) -> List[str]:
660
- supported_mm_modalities = self.info.get_supported_mm_modalities()
661
- input_modalities = []
662
- for modality in supported_mm_modalities:
663
- if modality in mm_data and mm_data[modality] != {}:
664
- input_modalities.append(modality)
665
- return input_modalities
666
-
667
- def get_modality_num_counter(self, modality: str) -> str:
668
- if modality == "image":
669
- return "image_sizes"
670
- elif modality == "video":
671
- return "video_image_sizes"
672
-
673
- raise NotImplementedError(modality)
674
-
675
- def get_num_slices_by_modality(self, inputs: dict[str, Any], modality: str,
676
- index: int) -> int:
677
- if modality == "image":
678
- return self.info.get_image_slice_nums(
679
- inputs[modality]["image_sizes"][index],
680
- self.info.get_max_slice_num())
681
- elif modality == "video":
682
- return self.info.get_image_slice_nums(
683
- inputs[modality]["video_image_sizes"][index],
684
- self.info.get_video_max_slice_num()
685
- ) * inputs[modality]["num_frames"][index]
686
- else:
687
- raise ValueError(f"Unexpected modality: {modality}")
688
-
689
- def get_prompt_texts_by_modality(self, inputs: dict[str, Any],
690
- modality: str, index: int) -> str:
691
- if modality == "image":
692
- return self.get_image_prompt_texts(
693
- inputs["image"]["image_sizes"][index], index)
694
- elif modality == "video":
695
- return self.get_video_prompt_texts(
696
- inputs["video"]["video_image_sizes"][index],
697
- inputs["video"]["num_frames"][index])
698
- else:
699
- raise ValueError(f"Unexpected modality: {modality}")
700
-
701
- def call_base_hf_processor(
598
+ def _base_call_hf_processor(
599
self,
703
- prompt: str,
704
- mm_data: Mapping[str, object],
600
+ prompts: list[str],
601
+ mm_data: Mapping[str, Sequence[object]],
602
mm_kwargs: Mapping[str, object],
706
- ) -> BatchFeature:
707
- return super()._call_hf_processor(prompt=prompt,
708
- mm_data=mm_data,
709
- mm_kwargs=mm_kwargs)
603
+ *,
604
+ out_keys: set[str],
605
+ ) -> Mapping[str, NestedTensors]:
606
+ # This processor supports zipping prompt and mm_data together
607
+ if self.info.get_model_version() == (2, 6):
608
+ inputs = super()._call_hf_processor(
609
+ prompt=prompts, # type: ignore
610
+ mm_data=mm_data,
611
+ mm_kwargs=mm_kwargs,
612
+ )
613
+ else:
614
+ inputs = defaultdict[str, list[torch.Tensor]](list)
615
+
616
+ for i, prompt in enumerate(prompts):
617
+ inputs_one = super()._call_hf_processor(
618
+ prompt=prompt,
619
+ mm_data={
620
+ k: v[i]
621
+ for k, v in mm_data.items()
622
+ },
623
+ mm_kwargs=mm_kwargs,
624
+ )
625
+
626
+ for k, v in inputs_one.items():
627
+ assert len(v) == 1, (k, len(v))
628
+ inputs[k].append(v[0])
629
+
630
+ return {k: inputs[k] for k in out_keys}
631
632
def _call_hf_processor(
633
self,
@@ -717,35 +638,12 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
638
# Do not support combination inputs of images and videos for now
639
# Try to handle interleaved multimodal data
640
tokenizer = self.info.get_tokenizer()
720
- inputs = self.process_mm_inputs(mm_data, mm_kwargs)
721
- mm_input_modalities = self.get_input_modalities(inputs)
722
-
723
- num_mm_slices_lst = {
724
- modality: list[int]()
725
- for modality in mm_input_modalities
726
- }
727
- for modality in mm_input_modalities:
728
- num_counter_key = self.get_modality_num_counter(modality)
729
- for index in range(len(inputs[modality][num_counter_key])):
730
- num_mm_slices_lst[modality].append(
731
- self.get_num_slices_by_modality(inputs, modality, index))
732
-
733
- num_mm_slices = {
734
- modality: torch.tensor(v)
735
- for modality, v in num_mm_slices_lst.items()
736
- }
641
+ mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)
642
643
return BatchFeature({
739
- "input_ids": np.array([tokenizer.encode(prompt)]),
740
- **{
741
- key: value
742
- for modality in inputs
743
- for key, value in inputs[modality].items()
744
- },
745
- **{
746
- f"{modality}_num_slices": num_mm_slices[modality]
747
- for modality in mm_input_modalities
748
- }
644
+ "input_ids":
645
+ torch.tensor([tokenizer.encode(prompt)]),
646
+ **mm_inputs,
647
})
648
649
def _hf_processor_applies_updates(
@@ -810,7 +708,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
708
hf_processor_mm_kwargs: Mapping[str, object],
709
return_mm_hashes: bool = False,
710
) -> MultiModalInputs:
813
- supported_mm_modalities = self.info.get_supported_mm_modalities()
711
if isinstance(prompt, list):
712
prompt = self.info.get_tokenizer().decode(prompt)
713
matches = re.findall(self.get_placeholder_match_pattern(), prompt)
@@ -818,7 +715,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
715
f"{modality}_orders":
716
torch.tensor(
717
[index for index, m in enumerate(matches) if m == modality])
821
- for modality in supported_mm_modalities
718
+ for modality in self.info.get_supported_mm_limits()
719
}
720
result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
721
return_mm_hashes)
@@ -884,35 +781,35 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
781
self,
782
input_ids: torch.Tensor,
783
image_inputs: Optional[MiniCPMVImageInputs],
887
- ) -> Tuple[torch.Tensor, torch.Tensor]:
784
+ ) -> torch.Tensor:
785
vlm_embedding: torch.Tensor = self.llm.get_input_embeddings(input_ids)
786
890
- if image_inputs is None: # No image
891
- vision_hidden_states = torch.tensor([], device=input_ids.device)
787
+ if image_inputs is None:
788
+ return vlm_embedding
789
+
790
+ if image_inputs["type"] == "image_embeds":
791
+ vision_hidden_states = image_inputs["image_embeds"].to(
792
+ device=vlm_embedding.device,
793
+ dtype=vlm_embedding.dtype,
794
+ )
795
else:
893
- if image_inputs["type"] == "image_embeds":
894
- vision_hidden_states = (image_inputs["data"].type(
895
- vlm_embedding.dtype).to(vlm_embedding.device))
896
- else:
897
- vision_hidden_states = self.get_vision_hidden_states(
898
- image_inputs)
899
-
900
- # See NOTE in _parse_and_validate_inputs
901
- image_bounds = image_inputs["image_bounds"]
902
- if len(image_bounds) > 0:
903
- image_indices = torch.stack([
904
- torch.arange(start, end, dtype=torch.long)
905
- for start, end in image_bounds.tolist()
906
- ]).to(vlm_embedding.device)
907
- vlm_embedding.scatter_(
908
- 0,
909
- image_indices.view(-1, 1).repeat(1,
910
- vlm_embedding.shape[-1]),
911
- vision_hidden_states.view(-1,
912
- vision_hidden_states.shape[-1]),
913
- )
796
+ vision_hidden_states = self.get_vision_hidden_states(image_inputs)
797
+
798
+ # See NOTE in _parse_and_validate_inputs
799
+ image_bounds = image_inputs["image_bounds"]
800
+ if len(image_bounds) > 0:
801
+ image_indices = torch.stack([
802
+ torch.arange(start, end, dtype=torch.long)
803
+ for start, end in image_bounds.tolist()
804
+ ]).to(vlm_embedding.device)
805
+
806
+ vlm_embedding.scatter_(
807
+ 0,
808
+ image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
809
+ vision_hidden_states.view(-1, vision_hidden_states.shape[-1]),
810
+ )
811
915
- return vlm_embedding, vision_hidden_states
812
+ return vlm_embedding
813
814
def _get_image_bounds(
815
self,
@@ -947,90 +844,115 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
844
input_ids: torch.Tensor,
845
**kwargs: object,
846
) -> Optional[MiniCPMVImageInputs]:
950
- mm_data = {
847
+ image_keys = {"pixel_values", "tgt_sizes"}
848
+ pixel_data = {
849
"image": {
952
- key: kwargs.pop(key, [])
953
- for key in ["pixel_values", "tgt_sizes", "image_num_slices"]
850
+ key: kwargs.pop(key, None)
851
+ for key in image_keys
852
},
853
"video": {
956
- "pixel_values": kwargs.pop("video_pixel_values", []),
957
- "tgt_sizes": kwargs.pop("video_tgt_sizes", []),
958
- "video_num_slices": kwargs.pop("video_num_slices", [])
854
+ key: kwargs.pop("video_" + key, None)
855
+ for key in image_keys
856
}
857
}
961
- im_start_id = kwargs.pop("im_start_id", None)
962
- im_end_id = kwargs.pop("im_end_id", None)
858
+ embed_data = {
859
+ "image": kwargs.pop("image_embeds", None),
860
+ "video": kwargs.pop("video_embeds", None),
861
+ }
862
+
863
+ all_pixel_data = [
864
+ v for vs in pixel_data.values() for v in vs.values()
865
+ if v is not None
866
+ ]
867
+ all_embed_data = [v for v in embed_data.values() if v is not None]
868
+ if len(all_pixel_data) == 0 and len(all_embed_data) == 0:
869
+ return None
870
+
871
+ im_start_id = kwargs.pop("im_start_id")
872
+ if not isinstance(im_start_id, torch.Tensor):
873
+ raise ValueError("Incorrect type of im_start_id. "
874
+ f"Got type: {type(im_start_id)}")
875
+
876
+ im_end_id = kwargs.pop("im_end_id")
877
+ if not isinstance(im_end_id, torch.Tensor):
878
+ raise ValueError("Incorrect type of im_end_id. "
879
+ f"Got type: {type(im_end_id)}")
880
+
881
slice_start_id = kwargs.pop("slice_start_id", None)
882
+ if slice_start_id is not None and not isinstance(
883
+ slice_start_id, torch.Tensor):
884
+ raise ValueError("Incorrect type of slice_start_id. "
885
+ f"Got type: {type(slice_start_id)}")
886
+
887
slice_end_id = kwargs.pop("slice_end_id", None)
965
- mm_orders = {
966
- f"{modality}": kwargs.pop(f"{modality}_orders", None)
967
- for modality in ["image", "video", "audio"]
968
- }
969
- batch_size = max(len(mm_data["image"]["pixel_values"]),
970
- len(mm_data["video"]["pixel_values"]))
971
- image_embeds = kwargs.pop("image_embeds", None)
972
- video_embeds = kwargs.pop("video_embeds", None)
973
- if image_embeds is not None and video_embeds is not None:
974
- raise ValueError(
975
- "Incorrect inputs for vision embeddings. "
976
- "Image embeds and video embeds can not exist simultaneously.")
977
- if video_embeds is not None:
978
- image_embeds = video_embeds
979
- if image_embeds is not None:
980
- if not isinstance(image_embeds, (torch.Tensor, list)):
981
- raise ValueError(f"Incorrect type of image embeds. "
982
- f"Got type: {type(image_embeds)}")
983
- image_embeds = torch.concat(
984
- [image_embeds[i] for i in range(len(image_embeds))])
888
+ if slice_end_id is not None and not isinstance(slice_end_id,
889
+ torch.Tensor):
890
+ raise ValueError("Incorrect type of slice_end_id. "
891
+ f"Got type: {type(slice_end_id)}")
892
+
893
+ if len(all_embed_data) > 0:
894
+ if len(all_embed_data) > 1:
895
+ raise ValueError("Incorrect inputs for vision embeddings. "
896
+ "Image embeds and video embeds can not "
897
+ "exist simultaneously.")
898
+
899
+ vision_embeds, = all_embed_data
900
+ if not isinstance(vision_embeds, (torch.Tensor, list)):
901
+ raise ValueError(f"Incorrect type of vision_embeds. "
902
+ f"Got type: {type(vision_embeds)}")
903
904
return MiniCPMVImageEmbeddingInputs(
905
+ type="image_embeds",
906
+ image_embeds=flatten_bn(flatten_2d_lists(vision_embeds),
907
+ concat=True),
908
image_bounds=self._get_image_bounds(input_ids, im_start_id,
909
im_end_id, slice_start_id,
910
slice_end_id),
990
- data=image_embeds,
991
- type="image_embeds",
911
)
993
- for modality, modality_mm_data in mm_data.items():
994
- if not isinstance(modality_mm_data["pixel_values"],
995
- (torch.Tensor, list)):
996
- raise ValueError(
997
- "Incorrect type of pixel values. "
998
- f"Got type: {type(modality_mm_data['pixel_values'])}")
999
-
1000
- if not isinstance(modality_mm_data["tgt_sizes"],
1001
- (torch.Tensor, list)):
1002
- raise ValueError(
1003
- "Incorrect type of target sizes. "
1004
- f"Got type: {type(modality_mm_data['tgt_sizes'])}")
1005
-
1006
- if len(modality_mm_data["pixel_values"]) != len(
1007
- modality_mm_data["tgt_sizes"]):
1008
- raise ValueError(
1009
- "Inconsistent batch lengths, found: "
1010
- f"{len(modality_mm_data['pixel_values'])} vs. "
1011
- f"{len(modality_mm_data['tgt_sizes'])}")
1012
-
1013
- pixel_values_flat: List[torch.Tensor] = []
1014
- tgt_sizes_flat: List[torch.Tensor] = []
912
+
913
+ order_data = dict[str, Union[torch.Tensor, list[torch.Tensor]]]()
914
+ for modality in ("image", "video"):
915
+ modality_orders = kwargs.pop(f"{modality}_orders", None)
916
+ if modality_orders is not None:
917
+ if not isinstance(modality_orders, (torch.Tensor, list)):
918
+ raise ValueError(f"Incorrect type of {modality}_orders. "
919
+ f"Got type: {type(modality_orders)}")
920
+
921
+ order_data[modality] = modality_orders
922
+
923
+ batch_sizes = {
924
+ modality: len(modality_orders)
925
+ for modality, modality_orders in order_data.items()
926
+ }
927
+ unique_batch_sizes = set(batch_sizes.values())
928
+ assert len(unique_batch_sizes) == 1, (
929
+ f"Found inconsistent batch sizes: {batch_sizes}")
930
+ batch_size, = unique_batch_sizes
931
+
932
+ pixel_values_flat = list[torch.Tensor]()
933
+ tgt_sizes_flat = list[torch.Tensor]()
934
for b in range(batch_size):
1016
- mm_counts = {"image": 0, "video": 0} if self.version == (2, 6) \
1017
- else {"image": 0}
1018
- mm_slice_counts = {"image": 0, "video": 0} \
1019
- if self.version == (2, 6) else {"image": 0}
1020
- mm_orders_b = [(index, modality) for modality in mm_counts
1021
- for index in mm_orders[modality][b]]
935
+ mm_orders_b = [(idx_b.item(), modality)
936
+ for modality, modality_orders in order_data.items()
937
+ for idx_b in modality_orders[b]]
938
+
939
for _, modality in sorted(mm_orders_b, key=lambda x: x[0]):
1023
- pos = mm_counts[modality]
1024
- num_slices = mm_data[modality][f"{modality}_num_slices"][b][
1025
- pos]
1026
- slice_start_idx = mm_slice_counts[modality]
1027
- slice_end_idx = slice_start_idx + num_slices
1028
- pixel_values_flat += mm_data[modality]["pixel_values"][b][
1029
- slice_start_idx:slice_end_idx]
1030
- tgt_sizes_flat += mm_data[modality]["tgt_sizes"][b][
1031
- slice_start_idx:slice_end_idx]
1032
- mm_counts[modality] += 1
1033
- mm_slice_counts[modality] += num_slices
940
+ modality_pixel_data = pixel_data[modality]
941
+
942
+ modality_pixel_values = modality_pixel_data["pixel_values"]
943
+ if not isinstance(modality_pixel_values, (torch.Tensor, list)):
944
+ raise ValueError(
945
+ f"Incorrect type of pixel_values for {modality=}. "
946
+ f"Got type: {type(modality_pixel_values)}")
947
+
948
+ modality_tgt_sizes = modality_pixel_data["tgt_sizes"]
949
+ if not isinstance(modality_tgt_sizes, (torch.Tensor, list)):
950
+ raise ValueError(
951
+ f"Incorrect type of tgt_sizes for {modality=}. "
952
+ f"Got type: {type(modality_tgt_sizes)}")
953
+
954
+ pixel_values_flat += flatten_2d_lists(modality_pixel_values[b])
955
+ tgt_sizes_flat += flatten_2d_lists(modality_tgt_sizes[b])
956
957
# NOTE: Input IDs does not contain image tokens during memory profiling,
958
# so we allow it to be empty
@@ -1042,16 +964,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
964
if len(pixel_values_flat) == 0:
965
return None
966
1045
- if im_start_id is None:
1046
- return None
1047
-
967
return MiniCPMVImagePixelInputs(
968
+ type="pixel_values",
969
+ pixel_values=pixel_values_flat,
970
+ tgt_sizes=torch.stack(tgt_sizes_flat),
971
image_bounds=self._get_image_bounds(input_ids, im_start_id,
972
im_end_id, slice_start_id,
973
slice_end_id),
1052
- data=pixel_values_flat,
1053
- tgt_sizes=torch.stack(tgt_sizes_flat),
1054
- type="pixel_values",
974
)
975
976
def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
@@ -1070,7 +989,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
989
else:
990
image_inputs = \
991
self._parse_and_validate_inputs(input_ids, **kwargs)
1073
- vlm_embeddings, _ = self.get_embedding_with_vision(
992
+ vlm_embeddings = self.get_embedding_with_vision(
993
input_ids, image_inputs)
994
995
# always pass the input via `inputs_embeds`
@@ -1136,16 +1055,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,
1055
prefix: str = "") -> nn.Module:
1056
raise NotImplementedError
1057
1139
- def get_vision_embedding(
1140
- self,
1141
- pixel_values: List[torch.Tensor],
1142
- patch_attn_mask: Optional[torch.Tensor] = None,
1143
- tgt_sizes: Optional[torch.Tensor] = None,
1144
- ) -> torch.Tensor:
1145
- raise NotImplementedError
1146
-
1147
- def get_vision_hidden_states(self,
1148
- data: MiniCPMVImageInputs) -> torch.Tensor:
1058
+ def get_vision_hidden_states(
1059
+ self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1060
raise NotImplementedError
1061
1062
@@ -1216,34 +1127,26 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
1127
return resampler.to(device=current_platform.device_type,
1128
dtype=torch.get_default_dtype())
1129
1219
- def get_vision_embedding(
1220
- self,
1221
- pixel_values: List[torch.Tensor],
1222
- patch_attn_mask: Optional[torch.Tensor] = None,
1223
- tgt_sizes: Optional[torch.Tensor] = None,
1224
- ) -> torch.Tensor:
1225
- res = []
1226
- dtype = self.vpm.pos_embed.data.dtype
1130
+ def get_vision_hidden_states(
1131
+ self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1132
+ pixel_values = data["pixel_values"]
1133
+
1134
+ P_h, P_w = self.vpm.patch_embed.patch_size
1135
+ dtype: torch.dtype = self.vpm.pos_embed.data.dtype
1136
+ num_prefix_tokens = getattr(self.vpm, "num_prefix_tokens", 0)
1137
+
1138
+ res = list[torch.Tensor]()
1139
for pixel_value in pixel_values:
1140
H, W = pixel_value[0].shape[-2:]
1229
- tgt_size = (
1230
- math.ceil(H / self.vpm.patch_embed.patch_size[0]),
1231
- math.ceil(W / self.vpm.patch_embed.patch_size[0]),
1232
- )
1141
+ tgt_size = (math.ceil(H / P_h), math.ceil(W / P_w))
1142
vision_embedding = self.vpm.forward_features(
1143
pixel_value.unsqueeze(0).type(dtype))
1235
- if (hasattr(self.vpm, "num_prefix_tokens")
1236
- and self.vpm.num_prefix_tokens > 0):
1237
- vision_embedding = vision_embedding[:, self.vpm.
1238
- num_prefix_tokens:]
1239
- res.append(self.resampler(vision_embedding, tgt_size))
1240
- return torch.vstack(res)
1144
1242
- def get_vision_hidden_states(self,
1243
- data: MiniCPMVImageInputs) -> torch.Tensor:
1244
- pixel_values = data["data"]
1145
+ if num_prefix_tokens > 0:
1146
+ vision_embedding = vision_embedding[:, num_prefix_tokens:]
1147
+ res.append(self.resampler(vision_embedding, tgt_size))
1148
1246
- return self.get_vision_embedding(pixel_values)
1149
+ return torch.vstack(res)
1150
1151
1152
class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
@@ -1299,45 +1202,41 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
1202
return resampler.to(device=current_platform.device_type,
1203
dtype=torch.get_default_dtype())
1204
1302
- def get_vision_embedding(
1303
- self,
1304
- pixel_values: List[torch.Tensor],
1305
- patch_attn_mask: Optional[torch.Tensor] = None,
1306
- tgt_sizes: Optional[torch.Tensor] = None,
1307
- ) -> torch.Tensor:
1308
- vision_embedding = self.vpm(pixel_values,
1309
- patch_attention_mask=patch_attn_mask)
1310
- vision_embedding = self.resampler(vision_embedding, tgt_sizes)
1311
- return vision_embedding
1312
-
1313
- def get_vision_hidden_states(self,
1314
- data: MiniCPMVImageInputs) -> torch.Tensor:
1315
- pixel_values = data["data"]
1205
+ def get_vision_hidden_states(
1206
+ self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1207
+ pixel_values = data["pixel_values"]
1208
tgt_sizes = data["tgt_sizes"]
1209
1318
- device = self.vpm.embeddings.position_embedding.weight.device
1319
- dtype = self.vpm.embeddings.position_embedding.weight.dtype
1320
- all_pixel_values_lst = [
1321
- i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
1322
- ]
1210
+ B = len(pixel_values)
1211
+ P = pixel_values[0].shape[-2]
1212
+ L = max(item.shape[-1] for item in pixel_values)
1213
+ device = pixel_values[0].device
1214
+ dtype = pixel_values[0].dtype
1215
1324
- max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
1325
- assert isinstance(max_patches, int)
1216
+ all_pixel_values = torch.zeros((B, 3, P, L),
1217
+ dtype=dtype,
1218
+ device=device)
1219
+ for i, pixel_values_item in enumerate(pixel_values):
1220
+ L_item = pixel_values_item.shape[-1]
1221
+ all_pixel_values[i, ..., :L_item] = pixel_values_item
1222
1327
- all_pixel_values = torch.nn.utils.rnn.pad_sequence(
1328
- all_pixel_values_lst, batch_first=True, padding_value=0.0)
1329
- B, L, _ = all_pixel_values.shape
1330
- all_pixel_values = all_pixel_values.permute(0, 2,
1331
- 1).reshape(B, 3, -1, L)
1223
+ num_patches = tgt_sizes.prod(-1)
1224
+ max_patches = num_patches.max().item()
1225
+ assert isinstance(max_patches, int)
1226
1333
- patch_attn_mask = torch.zeros((B, 1, max_patches),
1227
+ patch_attn_mask = torch.zeros((B, max_patches),
1228
dtype=torch.bool,
1229
device=device)
1336
- for i in range(B):
1337
- patch_attn_mask[i, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
1230
+ for i, num_patches_item in enumerate(num_patches):
1231
+ patch_attn_mask[i, :num_patches_item] = True
1232
1339
- return self.get_vision_embedding(all_pixel_values.type(dtype),
1340
- patch_attn_mask, tgt_sizes)
1233
+ vision_embedding = self.vpm(
1234
+ all_pixel_values,
1235
+ patch_attention_mask=patch_attn_mask.unsqueeze(1),
1236
+ tgt_sizes=None,
1237
+ )
1238
+
1239
+ return self.resampler(vision_embedding, tgt_sizes)
1240
1241
1242
class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
@@ -1394,47 +1293,37 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1293
return resampler.to(device=current_platform.device_type,
1294
dtype=torch.get_default_dtype())
1295
1397
- def get_vision_embedding(
1398
- self,
1399
- pixel_values: List[torch.Tensor],
1400
- patch_attn_mask: Optional[torch.Tensor] = None,
1401
- tgt_sizes: Optional[torch.Tensor] = None,
1402
- ) -> torch.Tensor:
1403
- vision_embedding = self.vpm(
1404
- pixel_values,
1405
- patch_attention_mask=patch_attn_mask,
1406
- tgt_sizes=tgt_sizes,
1407
- )
1408
- return vision_embedding
1409
-
1410
- def get_vision_hidden_states(self,
1411
- data: MiniCPMVImageInputs) -> torch.Tensor:
1412
- pixel_values = data["data"]
1296
+ def get_vision_hidden_states(
1297
+ self, data: MiniCPMVImagePixelInputs) -> torch.Tensor:
1298
+ pixel_values = data["pixel_values"]
1299
tgt_sizes = data["tgt_sizes"]
1300
1415
- device = self.vpm.embeddings.position_embedding.weight.device
1416
- dtype = self.vpm.embeddings.position_embedding.weight.dtype
1417
- all_pixel_values_lst = [
1418
- i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
1419
- ]
1301
+ B = len(pixel_values)
1302
+ P = pixel_values[0].shape[-2]
1303
+ L = max(item.shape[-1] for item in pixel_values)
1304
+ device = pixel_values[0].device
1305
+ dtype = pixel_values[0].dtype
1306
1421
- max_patches = (tgt_sizes[:, 0] * tgt_sizes[:, 1]).max().item()
1422
- assert isinstance(max_patches, int)
1307
+ all_pixel_values = torch.zeros((B, 3, P, L),
1308
+ dtype=dtype,
1309
+ device=device)
1310
+ for i, pixel_values_item in enumerate(pixel_values):
1311
+ L_item = pixel_values_item.shape[-1]
1312
+ all_pixel_values[i, ..., :L_item] = pixel_values_item
1313
1424
- all_pixel_values = torch.nn.utils.rnn.pad_sequence(
1425
- all_pixel_values_lst, batch_first=True, padding_value=0.0)
1426
- B, L, _ = all_pixel_values.shape
1427
- all_pixel_values = all_pixel_values.permute(0, 2,
1428
- 1).reshape(B, 3, -1, L)
1314
+ num_patches = tgt_sizes.prod(-1)
1315
+ max_patches = num_patches.max().item()
1316
+ assert isinstance(max_patches, int)
1317
1430
- patch_attn_mask = torch.zeros((B, 1, max_patches),
1318
+ patch_attn_mask = torch.zeros((B, max_patches),
1319
dtype=torch.bool,
1320
device=device)
1433
- for i in range(B):
1434
- patch_attn_mask[i, 0, :tgt_sizes[i][0] * tgt_sizes[i][1]] = True
1321
+ for i, num_patches_item in enumerate(num_patches):
1322
+ patch_attn_mask[i, :num_patches_item] = True
1323
+
1324
vision_embedding = self.vpm(
1436
- all_pixel_values.type(dtype),
1437
- patch_attention_mask=patch_attn_mask,
1325
+ all_pixel_values,
1326
+ patch_attention_mask=patch_attn_mask.unsqueeze(1),
1327
tgt_sizes=tgt_sizes,
1328
)
1329
vllm/multimodal/inputs.py
+7
@@ -665,6 +665,13 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
665
666
return cast(BatchedTensorInputs, json_mapped)
667
668
+ def __delitem__(self, key: str) -> None:
669
+ super().__delitem__(key)
670
+
671
+ for items in self._items_by_modality.values():
672
+ for item in items:
673
+ item.pop(key, None)
674
+
675
def __eq__(self, other: object) -> bool:
676
if not isinstance(other, self.__class__):
677
return False