22
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
23
import math
24
import re
25
+from collections import Counter
26
from functools import cached_property, partial
26
-from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27
- Set, Tuple, TypedDict, Union)
27
+from itertools import accumulate
28
+from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
29
+ Optional, Set, Tuple, TypedDict, Union)
30
31
+import numpy as np
32
import torch
33
import torch.types
34
from PIL import Image
35
from torch import nn
33
-from transformers import PretrainedConfig
34
-from typing_extensions import NotRequired
36
+from transformers import BatchFeature, PretrainedConfig
37
38
from vllm.attention import AttentionMetadata
39
from vllm.config import VllmConfig
38
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39
- InputContext, token_inputs)
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
42
get_2d_sincos_pos_embed)
48
from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51
-from vllm.multimodal.image import cached_get_image_processor
52
-from vllm.multimodal.utils import cached_get_tokenizer
53
-from vllm.sequence import IntermediateTensors, SequenceData
51
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
52
+ MultiModalInputs, PlaceholderRange)
53
+from vllm.multimodal.parse import (ImageItem, ImageSize, ModalityData,
54
+ ModalityDataItems, MultiModalDataItems,
55
+ MultiModalDataParser, VideoItem)
56
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
57
+ BaseProcessingInfo, PromptReplacement)
58
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
59
+from vllm.sequence import IntermediateTensors
60
61
from .idefics2_vision_model import Idefics2VisionTransformer
62
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
63
from .utils import AutoWeightsLoader, maybe_prefix
64
59
-RawImageType = Union[Image.Image, torch.Tensor]
60
-
61
-
62
-class MiniCPMVRawImageInput(TypedDict):
63
- """Input mapper input with auxiliary data for computing image bounds."""
64
- image: RawImageType
65
+CPU_DEVICE = torch.device("cpu")
66
66
- # Image bounds token ids in 0-dim scaler tensor.
67
- im_start_id: torch.Tensor
68
- im_end_id: torch.Tensor
69
- slice_start_id: NotRequired[torch.Tensor]
70
- slice_end_id: NotRequired[torch.Tensor]
67
+RawImageType = Union[Image.Image, torch.Tensor]
68
69
70
class MiniCPMVImagePixelInputs(TypedDict):
71
type: Literal["pixel_values"]
72
data: List[torch.Tensor]
73
"""
77
- Shape: `(batch_size * num_images, num_channels, height, width)`
74
+ Shape: `(batch_size * num_images * num_slices, num_channels, height, width)`
75
76
Note that the image size may vary, so we pass it as a list
77
instead of a batched tensor.
79
80
image_bounds: torch.Tensor
81
"""
85
- Shape: `(batch_size * num_images, 2)`
82
+ Shape: `(batch_size * num_images * num_slices, 2)`
83
84
This should be in `(start, stop)` format.
85
"""
86
87
tgt_sizes: torch.Tensor
88
"""
92
- Shape: `(batch_size * num_images, 2)`
89
+ Shape: `(batch_size * num_images * num_slices, 2)`
90
91
This should be in `(height, width)` format.
92
"""
96
type: Literal["image_embeds"]
97
data: torch.Tensor
98
"""
102
- Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
99
+ Shape: `(batch_size * num_images * num_slices,
100
+ image_feature_size, hidden_size)`
101
102
`hidden_size` must match the hidden size of language model backbone.
103
instead of a batched tensor.
105
106
image_bounds: torch.Tensor
107
"""
110
- Shape: `(batch_size * num_images, 2)`
108
+ Shape: `(batch_size * num_images * num_slices, 2)`
109
110
This should be in `(start, stop)` format.
111
"""
114
MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
115
MiniCPMVImageEmbeddingInputs]
116
117
+
118
+class MiniCPMVEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
119
+ dict[str, torch.Tensor]]):
120
+
121
+ def __init__(self, data: Dict, modality: str) -> None:
122
+ super().__init__(data, modality)
123
+
124
+ def get_processor_data(self) -> Mapping[str, object]:
125
+ return self.data
126
+
127
+ def get_passthrough_data(self) -> Mapping[str, object]:
128
+ return {}
129
+
130
+ def get_count(self) -> int:
131
+ return len(self.data[f"{self.modality}_embeds"])
132
+
133
+ def get(self, index: int) -> Dict[str, torch.Tensor]:
134
+ out = {}
135
+ for k, v in self.data.items():
136
+ out[k] = v[index]
137
+ return out
138
+
139
+
140
+class MiniCPMVImageEmbeddingItems(MiniCPMVEmbeddingItems):
141
+
142
+ def __init__(self, data: Dict) -> None:
143
+ super().__init__(data, "image")
144
+ image_embeds = self.data.get("image_embeds", None)
145
+ image_sizes = self.data.get("image_sizes", None)
146
+ if image_embeds is None:
147
+ raise ValueError("In correct type of image_embeds",
148
+ "Got type: None")
149
+ if not isinstance(image_embeds[0], torch.Tensor):
150
+ raise ValueError("In correct type of image_embeds",
151
+ f"Got type: {type(image_embeds[0])}")
152
+ if image_sizes is None:
153
+ raise ValueError(
154
+ "In correct type of image_sizes", "Got type: None."
155
+ "If you're using `image_size_list`, "
156
+ "please rename it to `image_sizes`")
157
+ if len(image_embeds[0].shape) == 2:
158
+ image_embeds = [image_embeds]
159
+ image_sizes = [image_sizes]
160
+ self.data["image_embeds"] = image_embeds
161
+ self.data["image_sizes"] = image_sizes
162
+
163
+ def get_image_size(self, index: int) -> ImageSize:
164
+ image_size = self.data["image_sizes"][index]
165
+ return ImageSize(width=image_size[0], height=image_size[1])
166
+
167
+
168
+class MiniCPMVVideoEmbeddingItems(MiniCPMVEmbeddingItems):
169
+
170
+ def __init__(self, data: Dict) -> None:
171
+ super().__init__(data, "video")
172
+ video_embeds = self.data.get("video_embeds", None)
173
+ image_sizes = self.data.get("image_sizes", None)
174
+ num_frames = self.data.get("num_frames", None)
175
+ if video_embeds is None:
176
+ raise ValueError("In correct type of video_embeds",
177
+ "Got type: None")
178
+ if not isinstance(video_embeds[0], torch.Tensor):
179
+ raise ValueError("In correct type of video_embeds",
180
+ f"Got type: {type(video_embeds[0])}")
181
+ if image_sizes is None:
182
+ raise ValueError(
183
+ "In correct type of image_sizes", "Got type: None."
184
+ "If you're using `image_size_list`, "
185
+ "please rename it to `image_sizes`")
186
+ if num_frames is None:
187
+ raise ValueError("In correct type of numframes", "Got type: None")
188
+ if len(video_embeds[0].shape) == 2:
189
+ video_embeds = [video_embeds]
190
+ image_sizes = [image_sizes]
191
+ num_frames = [num_frames]
192
+ self.data["video_embeds"] = video_embeds
193
+ self.data["image_sizes"] = image_sizes
194
+ self.data["num_frames"] = num_frames
195
+
196
+ def get_frame_size(self, index: int) -> ImageSize:
197
+ frame_size = self.data["image_sizes"][index]
198
+ return ImageSize(width=frame_size[0], height=frame_size[1])
199
+
200
+ def get_num_frames(self, index: int) -> int:
201
+ return self.data["num_frames"][index]
202
+
203
+
204
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
205
206
297
return x
298
299
215
-def _build_image_input(ctx: InputContext,
216
- image: RawImageType) -> MiniCPMVRawImageInput:
217
- tokenizer = cached_get_tokenizer(
218
- ctx.model_config.tokenizer,
219
- trust_remote_code=ctx.model_config.trust_remote_code)
220
- if hasattr(tokenizer, "slice_start_id"):
221
- return MiniCPMVRawImageInput(
222
- image=image,
223
- im_start_id=torch.tensor(tokenizer.im_start_id),
224
- im_end_id=torch.tensor(tokenizer.im_end_id),
225
- slice_start_id=torch.tensor(tokenizer.slice_start_id),
226
- slice_end_id=torch.tensor(tokenizer.slice_end_id))
227
- else:
228
- return MiniCPMVRawImageInput(
229
- image=image,
230
- im_start_id=torch.tensor(tokenizer.im_start_id),
231
- im_end_id=torch.tensor(tokenizer.im_end_id))
232
-
233
-
300
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
301
version_float = getattr(config, "version", None)
302
306
if config.hidden_size == 2304 and config.query_num == 64:
307
return (2, 0)
308
return (2, 5)
243
-
309
version_str = str(version_float)
310
return tuple(int(x) for x in version_str.split("."))
311
312
248
-def get_max_minicpmv_image_tokens(ctx: InputContext):
249
- hf_config = ctx.get_hf_config()
250
- return getattr(hf_config, "query_num", 64)
313
+class MiniCPMVMultiModalDataParser(MultiModalDataParser):
314
+
315
+ def _parse_image_data(
316
+ self,
317
+ data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
318
+ ) -> ModalityDataItems[Any, Any]:
319
+ if isinstance(data, dict):
320
+ return MiniCPMVImageEmbeddingItems(data)
321
+ return super()._parse_image_data(data)
322
+
323
+ def _parse_video_data(
324
+ self,
325
+ data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
326
+ ) -> ModalityDataItems[Any, Any]:
327
+ if isinstance(data, dict):
328
+ return MiniCPMVVideoEmbeddingItems(data)
329
+ return super()._parse_video_data(data)
330
+
331
+
332
+class MiniCPMVProcessingInfo(BaseProcessingInfo):
333
+ image_pattern = "(<image>./</image>)"
334
+ video_pattern = "(<video>./</video>)"
335
+
336
+ def get_hf_config(self):
337
+ return self.ctx.get_hf_config()
338
+
339
+ def get_hf_processor(
340
+ self,
341
+ **kwargs: object,
342
+ ):
343
+ hf_processor = self.ctx.get_hf_processor()
344
+ return hf_processor
345
+
346
+ def get_image_processor(self):
347
+ hf_processor = self.get_hf_processor()
348
+ image_processor = hf_processor.image_processor # type: ignore
349
+ return image_processor
350
+
351
+ def get_model_version(self):
352
+ return get_version_by_config(self.get_hf_config())
353
+
354
+ def get_supported_mm_modalities(self) -> List[str]:
355
+ if self.get_model_version() == (2, 6):
356
+ return ["image", "video"]
357
+ else:
358
+ return ["image"]
359
+
360
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
361
+ if self.get_model_version() == (2, 6):
362
+ return {"image": None, "video": None}
363
+ else:
364
+ return {"image": None}
365
+
366
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
367
+ mm_max_tokens = {"image": self.get_max_image_tokens()}
368
+ if self.get_model_version() == (2, 6):
369
+ mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)
370
+ return mm_max_tokens
371
+
372
+ def get_max_video_frame_tokens(self) -> int:
373
+ frame_size = self.get_video_frame_size_with_most_features()
374
+ return self.get_num_image_tokens(frame_size,
375
+ self.get_video_max_slice_num())
376
+
377
+ def get_max_video_tokens(self, seq_len: int) -> int:
378
+ return self.get_max_video_frame_tokens(
379
+ ) * self.get_num_frames_with_most_features(seq_len)
380
+
381
+ def get_max_audio_tokens(self) -> int:
382
+ return self.get_max_audio_tokens_per_chunk(
383
+ ) * self.get_max_audio_chunks_with_most_features()
384
+
385
+ def get_slice_query_num(self) -> int:
386
+ hf_config = self.get_hf_config()
387
+ query_num = getattr(hf_config, "query_num", 64)
388
+ return query_num
389
+
390
+ def get_max_slice_num(self) -> int:
391
+ hf_config = self.get_hf_config()
392
+ max_slice_num = getattr(hf_config, "max_slice_num", 9)
393
+ return max_slice_num
394
+
395
+ def get_sliced_grid(self, image_size: ImageSize,
396
+ max_slice_num: int) -> Tuple[int, int]:
397
+ if self.get_model_version() == (2, 6):
398
+ slice_grid = self.get_image_processor().get_sliced_grid(
399
+ image_size, max_slice_num)
400
+ else:
401
+ slice_grid = self.get_image_processor().get_sliced_grid(image_size)
402
+ return slice_grid
403
+
404
+ def get_num_image_tokens(self, image_size: ImageSize,
405
+ max_slice_num: int) -> int:
406
+ slice_grid = self.get_sliced_grid(image_size, max_slice_num)
407
+ num_tokens = self.get_slice_query_num(
408
+ ) + 2 # <image>(<unk> * query_num)</image>
409
+ if slice_grid is not None:
410
+ if self.get_model_version() == (2, 6):
411
+ num_additional_tokens = 0
412
+ else:
413
+ # <slice><image>(<unk> * query_num)</image></slice>
414
+ num_additional_tokens = 2
415
+ num_tokens += ((self.get_slice_query_num() + 2) \
416
+ * slice_grid[0] * slice_grid[1]) \
417
+ + slice_grid[1] - 1 + num_additional_tokens
418
+ return num_tokens
419
420
+ def get_image_slice_nums(self, image_size: torch.Tensor,
421
+ max_slice_nums: int) -> int:
422
+ grid = self.get_sliced_grid(image_size, max_slice_nums)
423
+ return 1 if grid is None else grid[0] * grid[1] + 1
424
253
-def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
254
- return SequenceData.from_prompt_token_counts((0, seq_len))
425
+ def get_max_image_tokens(self) -> int:
426
+ image_size = self.get_image_size_with_most_features()
427
+ return self.get_num_image_tokens(image_size, self.get_max_slice_num())
428
429
+ def get_image_size_with_most_features(self) -> ImageSize:
430
+ # Result in the max possible feature size (h:w = 9:1)
431
+ return self.get_default_image_sizes(self.get_max_slice_num())
432
257
-def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
258
- num_images: int):
259
- width = height = hf_config.image_size
260
- image = _build_image_input(ctx,
261
- image=Image.new("RGB", (width, height),
262
- color=0))
263
- return {"image": [image] if num_images == 1 else [image] * num_images}
433
+ def get_video_max_slice_num(self) -> int:
434
+ return 1
435
436
+ def get_video_frame_size_with_most_features(self) -> ImageSize:
437
+ return self.get_default_image_sizes(self.get_video_max_slice_num())
438
266
-def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
267
- mm_counts: Mapping[str, int]):
268
- hf_config = ctx.get_hf_config()
269
- num_images = mm_counts["image"]
439
+ def get_max_video_frames(self, max_tokens: int) -> int:
440
+ num_frame_tokens = self.get_max_video_frame_tokens()
441
+ num_frames = max_tokens // num_frame_tokens
442
+ return num_frames
443
271
- seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
272
- mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
444
+ def get_num_frames_with_most_features(self, seq_len: int) -> int:
445
+ mm_config = self.ctx.get_mm_config()
446
+ max_images = mm_config.limit_per_prompt.get("image", 1)
447
+ max_videos = mm_config.limit_per_prompt.get("video", 1)
448
274
- return DummyData(seq_data, mm_data)
449
+ # count <image_idx></image_idx> tokens
450
+ # which are not in get_max_image_tokens
451
+ max_image_tokens = self.get_max_image_tokens(
452
+ ) * max_images + 4 * max_images
453
+ max_total_frames = self.get_max_video_frames(seq_len -
454
+ max_image_tokens)
455
456
+ num_frames = max(max_total_frames // max(max_videos, 1), 1)
457
277
-def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
278
- multi_modal_data = inputs.get("multi_modal_data")
279
- if multi_modal_data is None or "image" not in multi_modal_data:
280
- return inputs
281
- model_config = ctx.model_config
282
- version = get_version_by_config(model_config.hf_config)
283
- tokenizer = cached_get_tokenizer(
284
- model_config.tokenizer,
285
- trust_remote_code=model_config.trust_remote_code)
286
- image_processor = cached_get_image_processor(model_config.tokenizer)
458
+ return num_frames
459
288
- def get_placeholder(image_size: Tuple[int, int], num_image: int):
460
+ def get_default_image_sizes(self, num_slices: int) -> ImageSize:
461
+ image_size = getattr(self.get_hf_config(), "image_size", 448)
462
+ return ImageSize(width=image_size, height=image_size * num_slices)
463
+
464
+
465
+class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
466
+ ):
467
+
468
+ def get_dummy_processor_inputs(
469
+ self,
470
+ seq_len: int,
471
+ mm_counts: Mapping[str, int],
472
+ ) -> ProcessorInputs:
473
+ num_images = mm_counts.get("image", 0)
474
+ num_videos = mm_counts.get("video", 0)
475
+
476
+ image_width, image_height = \
477
+ self.info.get_image_size_with_most_features()
478
+ video_width, video_height = \
479
+ self.info.get_video_frame_size_with_most_features()
480
+ num_video_frames = \
481
+ self.info.get_num_frames_with_most_features(seq_len)
482
+
483
+ mm_data = {
484
+ "image":
485
+ self._get_dummy_images(width=image_width,
486
+ height=image_height,
487
+ num_images=num_images),
488
+ "video": [
489
+ self._get_dummy_images(width=video_width,
490
+ height=video_height,
491
+ num_images=num_video_frames)
492
+ ] * num_videos,
493
+ }
494
+
495
+ image_prompt_texts = self.info.image_pattern * num_images
496
+ video_prompt_texts = self.info.video_pattern * num_videos
497
+
498
+ return ProcessorInputs(prompt_text=image_prompt_texts +
499
+ video_prompt_texts,
500
+ mm_data=mm_data)
501
+
502
+
503
+class MiniCPMVMultiModalProcessor(
504
+ BaseMultiModalProcessor[MiniCPMVProcessingInfo]):
505
+
506
+ def _get_data_parser(self) -> MultiModalDataParser:
507
+ return MiniCPMVMultiModalDataParser()
508
+
509
+ def get_slice_image_placeholder(self, image_size: ImageSize,
510
+ **kwargs) -> str:
511
+ image_processor = self.info.get_image_processor()
512
+ version = self.info.get_model_version()
513
if version == (2, 0) or version == (2, 5):
514
return image_processor.get_slice_image_placeholder(image_size)
515
return image_processor.get_slice_image_placeholder(
292
- image_size, num_image)
293
-
294
- prompt = inputs.get("prompt")
295
- token_ids = inputs.get("prompt_token_ids")
296
- if prompt is None:
297
- prompt = tokenizer.decode(token_ids)
298
-
299
- pattern = "(<image>./</image>)"
300
- images = multi_modal_data["image"]
301
- image_tags = re.findall(pattern, prompt)
302
- if len(image_tags) == 0:
303
- new_token_ids = token_ids
304
- new_prompt = prompt
305
- else:
306
- if isinstance(images, dict):
307
- image_size_list = images.get("image_size_list")
308
- images = [images.get("image_embeds")]
516
+ image_size, **kwargs)
517
+
518
+ def get_image_prompt_texts(self,
519
+ image_size: ImageSize,
520
+ image_idx: int = 0) -> str:
521
+ prompt_texts = self.get_slice_image_placeholder(image_size,
522
+ image_idx=image_idx)
523
+ return prompt_texts
524
+
525
+ def get_video_prompt_texts(self, image_size: ImageSize,
526
+ num_frames: int) -> str:
527
+ prompt_texts = "".join(
528
+ self.get_slice_image_placeholder(
529
+ image_size=image_size,
530
+ image_idx=0,
531
+ max_slice_nums=self.info.get_video_max_slice_num(),
532
+ use_image_id=False) for image_idx in range(num_frames))
533
+ return prompt_texts
534
+
535
+ def get_special_tokens(self) -> Dict[str, torch.Tensor]:
536
+ tokenizer = self.info.get_tokenizer()
537
+ special_tokens = {
538
+ "im_start_id": torch.tensor(tokenizer.im_start_id),
539
+ "im_end_id": torch.tensor(tokenizer.im_end_id)
540
+ }
541
+ if hasattr(tokenizer, "slice_start_id"):
542
+ special_tokens["slice_start_id"] = torch.tensor(
543
+ tokenizer.slice_start_id)
544
+ special_tokens["slice_end_id"] = torch.tensor(
545
+ tokenizer.slice_end_id)
546
+ return special_tokens
547
+
548
+ @staticmethod
549
+ def repack_processor_outputs(outputs: Any) -> BatchFeature:
550
+ valid_keys = ["pixel_values", "image_sizes", "tgt_sizes"]
551
+ outputs = {key: outputs[key][0] for key in valid_keys}
552
+ return outputs
553
+
554
+ def process_images(self, mm_data: Mapping[str, object],
555
+ mm_kwargs: Mapping[str, object]) -> Dict[str, object]:
556
+ images = mm_data.pop("images", [])
557
+ image_embeds = mm_data.pop("image_embeds", [])
558
+ if isinstance(images, Image.Image):
559
+ images = [images]
560
+ if isinstance(images, (list, torch.Tensor)) and len(images) > 0:
561
+ image_outputs = super()._call_hf_processor(
562
+ prompt=self.info.image_pattern * len(images),
563
+ mm_data={"images": images},
564
+ mm_kwargs=mm_kwargs)
565
+ image_outputs = MiniCPMVMultiModalProcessor.\
566
+ repack_processor_outputs(image_outputs)
567
+ elif len(image_embeds) > 0:
568
+ image_sizes = mm_data.pop("image_sizes", None)
569
+ image_outputs = {
570
+ "image_embeds": torch.cat(image_embeds),
571
+ "image_sizes": image_sizes
572
+ }
573
else:
310
- if isinstance(images, Image.Image):
311
- images = [images]
312
- image_size_list = [image.size for image in images]
313
-
314
- text_chunks = prompt.split(pattern)
315
- new_prompt_chunks: List[str] = []
316
- for i in range(len(image_size_list)):
317
- new_prompt_chunks += [
318
- text_chunks[i],
319
- get_placeholder(image_size_list[i], i)
320
- ]
321
- new_prompt_chunks.append(text_chunks[-1])
322
- new_prompt = "".join(new_prompt_chunks)
323
- new_token_ids = tokenizer.encode(new_prompt)
324
-
325
- multi_modal_data["image"] = [
326
- _build_image_input(ctx, image) for image in images
327
- ]
574
+ image_outputs = {}
575
+ return image_outputs
576
+
577
+ def process_videos(self, mm_data: Mapping[str, object],
578
+ mm_kwargs: Mapping[str, object]) -> Dict[str, object]:
579
+ videos = mm_data.pop("videos", [])
580
+ video_embeds = mm_data.pop("video_embeds", [])
581
+ if len(videos) > 0 and isinstance(videos[0], Image.Image):
582
+ videos = [videos]
583
+ if isinstance(videos, list) and len(videos) > 0:
584
+ video_outputs = {
585
+ "video_pixel_values": [],
586
+ "video_image_sizes": [],
587
+ "video_tgt_sizes": [],
588
+ "num_frames": []
589
+ }
590
+ for video in videos:
591
+ parsed_video = []
592
+ for frame in video:
593
+ if isinstance(frame, np.ndarray):
594
+ parsed_video.append(Image.fromarray(frame))
595
+ else:
596
+ parsed_video.append(frame)
597
+ video = parsed_video
598
+ single_video_outputs = super()._call_hf_processor(
599
+ prompt=self.info.image_pattern * len(video),
600
+ mm_data={"images": video},
601
+ mm_kwargs={
602
+ **mm_kwargs, "max_slice_nums":
603
+ self.info.get_video_max_slice_num()
604
+ })
605
+ video_outputs["num_frames"].append(len(video))
606
+ for key in single_video_outputs:
607
+ if "video_" + key in video_outputs:
608
+ if key == "image_sizes":
609
+ video_outputs["video_" + key].append(
610
+ single_video_outputs[key][0][0])
611
+ else:
612
+ video_outputs["video_" +
613
+ key] += single_video_outputs[key][0]
614
+ elif len(video_embeds):
615
+ image_sizes = mm_data.pop("image_sizes", None)
616
+ num_frames = mm_data.pop("num_frames", None)
617
+ video_outputs = {
618
+ "video_embeds": torch.cat(video_embeds),
619
+ "video_image_sizes": image_sizes,
620
+ "num_frames": num_frames
621
+ }
622
+ else:
623
+ video_outputs = {}
624
+ return video_outputs
625
329
- return token_inputs(
330
- prompt_token_ids=new_token_ids,
331
- prompt=new_prompt,
332
- multi_modal_data=multi_modal_data,
333
- )
626
+ def get_placeholder_match_pattern(self) -> str:
627
+ return r"\(<(image|video)>./</\1>\)"
628
629
+ def get_placeholder_split_pattern(self) -> str:
630
+ return r"\(<(?:image|video)>./</(?:image|video)>\)"
631
336
-def input_mapper_for_minicpmv(ctx: InputContext, data: object):
337
- model_config = ctx.model_config
632
+ def process_mm_inputs(self, mm_data, mm_kwargs) -> object:
633
+ return {
634
+ "image": self.process_images(mm_data, mm_kwargs),
635
+ "video": self.process_videos(mm_data, mm_kwargs)
636
+ }
637
339
- image_processor = cached_get_image_processor(
340
- model_config.model, trust_remote_code=model_config.trust_remote_code)
341
- if image_processor is None:
342
- raise RuntimeError("No HuggingFace processor is available "
343
- "to process the image object")
638
+ def get_input_modalities(self, mm_data) -> List[str]:
639
+ supported_mm_modalities = self.info.get_supported_mm_modalities()
640
+ input_modalities = []
641
+ for modality in supported_mm_modalities:
642
+ if modality in mm_data and mm_data[modality] != {}:
643
+ input_modalities.append(modality)
644
+ return input_modalities
645
+
646
+ def get_modality_num_counter(self, modality: str) -> str:
647
+ if modality == "image":
648
+ return "image_sizes"
649
+ elif modality == "video":
650
+ return "video_image_sizes"
651
+
652
+ def get_num_slices_by_modality(self, inputs: Dict[str, object],
653
+ modality: str, index: int) -> int:
654
+ if modality == "image":
655
+ return self.info.get_image_slice_nums(
656
+ inputs[modality]["image_sizes"][index],
657
+ self.info.get_max_slice_num())
658
+ elif modality == "video":
659
+ return self.info.get_image_slice_nums(
660
+ inputs[modality]["video_image_sizes"][index],
661
+ self.info.get_video_max_slice_num()
662
+ ) * inputs[modality]["num_frames"][index]
663
+ else:
664
+ raise ValueError(f"UnExpected modality: {modality}")
665
+
666
+ def check_mm_inputs(self, inputs: Dict[str, object],
667
+ matches: List[str]) -> None:
668
+ counts = Counter(matches)
669
+ for modality, count in counts.items():
670
+ if modality not in inputs or not inputs[modality]:
671
+ raise ValueError(f"None input data of {modality}."
672
+ "But prompt requires.")
673
+ counter_key = self.get_modality_num_counter(modality)
674
+ if len(inputs[modality][counter_key]) != count:
675
+ raise ValueError(f"The prompt requires {count} "
676
+ f"{modality} inputs while you pass "
677
+ f"{len(inputs[modality][counter_key])}")
678
+
679
+ def get_prompt_texts_by_modality(self, inputs: Dict[str, object],
680
+ modality: str, index: int) -> str:
681
+ if modality == "image":
682
+ return self.get_image_prompt_texts(
683
+ inputs["image"]["image_sizes"][index], index)
684
+ elif modality == "video":
685
+ return self.get_video_prompt_texts(
686
+ inputs["video"]["video_image_sizes"][index],
687
+ inputs["video"]["num_frames"][index])
688
+ else:
689
+ raise ValueError(f"UnExpected modality: {modality}")
690
345
- if not isinstance(data, list):
346
- raise ValueError(
347
- "Image input must be list of MiniCPMVImageInput, got (%s)", data)
691
+ def call_base_hf_processor(
692
+ self,
693
+ prompt: str,
694
+ mm_data: Mapping[str, object],
695
+ mm_kwargs: Mapping[str, object],
696
+ ) -> BatchFeature:
697
+ return super()._call_hf_processor(prompt=prompt,
698
+ mm_data=mm_data,
699
+ mm_kwargs=mm_kwargs)
700
+
701
+ def _call_hf_processor(
702
+ self,
703
+ prompt: str,
704
+ mm_data: Mapping[str, object],
705
+ mm_kwargs: Mapping[str, object],
706
+ ) -> BatchFeature:
707
+ # Do not support combination inputs of images and videos for now
708
+ # Try to handle interleaved multimodal data
709
+ tokenizer = self.info.get_tokenizer()
710
+ inputs = self.process_mm_inputs(mm_data, mm_kwargs)
711
+ mm_input_modalities = self.get_input_modalities(inputs)
712
+ num_mm_slices = {modality: [] for modality in mm_input_modalities}
713
+ for modality in mm_input_modalities:
714
+ num_counter_key = self.get_modality_num_counter(modality)
715
+ for index in range(len(inputs[modality][num_counter_key])):
716
+ num_mm_slices[modality].append(
717
+ self.get_num_slices_by_modality(inputs, modality, index))
718
+ return {
719
+ "input_ids": np.array([tokenizer.encode(prompt)]),
720
+ **{
721
+ key: value
722
+ for modality in inputs
723
+ for key, value in inputs[modality].items()
724
+ },
725
+ **{
726
+ f"{modality}_num_slices": num_mm_slices[modality]
727
+ for modality in mm_input_modalities
728
+ }
729
+ }
730
349
- if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor):
350
- batch_data = {
351
- "image_embeds": data[0]['image'],
731
+ def _get_prompt_replacements(
732
+ self, mm_items: MultiModalDataItems,
733
+ hf_processor_mm_kwargs: Mapping[str, Any],
734
+ out_mm_kwargs: MultiModalKwargs) -> List[PromptReplacement]:
735
+ placeholder = {
736
+ "image": self.info.image_pattern,
737
+ "video": self.info.video_pattern,
738
}
353
- else:
354
- batch_data = image_processor \
355
- .preprocess([img["image"] for img in data], return_tensors="pt") \
356
- .data
739
358
- if len(data) > 0:
359
- batch_data["im_start_id"] = data[0]["im_start_id"]
360
- batch_data["im_end_id"] = data[0]["im_end_id"]
361
- if "slice_start_id" in data[0]:
362
- batch_data["slice_start_id"] = data[0]["slice_start_id"]
363
- batch_data["slice_end_id"] = data[0]["slice_end_id"]
740
+ def get_replacement_minicpmv(item_idx: int, modality: str):
741
+ if modality == "image":
742
+ return self.get_image_prompt_texts(
743
+ mm_items["image"].get_image_size(item_idx), item_idx)
744
+ else: # video
745
+ return self.get_video_prompt_texts(
746
+ mm_items["video"].get_frame_size(item_idx),
747
+ mm_items["video"].get_num_frames(item_idx))
748
+
749
+ return [
750
+ PromptReplacement(modality=modality,
751
+ target=placeholder[modality],
752
+ replacement=partial(get_replacement_minicpmv,
753
+ modality=modality))
754
+ for modality in ("image", "video")
755
+ ]
756
365
- return MultiModalKwargs(batch_data)
757
+ def _get_mm_fields_config(
758
+ self,
759
+ hf_inputs,
760
+ hf_processor_mm_kwargs: Mapping[str, object],
761
+ ) -> Mapping[str, MultiModalFieldConfig]:
762
+
763
+ def get_slices(num_slices: List[int]) -> List[int]:
764
+ slice_indices = [0] + list(accumulate(num_slices))
765
+ slices = [(slice_indices[i], slice_indices[i + 1])
766
+ for i in range(len(num_slices))]
767
+ return [slice(*slice_item) for slice_item in slices]
768
+
769
+ image_slices = get_slices(
770
+ hf_inputs.get("image_num_slices", torch.empty(0)))
771
+ video_slices = get_slices(
772
+ hf_inputs.get("video_num_slices", torch.empty(0)))
773
+
774
+ return dict(
775
+ pixel_values=MultiModalFieldConfig.flat("image", image_slices),
776
+ image_sizes=MultiModalFieldConfig.batched("image"),
777
+ tgt_sizes=MultiModalFieldConfig.flat("image", image_slices),
778
+ image_num_slices=MultiModalFieldConfig.batched("image"),
779
+ image_embeds=MultiModalFieldConfig.flat("image", image_slices),
780
+ video_pixel_values=MultiModalFieldConfig.flat(
781
+ "video", video_slices),
782
+ video_image_sizes=MultiModalFieldConfig.batched("video"),
783
+ video_tgt_sizes=MultiModalFieldConfig.flat("video", video_slices),
784
+ video_embeds=MultiModalFieldConfig.flat("video", video_slices),
785
+ video_num_slices=MultiModalFieldConfig.batched("video"))
786
+
787
+ def apply(
788
+ self,
789
+ prompt: Union[str, List[int]],
790
+ mm_data: MultiModalDataDict,
791
+ hf_processor_mm_kwargs: Mapping[str, object],
792
+ ) -> MultiModalInputs:
793
+ supported_mm_modalities = self.info.get_supported_mm_modalities()
794
+ if isinstance(prompt, list):
795
+ prompt = self.info.get_tokenizer().decode(prompt)
796
+ matches = re.findall(self.get_placeholder_match_pattern(), prompt)
797
+ mm_orders = {
798
+ f"{modality}_orders":
799
+ torch.tensor(
800
+ [index for index, m in enumerate(matches) if m == modality])
801
+ for modality in supported_mm_modalities
802
+ }
803
+ result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
804
+ # Exclude <image_id>x</image_id> from placeholders
805
+ if "image" in result["mm_placeholders"] and \
806
+ self.info.get_model_version() == (2, 6):
807
+ result["mm_placeholders"]["image"] = [
808
+ PlaceholderRange(offset=p["offset"] + 3 + idx // 10,
809
+ length=p["length"] - 3 - idx // 10)
810
+ for idx, p in enumerate(result["mm_placeholders"]["image"])
811
+ ]
812
+ result["mm_kwargs"].update(**mm_orders)
813
+ result["mm_kwargs"].update(**self.get_special_tokens())
814
+ return result
815
816
817
class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
858
859
return get_sampler()
860
412
- def get_embedding(
861
+ def get_embedding_with_vision(
862
self,
863
input_ids: torch.Tensor,
864
image_inputs: Optional[MiniCPMVImageInputs],
920
image_end_tokens[:valid_image_nums].unsqueeze(-1),
921
])
922
474
- def _parse_and_validate_inputs(
923
+ def _parse_and_validate_image_inputs(
924
self,
925
input_ids: torch.Tensor,
926
**kwargs: object,
927
) -> Optional[MiniCPMVImageInputs]:
479
- pixel_values = kwargs.pop("pixel_values", [])
480
- tgt_sizes = kwargs.pop("tgt_sizes", [])
928
+ mm_data = {
929
+ "image": {
930
+ key: kwargs.pop(key, [])
931
+ for key in ["pixel_values", "tgt_sizes", "image_num_slices"]
932
+ },
933
+ "video": {
934
+ "pixel_values": kwargs.pop("video_pixel_values", []),
935
+ "tgt_sizes": kwargs.pop("video_tgt_sizes", []),
936
+ "video_num_slices": kwargs.pop("video_num_slices", [])
937
+ }
938
+ }
939
im_start_id = kwargs.pop("im_start_id", None)
940
im_end_id = kwargs.pop("im_end_id", None)
941
slice_start_id = kwargs.pop("slice_start_id", None)
942
slice_end_id = kwargs.pop("slice_end_id", None)
943
+ mm_orders = {
944
+ f"{modality}": kwargs.pop(f"{modality}_orders", None)
945
+ for modality in ["image", "video", "audio"]
946
+ }
947
+ batch_size = max(len(mm_data["image"]["pixel_values"]),
948
+ len(mm_data["video"]["pixel_values"]))
949
image_embeds = kwargs.pop("image_embeds", None)
486
-
950
+ video_embeds = kwargs.pop("video_embeds", None)
951
+ if image_embeds is not None and video_embeds is not None:
952
+ raise ValueError(
953
+ "Incorrect inputs for vision embeddings. "
954
+ "Image embeds and video embeds can not exist simultaneously.")
955
+ if video_embeds is not None:
956
+ image_embeds = video_embeds
957
if image_embeds is not None:
958
if not isinstance(image_embeds, (torch.Tensor, list)):
959
raise ValueError(f"Incorrect type of image embeds. "
960
f"Got type: {type(image_embeds)}")
491
- if isinstance(image_embeds, list):
492
- image_embeds = torch.concat(image_embeds)
961
+ image_embeds = torch.concat(
962
+ [image_embeds[i] for i in range(len(image_embeds))])
963
964
return MiniCPMVImageEmbeddingInputs(
965
image_bounds=self._get_image_bounds(input_ids, im_start_id,
968
data=image_embeds,
969
type="image_embeds",
970
)
501
-
502
- if not isinstance(pixel_values, (torch.Tensor, list)):
503
- raise ValueError("Incorrect type of pixel values. "
504
- f"Got type: {type(pixel_values)}")
505
-
506
- if not isinstance(tgt_sizes, (torch.Tensor, list)):
507
- raise ValueError("Incorrect type of target sizes. "
508
- f"Got type: {type(tgt_sizes)}")
509
-
510
- if len(pixel_values) != len(tgt_sizes):
511
- raise ValueError("Inconsistent batch lengths, found: "
512
- f"{len(pixel_values)} vs. {len(tgt_sizes)}")
971
+ for modality, modality_mm_data in mm_data.items():
972
+ if not isinstance(modality_mm_data["pixel_values"],
973
+ (torch.Tensor, list)):
974
+ raise ValueError(
975
+ "Incorrect type of pixel values. "
976
+ f"Got type: {type(modality_mm_data['pixel_values'])}")
977
+
978
+ if not isinstance(modality_mm_data["tgt_sizes"],
979
+ (torch.Tensor, list)):
980
+ raise ValueError(
981
+ "Incorrect type of target sizes. "
982
+ f"Got type: {type(modality_mm_data['tgt_sizes'])}")
983
+
984
+ if len(modality_mm_data["pixel_values"]) != len(
985
+ modality_mm_data["tgt_sizes"]):
986
+ raise ValueError(
987
+ "Inconsistent batch lengths, found: "
988
+ f"{len(modality_mm_data['pixel_values'])} vs. "
989
+ f"{len(modality_mm_data['tgt_sizes'])}")
990
991
pixel_values_flat: List[torch.Tensor] = []
992
tgt_sizes_flat: List[torch.Tensor] = []
516
- for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
517
- if len(pixel_b) != len(tgt_b):
518
- raise ValueError("Inconsistent N lengths, found: "
519
- f"{len(pixel_b)} vs {len(tgt_b)}")
520
-
521
- for pixel_n, tgt_n in zip(pixel_b, tgt_b):
522
- pixel_values_flat += pixel_n
523
- tgt_sizes_flat += tgt_n
993
+ for b in range(batch_size):
994
+ mm_counts = {"image": 0, "video": 0} if self.version == (2, 6) \
995
+ else {"image": 0}
996
+ mm_slice_counts = {"image": 0, "video": 0} \
997
+ if self.version == (2, 6) else {"image": 0}
998
+ mm_orders_b = [(index, modality) for modality in mm_counts
999
+ for index in mm_orders[modality][b]]
1000
+ for _, modality in sorted(mm_orders_b, key=lambda x: x[0]):
1001
+ pos = mm_counts[modality]
1002
+ num_slices = mm_data[modality][f"{modality}_num_slices"][b][
1003
+ pos]
1004
+ slice_start_idx = mm_slice_counts[modality]
1005
+ slice_end_idx = slice_start_idx + num_slices
1006
+ pixel_values_flat += mm_data[modality]["pixel_values"][b][
1007
+ slice_start_idx:slice_end_idx]
1008
+ tgt_sizes_flat += mm_data[modality]["tgt_sizes"][b][
1009
+ slice_start_idx:slice_end_idx]
1010
+ mm_counts[modality] += 1
1011
+ mm_slice_counts[modality] += num_slices
1012
1013
# NOTE: Input IDs does not contain image tokens during memory profiling,
1014
# so we allow it to be empty
1032
type="pixel_values",
1033
)
1034
1035
+ def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
1036
+ **kwargs: object):
1037
+ return self._parse_and_validate_image_inputs(input_ids, **kwargs)
1038
+
1039
def forward(
1040
self,
1041
input_ids: torch.Tensor,
1048
if intermediate_tensors is not None:
1049
vlm_embeddings = None
1050
else:
559
- image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs)
560
-
561
- vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
1051
+ image_inputs = \
1052
+ self._parse_and_validate_inputs(input_ids, **kwargs)
1053
+ vlm_embeddings, _ = self.get_embedding_with_vision(
1054
+ input_ids, image_inputs)
1055
1056
# always pass the input via `inputs_embeds`
1057
# to make sure the computation graph is consistent
1457
_SUPPORT_VERSION = {
1458
(2, 0): MiniCPMV2_0,
1459
(2, 5): MiniCPMV2_5,
967
- (2, 6): MiniCPMV2_6
1460
+ (2, 6): MiniCPMV2_6,
1461
}
1462
1463
971
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_minicpmv)
972
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
973
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
974
-@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
975
-class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1464
+@MULTIMODAL_REGISTRY.register_processor(
1465
+ MiniCPMVMultiModalProcessor,
1466
+ info=MiniCPMVProcessingInfo,
1467
+ dummy_inputs=MiniCPMVDummyInputsBuilder)
1468
+class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1469
"""
1470
Different versions of MiniCPMV use different visual encoders and LLMs,
1471
which is not conducive to the current integration logic of LoRA and